import struct
from collections import namedtuple
from collections.abc import MutableSequence

import numpy as np

from . import classes, lzw, util

@util.proxy('slots', 'properties', logical_screen_descriptor=classes.LogicalScreenDescriptor)
class GIF(MutableSequence):
    """
    Represents an animated GIF.

    This is an opinionated class. It assumes a lot of things about
    the GIF to be created: that it'll be animated, that it'll have
    both a global color table and delay time, etc.
    All assumed values are manually resettable, however.
    """
    def __init__(self,
      loop_count: int = 0,
      *,
      delay_time: int = 0,
      canvas_width: int = None,
      canvas_height: int = None
    ):
        self.header = classes.Header()
        self.logical_screen_descriptor = classes.LogicalScreenDescriptor(canvas_width, canvas_height)
        self.global_color_table = classes.ColorTable()
        self.netscape_looping_extension = classes.NetscapeApplicationExtension(loop_count)

        self.images = []
        self.global_delay_time = delay_time

    def __bytes__(self):
        if self:
            self[0].use_graphic_control_extension = True
        return b''.join(map(bytes, [
          self.header,
          self.logical_screen_descriptor,
          self.global_color_table,
          self.netscape_looping_extension,
          *self,
          b'\x3b'
        ]))

    def __getitem__(self, idx):
        return self.images.__getitem__(idx)

    def __setitem__(self, idx, value):
        if not isinstance(value, Frame):
            value = self.create_frame(value)
        self.update_dims(value.image_descriptor)
        self.images.__setitem__(idx, value)

    def __delitem__(self, idx):
        self.images.__delitem__(idx)

    def __len__(self):
        return self.images.__len__()

    def insert(self, idx, value):
        if not isinstance(value, Frame):
            value = self.create_frame(value)
        self.update_dims(value.image_descriptor)
        self.images.insert(idx, value)

    def create_frame(self, pixels, delay_time=None, *, transparent_color_index=None):
        use_graphic_control_extension = False

        if delay_time is None:
            delay_time = self.global_delay_time
        elif delay_time != self.global_delay_time:
            use_graphic_control_extension = True
        if transparent_color_index is None:
            transparent_color_index = self.global_color_table.transparent_color_index
        elif transparent_color_index != self.global_color_table.transparent_color_index:
            use_graphic_control_extension = True

        return Frame(
          pixels,
          self,
          delay_time=delay_time,
          transparent_color_index=transparent_color_index,
          use_graphic_control_extension=use_graphic_control_extension
        )

    def update_dims(self, image_descriptor):
        self.canvas_width = image_descriptor.width
        self.canvas_height = image_descriptor.height

    def update_color_table_size(self):
        self.global_color_table_size = self.global_color_table.size()

    def difference_compress(self):
        if not self:
            return
        prev = self[0]
        for cur in self[1:]:
            prev %= cur

@util.proxy('slots', image_descriptor=classes.ImageDescriptor)
@util.proxy('slots', 'properties', _graphic_control_extension=classes.GraphicControlExtension)
class Frame:
    def __init__(self,
      pixels,
      gif,
      *,
      use_graphic_control_extension=False,
      color_table=None,
      color_indices=None,
      delay_time=None,
      transparent_color_index=None,
    ):
        self.gif = gif
        if not isinstance(pixels, np.ndarray):
            pixels = np.array(pixels)
        self._color_table = color_table
        self.pixels = pixels
        self.colors = set(map(tuple, np.unique(self.pixels.reshape(-1, 3), axis=0)))
        self.update_color_table()
        if color_indices is None:
            _ctable = self.color_table
            color_indices = np.apply_along_axis(
              lambda color: _ctable[tuple(color)],
              2,
              pixels
            )
        self.color_indices = color_indices
        self.image_descriptor = classes.ImageDescriptor(*self.color_indices.shape)
        self._graphic_control_extension = classes.GraphicControlExtension(
          self.gif.global_delay_time if delay_time is None else delay_time,
          transparent_color_index
        )
        self.use_graphic_control_extension = use_graphic_control_extension or color_table is not None

    def __eq__(self, other):
        if not isinstance(other, Frame):
            return NotImplemented
        return (self.pixels == other.pixels).all(2)

    def __imod__(self, other):
        if not isinstance(other, Frame):
            return NotImplemented
        # TODO: use self.pixels and classes.ColorTable.TRANSPARENT here instead
        transparent = self.color_table.transparent_color_index
        self.color_indices[self == other] = transparent
        eq = self.color_indices == transparent
        # argmin() gets the first False value's index,
        # aka the number of starting Trues to erase
        cols, rows = eq.all(0).argmin(), eq.all(1).argmin()
        self.color_indices = self.color_indices[rows:, cols:]
        self.pixels = self.pixels[rows:, cols:]
        self.left = cols
        self.top = rows
        return self

    def __bytes__(self):
        ba = bytearray()
        if self.use_graphic_control_extension:
            ba.extend(bytes(self._graphic_control_extension))
        ba.extend(bytes(self.image_descriptor))
        ba.extend(lzw.compress(self.color_indices.flat, self.color_table))
        return bytes(ba)

    @property
    def color_table(self):
        if self._color_table is None:
            return self.gif.global_color_table
        return self._color_table

    @color_table.setter
    def color_table(self, value):
        self._color_table = value

    def update_color_table(self):
        self.color_table.extend(self.colors.difference(self.color_table.underlying))
        if self._color_table is None:
            self.gif.update_color_table_size()
        else:
            self.local_color_table_size = self.color_table.size


'''
    This material is based upon work supported by the
    Defense Advanced Research Projects Agency (DARPA)
    and Naval Information Warfare Center Pacific
    (NIWC Pacific) under Contract Number N66001-20-C-4024.

    The views, opinions, and/or findings expressed are
    those of the author(s) and should not be interpreted
    as representing the official views or policies of
    the Department of Defense or the U.S. Government.

    Distribution Statement "A" (Approved for Public Release,
    Distribution Unlimited)
'''

import copy, os, sys

sys.path.append(os.path.dirname(sys.path[0]))
from collections import defaultdict
from glob import glob
from pathlib import Path

import networkx as nx
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from tqdm import tqdm

class ACFGDataset:
    def __init__(self, pcode2vec="none"):
        self.func2idx = {}
        self.end2end_dataset = []
        self.pcode2vec = pcode2vec

        self.graphs = {}
        self.must_go_testset_pkg_list = ['ch05','ch06']

    def load_1_binary(self, data_path,num_features):
        contents = self._read_acfg_folder(data_path)
        thunk_func_names = self._get_all_thunk_functions(contents)

        tqdm_bar = tqdm(contents)
        for arch_pkg_bin in tqdm_bar:

            arch = arch_pkg_bin[0]
            pkg = arch_pkg_bin[1]
            bin = arch_pkg_bin[2]

            tqdm_bar.set_description("Processing %s, %s, %s" % (arch, pkg, bin))

            for archi_dir_path in contents[arch_pkg_bin]:
                bin_cg = nx.DiGraph()

                call_graph_edges = self._get_callgraph_edges(archi_dir_path/"callgraph.csv")
                #print("archi_dir_path: " + str(archi_dir_path))
                block2features, func2blocks = self._get_blockfeatures(archi_dir_path/"attributes.csv")
                block2blocks = self._get_acfg_edges(archi_dir_path/"edges.csv")

                if self.pcode2vec == 'bb':
                    block2features_pcode, missing_basic_blocks = self._get_pcodefeatures(archi_dir_path/"bb_pcode_attribute.csv", block2features)
                    block2features = block2features_pcode
                    # missing basic blocks are the blocks in bb_pcode_attribute.csv but not in attribute.csv.
                    self._error_handling_missing_blocks(missing_basic_blocks, func2blocks, block2features, block2blocks)

                elif self.pcode2vec == 'func':
                    func2features_pcode = self._get_func_pcodefeatures(archi_dir_path/"pcode_attribute.csv")
                    pass

                elif self.pcode2vec == 'bb_func':
                    block2features_pcode, missing_basic_blocks = self._get_pcodefeatures(archi_dir_path/"bb_pcode_attribute.csv", block2features)
                    block2features = block2features_pcode
                    # missing basic blocks are the blocks in bb_pcode_attribute.csv but not in attribute.csv.
                    self._error_handling_missing_blocks(missing_basic_blocks, func2blocks, block2features, block2blocks)
                    func2features_pcode = self._get_func_pcodefeatures(archi_dir_path/"pcode_attribute.csv")
                    pass

                else:
                    pass

                for func, blocks in func2blocks.items():
                    func_cfg = nx.DiGraph()
                    for block in blocks:
                        block_index = "%s.%s" % (func, block)
                        if block_index in block2features:
                            func_cfg.add_node(block_index, features=block2features[block_index])
                            for neighbor in block2blocks[block_index]:
                                func_cfg.add_edge(block_index, neighbor)

                    if 'func' in self.pcode2vec:
                        try:
                            bin_cg.add_node(func, features=func_cfg, type="normal", func_pcode_features=func2features_pcode[func])
                        except KeyError:
                            import pdb; pdb.set_trace()
                    else:
                        bin_cg.add_node(func, features=func_cfg, type="normal")

                feature_dim_bb = num_features+12 if 'bb' in self.pcode2vec else 12
                feature_dim_func = num_features if 'func' in self.pcode2vec else 0

                for from_edge, to_edge in call_graph_edges:
                    if not from_edge in bin_cg.nodes():
                        if from_edge in thunk_func_names[(arch, pkg, bin)]:
                            if 'func' in self.pcode2vec:
                                bin_cg.add_node(from_edge, features=nx.DiGraph(), type="normal", func_pcode_features=[0.0]*feature_dim_func)
                            else:
                                bin_cg.add_node(from_edge, features=nx.DiGraph(), type="normal")
                            bin_cg.nodes()[from_edge]['type'] = "thunk"
                            bin_cg.nodes()[from_edge]['features'].add_node("dummy", features=[0.0]*feature_dim_bb)

                    if not to_edge in bin_cg.nodes():
                        if to_edge in thunk_func_names[(arch, pkg, bin)]:
                            if 'func' in self.pcode2vec:
                                bin_cg.add_node(to_edge, features=nx.DiGraph(), type="normal", func_pcode_features=[0.0]*feature_dim_func)
                            else:
                                bin_cg.add_node(to_edge, features=nx.DiGraph(), type="normal")
                            bin_cg.nodes()[to_edge]['type'] = "thunk"
                            bin_cg.nodes()[to_edge]['features'].add_node("dummy", features=[0.0]*feature_dim_bb)

                    if from_edge in bin_cg.nodes() and to_edge in bin_cg.nodes():
                        bin_cg.add_edge(from_edge, to_edge)

                #### the process that filters the FUN_XXX functions.
                # call_graph.remove_nodes_from([x for x in call_graph.nodes() if "FUN_" in x])
                bin_cg.store_path = archi_dir_path
                # we should append func features roughly here to each bin_cg
                if arch not in self.graphs:
                    self.graphs[arch]={}
                if pkg not in self.graphs[arch]:
                    self.graphs[arch][pkg] = {}
                self.graphs[arch][pkg][bin] = bin_cg

        for arch, pkgs in self.graphs.items():
            for pkg, bins in pkgs.items():
                func_dict_in_pkg = {}
                for bin, cg in bins.items():
                    # import pdb; pdb.set_trace()
                    for func, func_attr in cg.nodes(data=True):
                        if func_attr['type'] != "thunk":
                            func_dict_in_pkg[func] = func_attr
                # import pdb; pdb.set_trace()
                for bin, cg in bins.items():
                    for func, func_attr in cg.nodes(data=True):
                        if func_attr['type'] == 'thunk' and func in func_dict_in_pkg:
                            func_attr["features"] = func_dict_in_pkg[func]['features']
                            if "func_pcode_features" in func_dict_in_pkg[func]:
                                func_attr["func_pcode_features"] = func_dict_in_pkg[func]['func_pcode_features']
                            func_attr["type"] = "external"

        # removed isolated nodes.
        for arch, pkgs in self.graphs.items():
            for pkg, bins in pkgs.items():
                for bin, cg in bins.items():
                    cg.remove_nodes_from(list(nx.isolates(cg)))

        for arch, pkgs in self.graphs.items():
            for pkg, bins in pkgs.items():
                for bin, cg in bins.items():
                    thunk_func = [func_name for func_name, func_attr in cg.nodes(data=True) if func_attr['type'] == "thunk"]
                    cg.remove_nodes_from(thunk_func)

        no_arch_func_names = set()
        for pkgs in self.graphs.values():
            for bins in pkgs.values():
                for cg in bins.values():
                    for func_name, func_attr in cg.nodes(data=True):
                        if func_attr['type'] != "thunk":
                            no_arch_func_names.add(func_name)

        no_arch_func_names.add("thunk")

        self.func2idx = {v:k for k, v in enumerate(list(no_arch_func_names))}
        self.idx2func = {v:k for k, v in self.func2idx.items()}

    def load(self, dataset_path, num_features, vis_only=False):
        contents = self._read_acfg_folders(dataset_path)
        thunk_func_names = self._get_all_thunk_functions(contents)

        tqdm_bar = tqdm(contents)
        for arch_pkg_bin in tqdm_bar:

            arch = arch_pkg_bin[0]
            pkg = arch_pkg_bin[1]
            bin = arch_pkg_bin[2]

            tqdm_bar.set_description("Processing %s, %s, %s" % (arch, pkg, bin))

            for archi_dir_path in contents[arch_pkg_bin]:
                bin_cg = nx.DiGraph()

                call_graph_edges = self._get_callgraph_edges(archi_dir_path/"callgraph.csv")
                #print("archi_dir_path: " + str(archi_dir_path))
                block2features, func2blocks = self._get_blockfeatures(archi_dir_path/"attributes.csv")
                block2blocks = self._get_acfg_edges(archi_dir_path/"edges.csv")

                if self.pcode2vec == 'bb':
                    block2features_pcode, missing_basic_blocks = self._get_pcodefeatures(archi_dir_path/"bb_pcode_attribute.csv", block2features)
                    block2features = block2features_pcode
                    # missing basic blocks are the blocks in bb_pcode_attribute.csv but not in attribute.csv.
                    try:
                        self._error_handling_missing_blocks(missing_basic_blocks, func2blocks, block2features, block2blocks)
                    except:
                        continue

                elif self.pcode2vec == 'func':
                    func2features_pcode = self._get_func_pcodefeatures(archi_dir_path/"pcode_attribute.csv")
                    pass

                elif self.pcode2vec == 'bb_func':
                    block2features_pcode, missing_basic_blocks = self._get_pcodefeatures(archi_dir_path/"bb_pcode_attribute.csv", block2features)
                    block2features = block2features_pcode
                    # missing basic blocks are the blocks in bb_pcode_attribute.csv but not in attribute.csv.
                    self._error_handling_missing_blocks(missing_basic_blocks, func2blocks, block2features, block2blocks)
                    func2features_pcode = self._get_func_pcodefeatures(archi_dir_path/"pcode_attribute.csv")
                    pass

                else:
                    pass

                for func, blocks in func2blocks.items():
                    func_cfg = nx.DiGraph()
                    for block in blocks:
                        block_index = "%s.%s" % (func, block)
                        if block_index in block2features:
                            func_cfg.add_node(block_index, features=block2features[block_index])
                            for neighbor in block2blocks[block_index]:
                                func_cfg.add_edge(block_index, neighbor)

                    if 'func' in self.pcode2vec:
                        try:
                            bin_cg.add_node(func, features=func_cfg, type="normal", func_pcode_features=func2features_pcode[func])
                        except KeyError:
                            import pdb; pdb.set_trace()
                    else:
                        bin_cg.add_node(func, features=func_cfg, type="normal")

                feature_dim_bb = num_features+12 if 'bb' in self.pcode2vec else 12
                feature_dim_func = num_features if 'func' in self.pcode2vec else 0

                for from_edge, to_edge in call_graph_edges:
                    if not from_edge in bin_cg.nodes():
                        if from_edge in thunk_func_names[(arch, pkg, bin)]:
                            if 'func' in self.pcode2vec:
                                bin_cg.add_node(from_edge, features=nx.DiGraph(), type="normal", func_pcode_features=[0.0]*feature_dim_func)
                            else:
                                bin_cg.add_node(from_edge, features=nx.DiGraph(), type="normal")
                            bin_cg.nodes()[from_edge]['type'] = "thunk"
                            bin_cg.nodes()[from_edge]['features'].add_node("dummy", features=[0.0]*feature_dim_bb)

                    if not to_edge in bin_cg.nodes():
                        if to_edge in thunk_func_names[(arch, pkg, bin)]:
                            if 'func' in self.pcode2vec:
                                bin_cg.add_node(to_edge, features=nx.DiGraph(), type="normal", func_pcode_features=[0.0]*feature_dim_func)
                            else:
                                bin_cg.add_node(to_edge, features=nx.DiGraph(), type="normal")
                            bin_cg.nodes()[to_edge]['type'] = "thunk"
                            bin_cg.nodes()[to_edge]['features'].add_node("dummy", features=[0.0]*feature_dim_bb)

                    if from_edge in bin_cg.nodes() and to_edge in bin_cg.nodes():
                        bin_cg.add_edge(from_edge, to_edge)

                #### the process that filters the FUN_XXX functions.
                # call_graph.remove_nodes_from([x for x in call_graph.nodes() if "FUN_" in x])
                bin_cg.store_path = archi_dir_path
                # we should append func features roughly here to each bin_cg
                if arch not in self.graphs:
                    self.graphs[arch]={}
                if pkg not in self.graphs[arch]:
                    self.graphs[arch][pkg] = {}
                self.graphs[arch][pkg][bin] = bin_cg

        for arch, pkgs in self.graphs.items():
            for pkg, bins in pkgs.items():
                func_dict_in_pkg = {}
                for bin, cg in bins.items():
                    # import pdb; pdb.set_trace()
                    for func, func_attr in cg.nodes(data=True):
                        if func_attr['type'] != "thunk":
                            func_dict_in_pkg[func] = func_attr
                # import pdb; pdb.set_trace()
                for bin, cg in bins.items():
                    for func, func_attr in cg.nodes(data=True):
                        if func_attr['type'] == 'thunk' and func in func_dict_in_pkg:
                            func_attr["features"] = func_dict_in_pkg[func]['features']
                            if "func_pcode_features" in func_dict_in_pkg[func]:
                                func_attr["func_pcode_features"] = func_dict_in_pkg[func]['func_pcode_features']
                            func_attr["type"] = "external"

        # removed isolated nodes.
        for arch, pkgs in self.graphs.items():
            for pkg, bins in pkgs.items():
                for bin, cg in bins.items():
                    cg.remove_nodes_from(list(nx.isolates(cg)))

        # "Visualizing Packing Dataset"
        for arch, pkgs in self.graphs.items():
            for pkg, bins in pkgs.items():
                for bin, cg in bins.items():
                    # print("Visualizing %s.%s.%s" % (arch, pkg, bin))
                    for func_name, func_attr in cg.nodes(data=True):
                        if func_attr['type'] == "thunk":
                            func_attr['fontcolor'] = "red"
                            func_attr['color'] = "red"
                        elif func_attr['type'] == "external":
                            func_attr['fontcolor'] = "blue"
                            func_attr['color'] = "blue"
                        else:
                            func_attr['fontcolor'] = "black"
                            func_attr['color'] = "black"
                    # parent_dirpath = (cg.store_path/"..").resolve()
                    # nx.drawing.nx_pydot.write_dot(cg, str(parent_dirpath) + "/%s.dot" % parent_dirpath.name)
                    # for func_name, func_attr in cg.nodes(data=True):
                    #     func_cfg = func_attr['features']
                    #     parent_folder_path = (cg.store_path/"..").resolve()
                    #     ccode_path = parent_folder_path / (parent_folder_path.name + "-ccode")
                    #     nx.drawing.nx_pydot.write_dot(func_cfg, str(ccode_path) + "/%s.dot" % func_name)

        if vis_only == True:
            return

        for arch, pkgs in self.graphs.items():
            for pkg, bins in pkgs.items():
                for bin, cg in bins.items():
                    thunk_func = [func_name for func_name, func_attr in cg.nodes(data=True) if func_attr['type'] == "thunk"]
                    cg.remove_nodes_from(thunk_func)

        no_arch_func_names = set()
        for pkgs in self.graphs.values():
            for bins in pkgs.values():
                for cg in bins.values():
                    for func_name, func_attr in cg.nodes(data=True):
                        if func_attr['type'] != "thunk":
                            no_arch_func_names.add(func_name)

        '''for pkgs in self.graphs.values():
            for bins in pkgs.values():
                for cg in bins.values():'''
                    #import pdb; pdb.set_trace()

        no_arch_func_names.add("thunk")

        self.func2idx = {v:k for k, v in enumerate(list(no_arch_func_names))}
        self.idx2func = {v:k for k, v in self.func2idx.items()}

    def pack4cfg2vec(self):
        from torch_geometric.utils.convert import from_networkx
        self.end2end_dataset = []
        for arch, pkgs in self.graphs.items():
            for pkg, bins in pkgs.items():
                for bin, cg in bins.items():
                    data_obj = from_networkx(cg)
                    features = []
                    for feature in data_obj.features:
                        data = from_networkx(feature)
                        data.edge_index = data.edge_index.long()
                        try:
                            data.features = data.features.float()
                        except:
                            print(pkg, bin, arch)
                        features.append(data)
                    data_obj.features =  features
                    # data_obj.func_features = xxx
                    data_obj.archi = arch
                    data_obj.package = "%s.%s" % (pkg, bin)
                    function_names = []
                    function_labels = []
                    for func_name, func_attr in cg.nodes(data=True):
                        if func_attr['type'] == "thunk":
                            function_names.append("thunk")
                            function_labels.append(self.func2idx["thunk"])
                        else:
                            function_names.append(func_name)
                            function_labels.append(self.func2idx[func_name])
                    data_obj.function_names = function_names
                    data_obj.function_labels = function_labels
                    self.end2end_dataset.append(data_obj)

    def pack4gemini(self):
        import torch
        from core.utils import read_graph_allstar
        func2graphs = {}
        for arch, pkgs in self.graphs.items():
            for pkg, bins in pkgs.items():
                for bin, cg in bins.items():
                    for func_name, func_attr in cg.nodes(data=True):
                        func_acfg = func_attr["features"]

                        node_info = dict(func_acfg.nodes("features"))
                        node_names = list(node_info.keys())
                        features = torch.from_numpy(np.vstack(node_info.values())).float()

                        source_edges = []
                        target_edges = []
                        for edge in func_acfg.edges:
                            source_edges.append(node_names.index(edge[0]))
                            target_edges.append(node_names.index(edge[1]))

                        # the shape of A is expected to be [2, num_edges]
                        if len(source_edges) > 0:
                            A = torch.transpose(torch.LongTensor(list(zip(source_edges, target_edges))), 0, 1) # creating A (initial node adjancency information)
                        else:
                            A = torch.empty(size=[2,0], dtype=torch.long)

                        if func_attr['type'] == "thunk":
                            label = self.func2idx["thunk"]
                        else:
                            label = self.func2idx[func_name]
                        func2graphs[(arch, pkg, bin, func_name)] = (features, A, node_names, label)
        print("Converting Allstar data to Gemini format")
        self.Gs, self.classes, self.packages = read_graph_allstar(func2graphs, self.func2idx)

    def split_dataset_by_package(self, test_size, seed):
        ## add-in must go testset pkg list.
        package_list = np.unique([x.package.split('.')[0] for x in self.end2end_dataset if x.package.split('.')[0] not in self.must_go_testset_pkg_list]).tolist()
        np.random.seed(seed)
        perm=np.random.permutation(len(package_list))
        train_package_set = []
        test_package_set = []
        for pkg_idx in perm[0:int(len(package_list)*(1-test_size))]:
            train_package_set.append(package_list[pkg_idx])
        for pkg_idx in perm[int(len(package_list)*(1-test_size)):]:
            test_package_set.append(package_list[pkg_idx])
        # train_package_set, test_package_set = train_test_split(package_list, test_size=test_size, \
        #                                                        shuffle=True, random_state=seed)
        train_set = []
        test_set = []
        eval_set = []
        for data_point in self.end2end_dataset:
            pkg_name = data_point.package.split('.')[0]
            if pkg_name in train_package_set:
                train_set.append(data_point)
            elif pkg_name in test_package_set:
                test_set.append(data_point)
            elif pkg_name in self.must_go_testset_pkg_list:
                eval_set.append(data_point)

        print("Spliting dataset into Training/Testing/Eval sets (%d, %d, %d)" % (len(train_set), len(test_set), len(eval_set)))
        return train_set, test_set, eval_set

    def split_dataset_by_binary(self, test_size, seed):
        ## add-in must go testset pkg list.
        package_list = np.unique([x.package for x in self.end2end_dataset]).tolist()
        train_package_set, test_package_set = train_test_split(package_list, test_size=test_size, \
                                                               shuffle=True, random_state=seed)
        train_set = []
        test_set = []
        for data_point in self.end2end_dataset:
            if data_point.package in train_package_set:
                train_set.append(data_point)
            elif data_point.package in test_package_set:
                test_set.append(data_point)

        print("Spliting dataset into Training/Testing sets (%d, %d)" % (len(train_set), len(test_set)))
        return train_set, test_set

    def _get_all_thunk_functions(self, contents):
        thunk_func_names = defaultdict(list)
        for arch_pkg_bin in contents:
            for arch_dir_path in contents[arch_pkg_bin]:
                parent_folder_path = (arch_dir_path/"..").resolve()
                ccode_path = parent_folder_path / (parent_folder_path.name + "-ccode")
                for code_text_path in glob("%s/**" % ccode_path):
                    if "_thunk" ==  Path(code_text_path).stem[-6:]:
                        thunk_func_names[arch_pkg_bin].append(Path(code_text_path).stem[:-6])

        # ex:  'i386.adns-tools.libadns.so.1.5': ['__assert_fail',
        #                                         'realloc',
        #                                         '_ITM_registerTMCloneTable']

        return thunk_func_names

    def _read_acfg_folders(self, dataset_path):
        ''' acfg_folders is structured using package name as the key a list of containing acfg folders as content'''
        acfg_folders = defaultdict(list)

        for folder_1st_level in glob("%s/**" % str(dataset_path)):
            for folder_2nd_level in glob("%s/**" % str(folder_1st_level)):
                acfg_folder_name = Path(folder_2nd_level).name
                if acfg_folder_name.endswith("acfg"):
                    package_binary = "-".join(acfg_folder_name.split("-")[:-2])
                    archi = acfg_folder_name.split("-")[-2].split(".")[0]
                    package = package_binary.split("___")[0]
                    bin = package_binary.split("___")[1]
                    acfg_folders[(archi,package,bin)].append(Path(folder_2nd_level))
        return acfg_folders

    def _read_acfg_folder(self, dataset_path):
        ''' acfg_folders is structured using package name as the key a list of containing acfg folders as content'''
        acfg_folders = defaultdict(list)
        for folder_2nd_level in glob("%s/**" % str(dataset_path)):
            acfg_folder_name = Path(folder_2nd_level).name
            if acfg_folder_name.endswith("acfg"):
                package_binary = "-".join(acfg_folder_name.split("-")[:-2])
                archi = acfg_folder_name.split("-")[-2].split(".")[0]
                package = package_binary.split("___")[0]
                bin = package_binary.split("___")[1]
                acfg_folders[(archi,package,bin)].append(Path(folder_2nd_level))
        return acfg_folders

    def _get_callgraph_edges(self, callgraph_edge_file_path):
        call_graph_edges = list()
        with open(str(callgraph_edge_file_path), "r") as fin:
            for line in fin.readlines():
                content = line.strip().split(", ")
                if (not (content[1], content[2]) in call_graph_edges):
                    call_graph_edges.append((content[1], content[2]))
        return call_graph_edges

    def _get_blockfeatures(self, block2feature_file_path):
        block2features = {}
        func2blocks = defaultdict(list)

        with open(str(block2feature_file_path), "r") as fin:
            for line in fin.readlines():
                if line.strip() != "":
                    data_row = line.strip().split(",")
                    func_name = data_row[0] # currently not used.
                    block_name = data_row[1]
                    block_feature = data_row[2:]
                    block_index = "%s.%s" % (func_name, block_name)
                    features = np.array(list(map(int, block_feature)))
                    block2features[block_index] = features
                    func2blocks[func_name].append(block_name)
        return block2features, func2blocks

    def _get_pcodefeatures(self, pcode_file_path, block2features):

        curr_block2features = copy.deepcopy(block2features)
        with open(str(pcode_file_path), "r") as fin:
            for line in fin.readlines():
                func_name = line.strip().split(",")[0]
                block_name = line.strip().split(", ")[1]
                block_index = "%s.%s" % (func_name, block_name)

                if block_name == "" or curr_block2features[block_index].size>12:
                    # there can be some lines without block names.
                    continue

                block_pcode_feature = line.strip().split(", ")[2:]
                features = np.array(list(map(float, block_pcode_feature)))
                curr_block2features[block_index] = np.concatenate([curr_block2features[block_index].astype(np.float32), features])

        missing_basic_blocks = []
        for block_name, block_feature in curr_block2features.items():
            try:
                if block_feature.size == 12:
                    missing_basic_blocks.append(block_name)
                    print(pcode_file_path, block_name, block_feature)
            except ValueError:
                import pdb;pdb.set_trace()

        return curr_block2features, missing_basic_blocks

    def _get_func_pcodefeatures(self, pcode_file_path):

        block2features = {}
        with open(str(pcode_file_path), "r") as fin:
            for line in fin.readlines():

                func_name = line.strip().split(",")[0]
                func_pcode_feature = line.strip().split(",")[1:]
                features = np.array(list(map(float, func_pcode_feature)))
                block2features[func_name] = features

        return block2features

    def _get_acfg_edges(self, edge_file_path):
        block2blocks = defaultdict(list)
        with open(str(edge_file_path), "r") as fin:
            for line in fin.readlines():
                if line.strip():
                    func_name = line.strip().split(",")[0]
                    from_block, to_block = line.strip().split(",")[1:]
                    from_block_index = "%s.%s" % (func_name.strip(), from_block.strip())
                    to_block_index = "%s.%s" % (func_name.strip(), to_block.strip())
                    block2blocks[from_block_index].append(to_block_index)
        return block2blocks

    def _error_handling_missing_blocks(self, missing_bbs, func2blocks, block2features, block2blocks):
        for neighbor_list in block2blocks.values():
            for missed_basic_block in missing_bbs:
                if missed_basic_block in neighbor_list:
                    neighbor_list.remove(missed_basic_block)

        for blocks in func2blocks.values():
            for missed_basic_block in missing_bbs:
                if missed_basic_block in blocks:
                    blocks.remove(missed_basic_block)

        for missed_basic_block in missing_bbs:
            del block2blocks[missed_basic_block]

        for missed_basic_block in missing_bbs:
            del block2features[missed_basic_block]

    def __str__(self):
        num_acfgs = sum([len(acg.function_names) for acg in self.end2end_dataset])
        all_acfg_nodes = 0
        all_acfg_edges = 0

        for package in self.end2end_dataset:
            for function in package.features:
                all_acfg_nodes += function.features.shape[0]
                all_acfg_edges += function.edge_index.shape[1]

        results=[]
        results.append("number of ACFGs: {}".format(num_acfgs))
        results.append("number of packages/ACGs: {}".format(len(self.end2end_dataset)))
        results.append("average number of edges in an ACFG: {0:f}".format(all_acfg_edges / num_acfgs))
        results.append("average number of nodes in an ACFG: {0:f}".format(all_acfg_nodes / num_acfgs))
        results.append("number of unique labels in dataset: {}".format(len(self.func2idx.values())))

        return "\n".join(results)

    def vis_distribution(self, metrics_path):
        metrics_path.mkdir(parents=True, exist_ok=True)
        num_acfgs = sum([len(acg.function_names) for acg in self.end2end_dataset])
        acg_num_nodes = []
        acg_num_edges = []
        acfg_num_nodes = []
        acfg_num_edges = []
        acfg_num_instructions = []
        for package in self.end2end_dataset:
            acg_num_nodes.append(len(package.function_labels))
            acg_num_edges.append(package.edge_index.shape[1])
            for function in package.features:
                num_instructions = []
                acfg_num_nodes.append(function.features.shape[0])
                acfg_num_edges.append(function.edge_index.shape[1])
                for i in range (0, len(function.features[0])):
                    num_instructions.append(int(function.features[0][i].item()))
                acfg_num_instructions.append(num_instructions)

        acg_df = pd.DataFrame(columns=["acg nodes", "acg edges"])
        acg_df["acg nodes"] = acg_num_nodes
        acg_df["acg edges"] = acg_num_edges
        acg_df.to_csv(metrics_path / ('allstar-%d-acg-distributions.csv'% num_acfgs))

        acfg_df = pd.DataFrame(columns=["acfg nodes", "acfg edges"])
        acfg_df["acfg nodes"] = acfg_num_nodes
        acfg_df["acfg edges"] = acfg_num_edges
        acfg_df.to_csv(metrics_path / ('allstar-%d-acfg-distributions.csv'% num_acfgs))

        acfg_instr_df_filename = metrics_path / ('allstar-%d-acfg-instr-distributions.csv'% num_acfgs)
        acfg_instr_df = pd.DataFrame(columns=["total", "arithmetic", "logic", "transfer", "call", "datatransfer", "ssa", "compare", "pointer", "other", "totalConstants", "totalStrings"])
        acfg_instr_df.to_csv(acfg_instr_df_filename)
        acfg_instr_df["acfg nodes"] = acfg_num_nodes
        acfg_instr_df["acfg edges"] = acfg_num_edges
        for function in acfg_num_instructions:
            acfg_instr_df = pd.DataFrame(data=[function]).to_csv(acfg_instr_df_filename, mode='a', header=False)


#! /usr/bin/python3

import logging
import sys
from urllib.parse import urlparse
import socket

from http_browser.Browser import Browser
from http_browser.HttpServerMessage import HTTP_STATUS
from tcp.Exceptions import ClosedSocketException

__author__ = "njhazelh"

logging.basicConfig(
    format='[%(asctime)s] {%(filename)-20s:%(lineno)-3d} %(levelname)s - %(message)s',
    datefmt='%I:%M:%S')
log = logging.getLogger("rawhttpget")

def main(argv):
    """
    Get the resource from the internet.
    :param argv: The commmand line arguments parsed into a handy map.
    """
    dest, path, filename = parse_input(args.url)
    browser = Browser()
    print("Getting %s from %s" % (path, dest))
    try:
        response = browser.get(path, dest=dest)
    except ConnectionRefusedError:
        print("Could not connect to server")
        sys.exit()
    except ClosedSocketException:
        print("Socket was open but closed")
        sys.exit()
    print("Saving response to %s" % (filename))
    saveResponse(response, filename)
    browser.close()

def parse_input(url):
    """
    Convert the user input url into something we can use.

    If the url doesn't have a file name, use index.html

    :param url: The url to convert
    :return: A domain/port pair, the url, and the filename to save under.
    """
    if not url.startswith('http'):
        url = '%s%s' % ('http://', url)
    parsed = urlparse(url)

    # validate url
    if parsed.netloc == '':
        log.critical("domain name must be provided. Received %s", parsed)
        sys.exit(0)
    try:
        socket.gethostbyname(parsed.netloc)
    except OSError:
        print("Could not find IP address for domain name")
        sys.exit(0)

    domain = parsed.netloc
    port = 80

    # calculate filename to save result as
    if parsed.path == '' or parsed.path[-1] == "/":
        filename = "index.html"
    else:
        filename = parsed.path.split("/")[-1]

    return (domain, port), url, filename

def saveResponse(response, filename):
    """
    Save the response to a file.
    :param response: The HTTP Response object.
    :param filename: The name to save the data under
    """
    if response.status_code != HTTP_STATUS.OK:
        print("Failed to load resource. Status Code = %d" % (response.status_code))
    else:
        data = response.body
        file = open(filename, 'wb')
        file.write(data)

if __name__ == "__main__":
    import argparse

    parser = argparse.ArgumentParser(description="Basically wget")
    parser.add_argument("url", help="The url to get.  Will use filename from resource or index.html if it's a folder")
    parser.add_argument("--verbosity", "-v", default="CRITICAL",
                        choices=["DEBUG", "INFO", "WARN", "ERROR", "CRITICAL"],
                        help="The verbosity of the output.  Defaults to CRITICAL.")
    args = parser.parse_args()
    log.setLevel(args.verbosity)
    main(args)


#!/usr/bin/env python3

import requests

def upload_data(target, data, timeout=None):
    # first upload the data
    r = requests.put(f"https://{target}/update/stage", data=data, verify=False, timeout=timeout)
    print(r.content)

    # then trigger the install
    r = requests.post(f"https://{target}/update/install", json={"appControlMode":"Auto"}, verify=False, timeout=timeout)
    print(r.content)

if __name__ == "__main__":
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument("image", type=argparse.FileType("rb"))
    parser.add_argument("--target", default="192.168.35.2")
    args = parser.parse_args()

    upload_data(args.target, args.image.read())


import numpy as np
import matplotlib.pyplot as plt

n = m = 10
N = 17
gi = []
hi = []
gi_help = []
hi_help = []

for ind in range(1, 11):
    gi_res = 1.5 + (1 / N) * np.sin(ind)
    hi_res = 1.5 + (1 / N) * np.cos(ind)
    gi.append(gi_res)
    hi.append(hi_res)
    gi_help.append([ind, gi_res])
    hi_help.append([ind, hi_res])

print('gi ', gi_help)
print('hi ', hi_help)

gi.sort()
hi.sort(reverse=True)
print('gi ', gi)
print('hi ', hi)

y = [y * 2 for y in range(10)]
plt.step(gi, y)
plt.step(hi, y)
plt.grid()
plt.show()

Maga1Sem/Modeling of information processes/Semestrovaya 2.py


from unittest import TestCase, main as unittest_main, mock
from bson.objectid import ObjectId
from app import app

sample_game_id = ObjectId('5d55cffc4a3d4031f42827a3')
sample_game = {
    'title': 'Cat Images',
    'price': 'Cats acting weird',
    'images': 'https://upload.wikimedia.org/wikipedia/commons/e/ee/Grumpy_Cat_by_Gage_Skidmore.jpg'

}
sample_form_data = {
    'title': sample_game['title'],
    'price': sample_game['price'],
    'images': '\n'.join(sample_game['images'])
}

class gamesTests(TestCase):
    """Flask tests."""

    def setUp(self):
        """Stuff to do before every test."""

        # Get the Flask test client
        self.client = app.test_client()

        # Show Flask errors that happen during tests
        app.config['TESTING'] = True

    def test_index(self):
        """Test the games homepage."""
        result = self.client.get('/')
        self.assertEqual(result.status, '200 OK')
        self.assertIn(b'game', result.data)

    def test_new(self):
        """Test the new game creation page."""
        result = self.client.get('/games/new')
        self.assertEqual(result.status, '200 OK')
        self.assertIn(b'New game', result.data)

    @mock.patch('pymongo.collection.Collection.find_one')
    def test_show_game(self, mock_find):
        """Test showing a single game."""
        mock_find.return_value = sample_game

        result = self.client.get(f'/games/{sample_game_id}')
        self.assertEqual(result.status, '200 OK')
        self.assertIn(b'Cat images', result.data)

    @mock.patch('pymongo.collection.Collection.find_one')
    def test_edit_game(self, mock_find):
        """Test editing a single game."""
        mock_find.return_value = sample_game

        result = self.client.get(f'/games/{sample_game_id}/edit')
        self.assertEqual(result.status, '200 OK')
        self.assertIn(b'Cat images', result.data)

    @mock.patch('pymongo.collection.Collection.insert_one')
    def test_submit_game(self, mock_insert):
        """Test submitting a new game."""
        result = self.client.post('/games', data=sample_form_data)

        # After submitting, should redirect to that game's page
        self.assertEqual(result.status, '302 FOUND')
        mock_insert.assert_called_with(sample_game)

    @mock.patch('pymongo.collection.Collection.update_one')
    def test_update_game(self, mock_update):
        result = self.client.post(
            f'/games/{sample_game_id}', data=sample_form_data)

        self.assertEqual(result.status, '302 FOUND')
        mock_update.assert_called_with({'_id': sample_game_id}, {
                                       '$set': sample_game})

    @mock.patch('pymongo.collection.Collection.delete_one')
    def test_delete_game(self, mock_delete):
        form_data = {'_method': 'DELETE'}
        result = self.client.post(
            f'/games/{sample_game_id}/delete', data=form_data)
        self.assertEqual(result.status, '308 PERMANENT REDIRECT')
        mock_delete.assert_called_with({'_id': sample_game_id})

if __name__ == '__main__':
    unittest_main()


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import pandas as pd

# In[2]:

p = pd.date_range(start ='2020-05-01', periods=7)
print(p)

# ###https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#offset-aliases
# Frequency 옵션들

# In[4]:

p = pd.date_range(start='2020-01-01', periods = 4, freq='2D') # D == Day
print(p)

# In[6]:

p = pd.date_range(start='2021-04-01', periods=5, freq='W') # W == Week
print(p)

# In[7]:

p = pd.date_range(start='2021-02-01', periods=12, freq='2BM') #BM == Business Month, 주의 마지막 평일까지
print(p)

# In[9]:

p = pd.date_range(start='2021-01-03', periods=5, freq='QS') # QS == 분기의 시작일
print(p)

# In[10]:

p = pd.date_range(start='2021-01-01', periods=3, freq='AS') # AS == 매 년의 시작일
print(p)

# In[12]:

p = pd.date_range(start='2021-01-01 08:00', periods=10, freq='H') # H == hour, 시간단위로 period 만큼
print(p)

# In[13]:

p = pd.date_range(start='2021-01-01 08:00', periods=10, freq='BH') # BH == Business hour
print(p)

# In[17]:

print(pd.offsets.BusinessHour())

# In[19]:

import datetime

bh = pd.offsets.BusinessHour(start='09:00', end=datetime.time(18,0)) # 임의로 BH의 Range 를 정의해서 사용할 수 있다.
p2 = pd.date_range(start='2021-01-01 08:00', periods=10, freq='BH') +bh
print(p2)

# In[20]:

import warnings
warnings.simplefilter(action='ignore', category=pd.errors.PerformanceWarning) # 위의 performanceWarning 잡아주는것

# In[25]:

p = pd.date_range(start='2021-01-01 10:00', periods=4, freq='30min')
print(p)

# In[28]:

index_date = pd.date_range(start='2021-04-01', periods=5, freq='D')
p= pd.Series([51, 63, 59, 48, 60], index = index_date) # 날짜에 인덱스를 지정해줌
print(p)

# ## DataFrame

# df = pd.DataFrame(data, [,index=index_data, columns = columns_data])
# data : 리스트, 딕셔너리, Numpy배열, Series, DataFrame
# DataFrame의 세로축은 index, 가로축은 columns 라함
# index 와 columns 를 제외한 영역은 values 라함
# index와 columns의 기본값은 숫자 0 부터 입력

# In[30]:

df = pd.DataFrame([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
print(df)

# In[31]:

import numpy as np

# In[32]:

data_list = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) # 위의 DataFrame 을 Numpy 배열로 만든것
df = pd.DataFrame(data_list)
print(df)

# In[34]:

data = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12]])
index_date = pd.date_range('2020-05-01', periods=4)
columns_list = ['A', 'B', 'C']
df = pd.DataFrame(data, index = index_date, columns = columns_list)
print(df)

# In[36]:

# DataFrame 을 딕셔너리로 생성하면 key 가 columns 가 된다.
data = {'연도': [2015, 2016, 2016, 2017, 2018], '지사': ['서울','서울','부산','서울','부산'],
       '고객 수 ': [200, 250, 150, 300, 200]} #
df = pd.DataFrame(data)
print(df)

# In[39]:

data = {'연도': [2015, 2016, 2016, 2017, 2018], '지사': ['서울','서울','부산','서울','부산'],
       '고객 수': [200, 250, 150, 300, 200]}
df = pd.DataFrame(data, columns=['지사', '연도', '고객 수']) # 데이터를 건드리는것은 좋지않으므로 컬럼의 이름을줘서 추출
print(df)

# In[42]:

s1 = pd.Series([1, 2, 3, 4, 5])
s2 = pd.Series([10, 20, 30, 40, 50])

print(s1+s2)
print(s1-s2)
print(s1/s2)
print(s1*s2)

# In[43]:

s3 = pd.Series([1,2,3,4])
s4 = pd.Series([10, 20 ,30, 40, 50])
print(s3+s4)

# In[44]:

data1 = {'A': [1, 2, 3, 4, 5],
         'B': [10, 20, 30, 40, 50],
         'C': [100, 200, 300, 400, 500]}
df1 = pd.DataFrame(data1)
print(df1)

data2 = {'A': [6, 7, 8],
         'B': [60, 70, 80],
         'C': [600, 700, 800]}
df2 = pd.DataFrame(data2)
print(df2)

# In[45]:

print(df1 + df2)

# In[51]:

table_data3 = {'봄': [256.5, 264.3, 215.9, 223.2, 312.8],
             '여름': [770.6, 567.5, 599.8, 387.1, 446.2],
             '가을': [363.5, 231.2, 293.1, 247.7, 381.6],
             '겨울': [139.3, 59.9, 76.9, 109.1, 108.1]}
columns_list = ['봄', '여름', '가을', '겨울']
index_list = ['2012', '2013', '2014', '2015', '2016']

# In[52]:

df3 = pd.DataFrame(table_data3, columns = columns_list, index=index_list)
print(df3)

# In[53]:

print(df3.mean()) # 계절별 평균 --> columns
print('=' * 30)
print(df3.std()) # 계절별 표준편차

# In[55]:

print(df3.mean(axis=1)) # 연도별 --> index
print('=' * 30)
print(df3.std(axis=1))

# In[56]:

print(df3.describe()) # 전체를 다 표현해줌

# ## data slicing
#

# In[57]:

KTX_data = {'경부선 KTX': [39060, 39896, 42005, 43621, 41702, 41266, 32427],
           '호남선 KTX': [7313, 6967, 6873, 6626, 8675, 10622, 9228],
           '경전선 KTX': [3627, 4168, 4088, 4424, 4606, 4984, 5570],
           '전라선 KTX': [309, 1771, 1954, 2244, 3146, 3945, 5766],
           '동해선 KTX': [np.nan,np.nan, np.nan, np.nan, 2395, 3786, 6667]}
columns_list = ['경부선 KTX', '호남선 KTX', '경전선 KTX', '전라선 KTX', '동해선 KTX']
index_list = ['2011', '2012', '2013', '2014', '2015', '2016', '2017']

# In[58]:

df_KTX = pd.DataFrame(KTX_data, columns = columns_list, index = index_list)
print(df_KTX)

# ## 전체 데이터 중 처음이나 끝 일부분 선택
#
#   - head([n])
#   - tail([n])
#   - n 을 지정하지 않으면 기본값 5

# In[62]:

print(df_KTX.head())
print('='*30)
print(df_KTX.head(3))

# In[63]:

#slicing
print(df_KTX[2:5])

# In[64]:

# 인덱스 지정(행 선택)
print(df_KTX.loc['2011'])

# In[66]:

print(df_KTX.loc['2013':'2016'])

# In[67]:

print(df_KTX['경부선 KTX'][1:4]) #.loc 안쓰고 컬럼 추출

# In[68]:

print(df_KTX.loc['2016']['호남선 KTX'])
print('='*30)
print(df_KTX.loc['2016', '호남선 KTX'])
print('='*30)
print(df_KTX['호남선 KTX']['2016'])
print('='*30)
print(df_KTX['호남선 KTX'][5])
print('='*30)
print(df_KTX['호남선 KTX'].loc['2016'])
print('='*30)

# In[69]:

print(df_KTX.T) # 데이터 프레임을 행렬로 인식해서 만들어줌 (행과, 열을 바꿔줌)

# In[71]:

print(df_KTX[['동해선 KTX', '전라선  KTX', '경전선 KTX', '호남선  KTX', '경부선  KTX']])

# In[72]:

## 데이터 프레임 통합

# In[73]:

df1 = pd.DataFrame({'Class1': [95, 92 ,98, 100],
                    'Class2': [91,93,97,99]})
print(df1)

# In[74]:

df2 = pd.DataFrame({'Class1': [87, 89],
                    'Class2': [85, 90]})

# In[75]:

print(df1.append(df2)) # 같은 key 끼리 묶는거 append

# In[76]:

print(df1.append(df2, ignore_index=True)) # append로 합쳤을떄의 인덱스를 순서로 조정

# In[77]:

df3 = pd.DataFrame({'Class1': [96, 83]})

# In[78]:

print(df2.append(df3, ignore_index=True))

# In[79]:

df4 = pd.DataFrame({'Class3': [93, 91, 96, 98]})

# In[80]:

print(df1.join(df4)) # 옆으로 붙이는거 조인 (같은 index기준)

# In[81]:

index_label = ['a', 'b','c','d']
df1a = pd.DataFrame({'Class1': [95, 92, 98, 100],
                    'Class2': [91,93,95,99]}, index=index_label)
df4a = pd.DataFrame({'Class3': [92, 93, 94, 95]}, index=index_label)

print(df1a.join(df4a))

# In[82]:

df5 = pd.DataFrame({'Class4': [82, 92]})
print(df1.join(df5))

# In[83]:

df_A_B = pd.DataFrame({'판매월': ['1월', '2월', '3월', '4월'],
                      '제품A': [100, 150, 200, 130],
                      '제품B': [90, 110, 140, 170]})

df_C_D = pd.DataFrame({'판매월': ['1월', '2월', '3월', '4월'],
                      '제품C': [112, 141, 203, 134],
                      '제품D': [90, 110, 140, 170]})

# In[84]:

# 판매 월 이라는 동일한 컬럼이 겹쳐서 에러가남
print(df_A_B.join(df_C_D))

# In[87]:

#동일한 컬럼으로 조인 시킴
print(df_A_B.merge(df_C_D))
#append와의 차이
print(df_A_B.append(df_C_D))

# In[88]:

df_left = pd.DataFrame({'key': ['A', 'B','C'], 'left':[1, 2, 3]})
df_right = pd.DataFrame({'key': ['A','B','D'], 'right':[4, 5, 6]})
print(df_left, df_right)

# In[89]:

print(df_left.merge(df_right, how='left', on='key')) #기준을 잡아서 머지하는 것

# In[90]:

print(df_left.merge(df_right, how='outer', on='key')) # 양쪽 다

# In[91]:

print(df_left.merge(df_right, how='inner', on='key')) # 겹치는 부분만

# In[99]:

df = pd.read_csv('/Users/user/java-web-project0208/python-basic/data/sea_rain1.csv')

# In[96]:

print(df)

# In[100]:

df_WH = pd.DataFrame({'Weight': [62, 67, 55, 74],
                      'Height':[165, 177,160,180]},
                    index=['ID_1', 'ID_2', 'ID_3', 'ID_4' ])
df_WH.index.name = 'User'
print(df_WH)

# In[101]:

bmi = df_WH['Weight']/(df_WH['Height']/100) **2
print(bmi)

# In[102]:

df_WH['BMI'] = bmi
print(df_WH)

# In[103]:

df_WH.to_csv('/Users/user/java-web-project0208/python-basic/data/save_DataFrame.csv')

# In[104]:

import matplotlib.pyplot as plt

# In[105]:

data1 = [10, 14, 19, 20, 25]
plt.plot(data1)
plt.show()

# In[108]:

# 별도의 창으로 띄워줌
# %matplotlib auto

#jupyter 에서 쓸때
get_ipython().run_line_magic('matplotlib', 'inline')

# In[109]:

data1 = [10, 14, 19, 20, 25]
plt.plot(data1)
plt.show()

# In[112]:

x = np.arange(-4.5, 5, 0.5) # 범위 -4.5~ 5 0.5씩증가
y= 2*x**2
plt.plot(x, y)
plt.show()

# In[113]:

x = np.arange(-4.5, 5, 0.5)
y1 = 2*x**2
y2 = 5*x +30
y3 = 4*x**2 +10

# In[115]:

plt.plot(x, y1)
plt.plot(x, y2)
plt.plot(x, y3)
plt.show()

# In[116]:

plt.plot(x, y1, x, y2, x, y3)
plt.show()

# In[118]:

x = np.arange(-5, 5, 0.1)
y1 = x **2 -2
y2 = 20*np.cos(x) **2

plt.figure(1) # 1번 그래프 창을 생성합
plt.plot(x, y1) # 지정된 그래프 창에 그래프를 그림

plt.figure(2) # 2번 그래프 창을 생성합
plt.plot(x, y2)# 지정된 그래프 창에 그래프를 그림

plt.figure(1) # 이미 생성된 1번 그래프 창을 지정함
plt.plot(x, y2) # 지정된 그래프 창에 그래프를 그림

plt.figure(2) # 이미 생성된 2번 그래프창을 지정합
plt.clf() # 2번 그래프 창에 그려진 모든 그래프를 지움
plt.plot(x, y1) # 지정된 그래프 창에 그래프를 그림

plt.show()

# # 그래프 영역 나누기
#   - plt.subplot(m,n,p)
#   - m X n 행렬의 그래프 영역 중 p 번쨰 위치
#   - p는 왼쪽 -> 오른쪽 ,위 -> 아래
#

# In[119]:

x = np.arange(0, 10, 0.1)
y1 = 0.3*(x-5) **2 +1
y2 = -1.5*x +3
y3 = np.sin(x)**2 # Numpy에서 sin()은 np.sin()
y4 = 10*np.exp(-x) +1 # exp() 는 np.exp() 지수함수

# In[120]:

# 2 x 2 행렬로 이루어진 하위 그래프에서 p 에따라 위치를 지정

plt.subplot(2,2,1)  # p는 1
plt.plot(x,y1)

plt.subplot(2,2,2)  # p는 2
plt.plot(x,y2)

plt.subplot(2,2,3)  # p는 3
plt.plot(x,y3)

plt.subplot(2,2,4)  # p는 4
plt.plot(x,y4)

plt.show()

# In[121]:

x = np.linspace(-4, 4, 100) # [-4, 4] 범위에서 100개의 값 생성
y1 = x**3
y2 = 10*x**2 - 2

plt.plot(x, y1, x, y2)
plt.show()

# In[123]:

# 겹치는 부분의 데이터를 잘 확인할 수 있게해줌

plt.plot(x, y1, x, y2)
#x, y에 limit을 줘서 그부분만 확대 할수 있음
plt.xlim(-1, 1)
plt.ylim(-3, 3)
plt.show()

# ## 포맷 문자열 '[color][line_style][marker]'

# In[129]:

x = np.arange(0, 5, 1)
y1 = x
y2 = x + 1
y3 = x + 2
y4 = x + 3

plt.plot(x, y1, x, y2, x, y3, x, y4)
plt.show()

# In[130]:

plt.plot(x, y1, '>--r', x, y2, 's-g', x, y3, 'd:b', x, y4,'-.Xc')
plt.show()

# In[127]:

plt.plot(x,y1)
plt.xlabel('X-axis')
plt.ylabel('Y-axis')
plt.title('Graph title')
plt.grid()
plt.show()

# In[128]:

x = np.arange(0, 5, 1)
y1 = x
y2 = x + 1
y3 = x + 2
y4 = x + 3

# In[132]:

# 그래프의 이름 표시
plt.plot(x, y1, '>--r', x, y2, 's-g', x, y3, 'd:b', x, y4,'-.Xc')
plt.legend(['data1', 'data2', 'data3', 'data4'], loc = 'lower right') #loc 로 위치 조정가능
plt.show()

# In[136]:

plt.plot(x, y1, '>--r', x, y2, 's-g', x, y3, 'd:b', x, y4,'-.Xc')
plt.legend(['data1', 'data2', 'data3', 'data4'], loc=4)
plt.xlabel('X-axis')
plt.ylabel('Y-axis')
plt.title('Graph title')
plt.grid(True)
plt.show()

# In[142]:

plt.plot(x, y1, '>--r', x, y2, 's-g', x, y3, 'd:b', x, y4,'-.Xc')
plt.legend(['데이터1', '데이터2', '데이터3', '데이터4'], loc=4)
plt.xlabel('X-axis')
plt.ylabel('Y-axis')
plt.title('Graph title')
plt.grid(True)
plt.show()

# In[140]:

from matplotlib import rc # 한글출력할 수 있게 하는 임포트

# In[141]:

rc('font', family='AppleGothic')

# In[143]:

import platform
from matplotlib import font_manager, rc # 운영체제에 따라 구분
path = "c:/Windows/Fonts/malgun.ttf"
if platform.system() == 'Darwin':
    rc('font', family='AppleGothic')
elif platform.system() == 'Windows':
    font_name = font_manager.FontProperties(fname=path).get_name()
    rc('font', family = font_name)
else:
    print("Unknown system ..")

# In[147]:

plt.plot(x, y1, '>--r', x, y2, 's-g', x, y3, 'd:b', x, y4,'-.Xc')

# x, y 좌표를 지정해 그 위치에 문자열 출력 가능

plt.text(0, 6, "문자열 출력 1")
plt.text(0, 5, "문자열 출력 2")
plt.text(3, 1, "문자열 출력 3")
plt.text(3, 0, "문자열 출력 4")

plt.show()

# In[148]:

height = [165, 177, 160, 178, 185, 155, 172]
weight = [62, 67, 55, 74, 90, 43, 64]

# In[149]:

#scatter 그래프 그리기 => 지정위치에 점을 표현
plt.scatter(height, weight)
plt.xlabel('Height(m)')
plt.ylabel('weight(kg)')
plt.title('Height & Weight')
plt.grid(True)
plt.show()

# In[151]:

# 마커의 큭기와 색깔 설정
plt.scatter(height, weight, s=50, c='r')
plt.xlabel('Height(m)')
plt.ylabel('weight(kg)')
plt.title('Height & Weight')
plt.grid(True)
plt.show()

# In[152]:

#사이즈와 색을 변수에 담아 각자 다르게 출력하기

size = 100 * np.arange(1,8)
colors = ['r', 'g', 'b', 'c', 'm', 'k', 'y']
plt.scatter(height, weight, s=size, c=colors)
plt.xlabel('Height(m)')
plt.ylabel('weight(kg)')
plt.title('Height & Weight')
plt.grid(True)
plt.show()

# In[153]:

city = ['서울', '인천', '대전', '대구', '울산', '부산', '광주']
# 위도(latitude)와 경도(longitude)
lat = [37.56, 37.45, 36.35, 35.87, 35.53, 35.18, 35.16]
lon = [126.97, 126.70, 127.38, 128.60, 129.31, 129.07, 126.85]

# 인구 밀도(명/km^2): 2017년 통계청 자료
pop_den = [16154, 2751, 2839, 2790, 1099, 4454, 2995]

# In[154]:

size = np.array(pop_den) * 0.2
colors = ['r', 'g', 'b', 'c', 'm', 'k', 'y']

# In[155]:

plt.scatter(lon, lat, s= size, c=colors, alpha =0.5)
plt.xlabel('경도(longitude)')
plt.ylabel('밀도(latitude)')
plt.title('지역별 인구 밀도 (2017)')

for x, y, name in zip(lon, lat, city):
    plt.text(x, y, name) # 위도와 경도에 맞는 도시이름 출력

plt.show()

# In[158]:

#휘트니스 클럽 회원 4명의 운동 시작 전, 시작 한 달 후 윗몸 일으키키 횟수

member_IDs = ['m_01', 'm_02', 'm_03', 'm_04'] # 회원 ID
before_ex = [27, 35, 40, 33] # 운동 시작 전
after_ex = [30, 38, 42, 37] # 운동 한 달 후

# In[159]:

# 막대 그래프로 표시

n_data = len(member_IDs)  # 회원이 네명이르모 데이터 수는 4
index = np.arange(n_data) # numpy를 이용해 배열 생성 [0,1,2,3]
plt.bar(index, before_ex) # bar(x, y)에서 x=index, height=before_ex로 지정
plt.show()

# In[160]:

n_data = len(member_IDs)
index = np.arange(n_data)
plt.bar(index, before_ex, tick_label= member_IDs) # x축의 해당 멤버이름울 설정
plt.show()

# In[161]:

n_data = len(member_IDs)
index = np.arange(n_data)
colors = ['r','g','b','y']
plt.bar(index, before_ex, tick_label = member_IDs, color = colors)
plt.show()

# In[162]:

n_data = len(member_IDs)
index = np.arange(n_data)
colors = ['r','g','b','y']
plt.bar(index, before_ex, tick_label = member_IDs, color = colors, width =0.6)
plt.show()

# In[163]:

n_data = len(member_IDs)
index = np.arange(n_data)
colors = ['r','g','b','y']
plt.barh(index, before_ex, tick_label = member_IDs, color = colors) #bar horizontal 가로로
plt.show()

# In[165]:

n_data = len(member_IDs)
index = np.arange(n_data)
colors = ['r','g','b','y']
barWidth = 0.4
plt.bar(index, before_ex, color='c', align='edge', width= barWidth, label='before')
plt.bar(index+barWidth, after_ex, color='m', align='edge', width= barWidth, label='after')

plt.xticks(index+barWidth, member_IDs)
plt.legend()
plt.xlabel('회원 아이디')
plt.ylabel('윗몸일으키기 횟수')
plt.title('운동시작 전과 후의 근지구력 변화 비교')

plt.show()

# ## 히스토그램
#
# - plt.hist(x[,bins=bins_n 또는 'auto])
# - x 는 변량
# - bins 는 계급의 개수, 입력하지 않으면 기본값 10 (x축)
# - 'auto' 가 입력되면 x 에맞게 자동으로 할당

# In[169]:

math = [76, 82, 84, 83, 90, 86, 85, 92, 72, 71, 100, 87, 81, 76, 94, 78, 81, 60, 79, 69, 74, 87, 82, 68, 79]
plt.hist(math)
plt.show()

# In[170]:

plt.hist(math, bins =8)
plt.show()

# In[171]:

plt.hist(math, bins=8)
plt.xlabel('시험점수')
plt.ylabel('도수(frequency)')
plt.title('수학시험의 히스토그램')
plt.grid()
plt.show()

# In[175]:

# 학생 20명의 선호과일
# 3시를 기준으로 반시계방향으로 진행
fruit = ['사과','바나나','딸기','오렌지','포도']
result = [7, 6, 3, 2, 2]
plt.pie(result)
plt.title('학생 20명의 선호과일')
plt.show()

# In[176]:

plt.pie(result, labels=fruit, autopct='%.1f%%')
plt.show()

# In[180]:

plt.pie(result, labels=fruit, autopct='%.1f%%', startangle=90, counterclock=False) #시작점을 12시에서 시계방향으로
plt.show()

# In[186]:

explode_value = (0.1, 0, 0, 0, 0) # 인덱스에맞는 수 만큼 분리
plt.pie(result, labels=fruit, autopct='%.1f%%', startangle=90, counterclock=False, explode=explode_value, shadow=True)
plt.show()

# In[187]:

print(matplotlib.rcParams['figure.figsize'])

# In[188]:

print(matplotlib.rcParams['figure.dpi'])

# In[189]:

explode_value = (0.1, 0, 0, 0, 0)
plt.pie(result, labels=fruit, autopct='%.1f%%', startangle=90, counterclock=False, explode=explode_value, shadow=True)
plt.title('Save figures')
plt.savefig('/Users/user/java-web-project0208/python-basic/data/saveFigTest', dpi = 100) # 저장하기
plt.show()

# In[190]:

#사이즈 조정
plt.figure(figsize=(6,5))
plt.pie(result, labels=fruit, autopct='%.1f%%', startangle=90, counterclock=False, explode=explode_value, shadow=True)
plt.title('Save figures')
plt.savefig('/Users/user/java-web-project0208/python-basic/data/saveFigTest', dpi = 200)
plt.show()

# In[191]:

# pandas 로 그래프 그리기 ==> matplotlib 를 지원한다.
s1 = pd.Series([1,2,3,4,5,6,7,8,9,10])
print(s1)

# In[192]:

s1.plot()
plt.show()

# In[199]:

s2 = pd.Series([1,2,3,4,5,6,7,8,9,10], index = pd.date_range('2021-01-01', periods=10))
s2.plot(grid=True)
plt.show()

# In[203]:

df_rain = pd.read_csv('/Users/user/java-web-project0208/python-basic/data/sea_rain1.csv', index_col="연도")
df_rain.plot()
plt.show()

# In[206]:

rain_plot = df_rain.plot(grid=True, style=['r--*', 'g-o', 'b:*', 'm-.p'])
rain_plot.set_xlabel('연도')
rain_plot.set_ylabel('강수량')
rain_plot.set_title('연간 강수량')
plt.show()

# In[207]:

#연도별 1인당 주거면적
year = [2006, 2008, 2010, 2012, 2014, 2016] # 연도
area = [26.2, 27.8, 28.5, 31.7, 33.5, 33.2] # 1인당 주거면적
table = {'연도':year, '주거면적':area}

# In[208]:

df_area = pd.DataFrame(table, columns=['연도', '주거면적'])
print(df_area)

# In[209]:

df_area.plot(x='연도', y='주거면적', grid=True, title='연도별 1인당 주거면적')
plt.show()

# In[ ]:


## environment setup

import unicodedata
import re
import json

import nltk
from nltk.tokenize.toktok import ToktokTokenizer
from nltk.corpus import stopwords

import pandas as pd

import acquire

###################################################################################################################

def basic_clean(original):
    '''
    This function will take in a string of text and begin basic cleaning
    - will make all characters lowercase
    - will normalize unicode
    - will remove special characters
    - will return edited text
    '''

    article = original.lower()
    # make all characters lowercase

    article = unicodedata.normalize('NFKD', article)\
    .encode('ascii', 'ignore')\
    .decode('utf-8', 'ignore')
    # normalize unicode

    article = re.sub(r"[^a-z0-9'\s]", '', article)
    # remove any special characters

    return article

###################################################################################################################

def tokenize(article):
    '''
    This function will take in a string and run a ToktokTokenizer object
    - will return a string of the token words
    - expected to be used after basic_clean function
    '''

    tokenizer = nltk.tokenize.ToktokTokenizer()
    # make tokenizer object

    token_article = tokenizer.tokenize(article, return_str=True)
    # run tokenizer, return as str

    return token_article

###################################################################################################################

def stem(article):
    '''
    This function will take in a string and run a PorterStemmer object
    - will return a string of all the stems of the words in the article
    '''

    # Create the nltk stemmer object, then use it
    ps = nltk.porter.PorterStemmer()

    stems = [ps.stem(word) for word in article.split()]
    # run stemmer object on article to create stems

    article_stemmed = ' '.join(stems)
    # create stemmed article by joining stems

    return article_stemmed

###################################################################################################################

def lemmatize(article):
    '''
    This functions will take in a string and run a WordNetLemmatizer object
    - will return a string of lemmatized words from the article
    '''

    wnl = nltk.stem.WordNetLemmatizer()
    # create lemmatizer object

    lemmas = [wnl.lemmatize(word) for word in article.split()]
    ## run lemmatizer object on article

    article_lemmatized = ' '.join(lemmas)
    # create lemmatized article by joining lemmatized words together

    return article_lemmatized

###################################################################################################################

def remove_stopwords(article, append_stopword, remove_stopword):
    '''
    This function will take in a string in the form of an article and remove standard English stop words
    as well as a list of words to add to the stop words list if desired
    and a list of words to add to the stop words list if also desrired
    - will return a string of remaining words once all desired stop words have been removed
    '''

    stopword_list = stopwords.words('english')
    # create standard English stop words list

    for word in append_stopword:
        stopword_list.append(word)
    # add any extra stop words

    for word in remove_stopword:
        stopword_list.remove(word)
    # remove any unwanted stop words

    words = article.split()
    # split article into individual words

    filtered_words = [w for w in words if w not in stopword_list]
    # filter for words in stop words

    article_without_stopwords = ' '.join(filtered_words)
    # recreate article out of remaining words

    return article_without_stopwords

###################################################################################################################


import requests
import translate
from translate import Translator
from pyowm import OWM
from requests import get

owm = OWM('8c32dd4e6171baabc661ac89852c5509')
mgr = owm.weather_manager()

ip = get('https://ipapi.co/ip/').text

def find_city_by_ip(ip=ip):
    try:
        response = requests.get(url= f'http://ip-api.com/json/{ip}').json()
        city = response.get('city')
        return city
    except requests.exceptions.ConnectionError:
        print('[!] Please chel your connection!')
def find_country_lang_by_ip(ip=ip):
    try:
        response = requests.get(url= f'http://ip-api.com/json/{ip}').json()
        lang =response.get('countryCode')
        return lang
    except requests.exceptions.ConnectionError:
        print('[!] Please chel your connection!')

def wather_in_your_city():
    observation = mgr.weather_at_place(str(find_city_by_ip()))
    w = observation.weather
    wather = w.detailed_status
    return f'{wather}'

def temp_in_your_city():
    observation = mgr.weather_at_place(str(find_city_by_ip()))
    w = observation.weather
    temp =  w.temperature('celsius')['temp']

    return f'{temp} C'

def translate():
    translator= Translator(from_lang="english",to_lang='ru')
    translation = translator.translate(wather_in_your_city())
def check_lang():
    if find_country_lang_by_ip() != 'RU' or 'UA':
        print(f'Wether: {wather_in_your_city()}\nTemp: {temp_in_your_city()}')
    elif find_country_lang_by_ip() == 'RU' or 'UA':
        print(f'Погода: {translate()}\nТемпература: {temp_in_your_city()}')
def main():
    check_lang()

if __name__ == '__main__':
    main()


# -*- coding:utf-8 -*-
#@Time : 2020/6/3 下午2:49
#@Author: kkkkibj@163.com
#@File : headless_browser.py
#无头浏览器测试

from selenium import webdriver
chromedriver = '/Users/hongyanma/Desktop/chromedriver'
import time
# from selenium.webdriver.chrome.options import Options

# chrome_options = Options()
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument("--headless")
# chrome_options.add_argument("--disable-gpu")
driver = webdriver.Chrome(chrome_options=chrome_options, executable_path=chromedriver)
driver.set_window_size(1920, 1080)
driver.get("http://hmrc.palmgo.cn/lwzx1/a1c64c3e6c9b76efcbccb8effd58fcad.html")
time.sleep(int(10))
driver.get_screenshot_as_file("/Users/hongyanma/Desktop/lwzx_headless.png")
driver.close()


import numpy as np
from sklearn.decomposition import PCA
from sklearn.decomposition import FastICA
from sklearn.preprocessing import StandardScaler
import pandas as pd
import ta
from tqdm import tqdm
import warnings
import plotly.graph_objects as go
import os

from utils import read_df

warnings.simplefilter("ignore")

DATA_PATH = os.path.join('..', 'data', 'main.csv')
TOL = 1

class DataPreprocessor:
    def __init__(self, input_data= DATA_PATH):
        if isinstance(input_data, str):
            self.data = read_df(input_data)
        elif isinstance(input_data, pd.DataFrame):
            self.data = input_data
        else:
            raise ValueError("Invalid input. Please provide a file path or a DataFrame.")

    def plot_candlestick_with_signals(self, output_file=None):
        fig = go.Figure()

        fig.add_trace(go.Candlestick(x=self.data.index,
                open=self.data['open'],
                high=self.data['high'],
                low=self.data['low'],
                close=self.data['close'],
                name='Candlesticks'))

        buy_signals = self.data[self.data['target'] == 1]
        sell_signals = self.data[self.data['target'] == -1]

        fig.add_trace(go.Scatter(x=buy_signals.index, y=buy_signals['close'],
                                 mode='markers', name='Buy Signals',
                                 marker=dict(color='green', size=10, symbol='triangle-up')))

        fig.add_trace(go.Scatter(x=sell_signals.index, y=sell_signals['close'],
                                 mode='markers', name='Sell Signals',
                                 marker=dict(color='red', size=10, symbol='triangle-down')))

        fig.update_layout(
            title='Candlestick Chart with Buy/Sell Signals',
            xaxis_title='Date',
            yaxis_title='Price',
            template='plotly_dark',
            xaxis_rangeslider_visible=False
        )

        if output_file:
            fig.write_html(output_file)
        else:
            fig.show()

    def add_time_features(self):
        self.data['Year'] = self.data['datetime'].dt.year
        self.data['Month'] = self.data['datetime'].dt.month
        self.data['Day'] = self.data['datetime'].dt.day
        self.data['Hour'] = self.data['datetime'].dt.hour
        self.data['Minute'] = self.data['datetime'].dt.minute
        self.data['Day_of_Week'] = self.data['datetime'].dt.dayofweek
        self.data['Day_Sin'] = np.sin((self.data['Day'] - 1) * (2. * np.pi / 30))
        self.data['Day_Cos'] = np.cos((self.data['Day'] - 1) * (2. * np.pi / 30))
        self.data['Hour_Sin'] = np.sin(self.data['Hour'] * (2. * np.pi / 24))
        self.data['Hour_Cos'] = np.cos(self.data['Hour'] * (2. * np.pi / 24))
        self.data['Minute_Sin'] = np.sin(self.data['Minute'] * (2. * np.pi / 60))
        self.data['Minute_Cos'] = np.cos(self.data['Minute'] * (2. * np.pi / 60))
        self.data.drop(columns=['datetime'], inplace=True)

    def add_summaries(self):
        for col in ['open', 'high', 'low', 'close']:
            rolling_window = self.data[col].rolling(window=20)
            self.data[f'{col}_min_10'] = rolling_window.min()
            self.data[f'{col}_25th_10'] = rolling_window.quantile(0.25)
            self.data[f'{col}_median_10'] = rolling_window.median()
            self.data[f'{col}_75th_10'] = rolling_window.quantile(0.75)
            self.data[f'{col}_max_10'] = rolling_window.max()
            self.data[f'{col}_mean_10'] = rolling_window.mean()
            self.data[f'{col}_std_10'] = rolling_window.std()

    def add_technical_indicators(self):
        self.data['SMA_10'] = ta.trend.sma_indicator(self.data['close'], window=10)
        self.data['EMA_10'] = ta.trend.ema_indicator(self.data['close'], window=10)
        self.data['RSI'] = ta.momentum.RSIIndicator(self.data['close'], window=14).rsi()

        macd = ta.trend.MACD(self.data['close'])
        self.data['MACD'] = macd.macd()
        self.data['MACD_signal'] = macd.macd_signal()
        self.data['MACD_diff'] = macd.macd_diff()

        stoch = ta.momentum.StochasticOscillator(self.data['high'], self.data['low'], self.data['close'])
        self.data['Stoch_%K'] = stoch.stoch()
        self.data['Stoch_%D'] = stoch.stoch_signal()

        self.data['ATR'] = ta.volatility.AverageTrueRange(self.data['high'], self.data['low'], self.data['close']).average_true_range()

        bollinger = ta.volatility.BollingerBands(self.data['close'])
        self.data['Bollinger_hband'] = bollinger.bollinger_hband()
        self.data['Bollinger_lband'] = bollinger.bollinger_lband()
        self.data['Bollinger_mavg'] = bollinger.bollinger_mavg()

        self.data['ROC'] = ta.momentum.ROCIndicator(self.data['close']).roc()
        self.data['PPO'] = ta.momentum.PercentagePriceOscillator(self.data['close']).ppo()

        # Ichimoku Cloud
        self.data['Ichimoku_Conversion'] = (self.data['high'].rolling(window=9).max() + self.data['low'].rolling(window=9).min()) / 2
        self.data['Ichimoku_Base'] = (self.data['high'].rolling(window=26).max() + self.data['low'].rolling(window=26).min()) / 2
        self.data['Ichimoku_Leading_A'] = (self.data['Ichimoku_Conversion'] + self.data['Ichimoku_Base']) / 2
        self.data['Ichimoku_Leading_B'] = (self.data['high'].rolling(window=52).max() + self.data['low'].rolling(window=52).min()) / 2
        self.data['Ichimoku_Lagging'] = self.data['close'].shift(26)

        self.data['Pct_Change_1min'] = self.data['close'].pct_change(1)
        self.data['Pct_Change_5min'] = self.data['close'].pct_change(5)
        self.data['Pct_Change_10min'] = self.data['close'].pct_change(10)
        self.data['Pct_Change_30min'] = self.data['close'].pct_change(30)

        self.data['ATRP'] = ta.volatility.AverageTrueRange(self.data['high'], self.data['low'], self.data['close']).average_true_range()

        self.data['Historical_Volatility'] = self.data['close'].rolling(window=10).std() * (252**0.5)  # Annualized volatility
        self.data['Price_Oscillator'] = self.data['close'].diff(4)  # Difference between the current price and the price 4 periods ago
        self.data['Standard_Deviation'] = self.data['close'].rolling(window=14).std()

        self.data['CCI'] = ta.trend.CCIIndicator(self.data['high'], self.data['low'], self.data['close']).cci()

        donchian = ta.volatility.DonchianChannel(self.data['high'], self.data['low'], self.data['close'])
        self.data['Donchian_Channel_hband'] = donchian.donchian_channel_hband()
        self.data['Donchian_Channel_lband'] = donchian.donchian_channel_lband()
        self.data['Donchian_Channel_mband'] = donchian.donchian_channel_mband()

        self.data['DPO'] = ta.trend.DPOIndicator(self.data['close']).dpo()

        typical_price = (self.data['high'] + self.data['low'] + self.data['close']) / 3
        self.data['MFI'] = ta.volume.MFIIndicator(high=self.data['high'], low=self.data['low'], close=self.data['close'], volume=typical_price).money_flow_index()

        self.data['Trix'] = ta.trend.TRIXIndicator(self.data['close']).trix()

    def handle_missing_values(self):
        self.data.dropna(axis=0, inplace=True)

    def transform_for_pred(self, data):
        self.data = data
        self.data = self.analyze_sharp_changes(self.data, pred=True)
        self.add_time_features()
        self.add_summaries()
        self.add_technical_indicators()
        self.handle_missing_values()
        return self.data

    def analyze_sharp_changes(self, data, window_size=30, price_diff_threshold=20, tolerance=1, pred = False):

        # rolling_min = data['close'].rolling(window_size).min()
        # rolling_max = data['close'].rolling(window_size).max()
        #
        # price_diff = rolling_max - rolling_min
        # sharp_changes_idx = np.where(price_diff >= price_diff_threshold)[0]
        # sharp_changes_idx = sharp_changes_idx[sharp_changes_idx < len(data) - window_size + 1]
        # sharp_changes_info = []
        # for idx in sharp_changes_idx:
        #     min_price = rolling_min.iloc[idx]
        #     max_price = rolling_max.iloc[idx]
        #     actual_price = data['close'].iloc[idx]
        #     if np.abs(actual_price - min_price) > np.abs(max_price - actual_price):
        #         change_start_price = min_price
        #         change_magnitude = actual_price - min_price
        #     else:
        #         change_start_price = max_price
        #         change_magnitude = max_price - actual_price
        #
        #     if not any(np.abs(change_start_price - prev_price) < window_size for prev_price, _ in sharp_changes_info):
        #         sharp_changes_info.append((change_start_price, change_magnitude))
        #
        # # Sort the changes by magnitude in descending order, extract the prices, and then sort by price
        # lines = sorted([price for price, _ in sorted(sharp_changes_info, key=lambda x: x[1], reverse=True)])
        # data = data[data['close'] >= (10000 - 20)]

        rs_lines = pd.read_csv("data/stronglines.csv")
        lines = rs_lines.values.flatten()

        close_prices = data['close'].values[:, np.newaxis]
        lines = np.array(lines)
        lines_reshaped = lines[np.newaxis, :]

        diffs = lines_reshaped - close_prices
        partitioned_indices = np.argpartition(np.abs(diffs), 4, axis=1)[:, :4]
        closest_diffs = np.take_along_axis(diffs, partitioned_indices, axis=1)
        closest_rs_values = np.take_along_axis(lines_reshaped, partitioned_indices, axis=1)

        data['SL_1'] = closest_diffs[:, 0]
        data['SL_2'] = closest_diffs[:, 1]
        data['SL_3'] = closest_diffs[:, 2]
        data['SL_4'] = closest_diffs[:, 3]
        if not pred:
            data['target'] = np.nan
            tolerance = TOL
            close_prices = data['close'].values[:, np.newaxis]
            diffs = np.abs(lines_reshaped - close_prices)
            within_tolerance_indices = np.where(diffs <= tolerance)

            data = data.reset_index(drop=True)

            valid_indices = set(data.index)
            filtered_indices = [idx for idx in within_tolerance_indices[0] if idx in valid_indices]

            data.loc[filtered_indices, 'target'] = lines_reshaped[0, within_tolerance_indices[1]]

            data.loc[within_tolerance_indices[0], 'target'] = lines_reshaped[0, within_tolerance_indices[1]]
            data['target'].fillna(method='bfill', inplace=True)
            data.dropna(subset=['target'], inplace=True)
            diffs = data['target'] - data['close']
            data['target'] = np.where(diffs > 0, 1, np.where(diffs < 0, -1, 0))

        return data

    def transform_for_training(self, n = None):
        self.data = self.analyze_sharp_changes(self.data)
        self.add_time_features()
        self.add_summaries()
        self.add_technical_indicators()
        self.handle_missing_values()
        return self.data

if __name__ == "__main__":
    preprocessor = DataPreprocessor()
    preprocessor.transform_for_training(n=45000)
    preprocessor.plot_candlestick_with_signals()
    print(preprocessor.data.head())


from django.db import models

from wagtail.core.models import Orderable
from wagtail.snippets.models import register_snippet
from wagtail.admin.edit_handlers import (
  FieldPanel,
  MultiFieldPanel,
  InlinePanel,
  PageChooserPanel,
)

from modelcluster.fields import ParentalKey
from modelcluster.models import ClusterableModel

from django_extensions.db.fields import AutoSlugField

class SubNavigationItem(Orderable):

  link_title = models.CharField(
    blank=True,
    null=True,
    max_length=50
  )
  link_url = models.CharField(
    max_length=500,
    blank=True
  )
  link_page = models.ForeignKey(
    'wagtailcore.Page',
    null=True,
    blank=True,
    related_name="+",
    on_delete=models.CASCADE,
  )
  open_in_new_tab = models.BooleanField(default=False, blank=True)

  parent = ParentalKey("MenuItem", related_name="children", null=True)

  @property
  def link(self) -> str:
    if self.link_page:
      return self.link_page.url
    elif self.link_url:
      return self.link_url
    return '#'

  @property
  def title(self) -> str:
    if self.link_page and not self.link_title:
      return self.link_page.title
    elif self.link_title:
      return self.link_title
    return 'Missing Title'

SubNavigationItem.panels = [
  FieldPanel('link_title'),
  FieldPanel('link_url'),
  PageChooserPanel('link_page'),
  FieldPanel('open_in_new_tab'),
]

# Single menu item
class MenuItem(Orderable, ClusterableModel):

  link_title = models.CharField(
    blank=True,
    null=True,
    max_length=50
  )
  link_url = models.CharField(
    max_length=500,
    blank=True
  )
  link_page = models.ForeignKey(
    'wagtailcore.Page',
    null=True,
    blank=True,
    related_name="+",
    on_delete=models.CASCADE,
  )
  open_in_new_tab = models.BooleanField(default=False, blank=True)

  page = ParentalKey("Menu", related_name="menu_items", null=True)

  @property
  def link(self) -> str:
    if self.link_page:
      return self.link_page.url
    elif self.link_url:
      return self.link_url
    return '#'

  @property
  def title(self) -> str:
    if self.link_page and not self.link_title:
      return self.link_page.title
    elif self.link_title:
      return self.link_title
    return 'Missing Title'

MenuItem.panels = [
  FieldPanel('link_title'),
  FieldPanel('link_url'),
  PageChooserPanel('link_page'),
  FieldPanel('open_in_new_tab'),
  MultiFieldPanel([
    InlinePanel('children', label="Submenu item")
  ], heading="Submenu", classname='collapsible collapsed')
]

# Main menu
@register_snippet
class Menu(ClusterableModel):

  title = models.CharField(max_length=100, default="")
  identifier = AutoSlugField(populate_from="title", editable=True, null=True)

  def __str__(self):
    return self.title

Menu.panels = [
  MultiFieldPanel([
    FieldPanel('title'),
    FieldPanel('identifier'),
  ], heading="Menu"),
  InlinePanel('menu_items', label="Menu Item")
]


#!/usr/bin/env python

"""
deepthought@42 ~ $ pdlisten

[{constants.success}]: Server is listening on 192.168.1.58:8080
[{constants.success}]: Connection Key: 192@168@1@58+8080
[{constants.success}]: Waiting for a connection for files...

This program is used to listen for incoming connections from the
pdsnd.py program. If it recieves a connection, it will write the
file in the directory.

The pdsnd command is used for sending a file to the server
created by the server "pdlisten" program.

### PDLISTEN ###

The pdlisten command when run gives a connection key which
is needed to send a file while using pdsnd.

The -port argument may be used to change the port of the server.

Thanks for having interest in this library.
This file is the script for the pdlisten program.
"""

# Import libraries

import sys
import socket
import readline
import os
import constants
from rich import print

# Decodes the key

def keymaker(key: str) -> str:
    return key.replace(".", "@").replace(":", "+")

# HEADER Variable
HEADER = 64

# CLIENT
# A class which makes life easier.
class Client:
    def __init__(self, server, port):

        # CHANGE STUFF HERE
        self.SERVER_HOST = server
        self.SERVER_PORT = port
        # CHANGE STUFF HERE

        self.BUFFER_SIZE = 4096
        self.SEPARATOR = "<pyserved21>"
        self.s = socket.socket()
        self.s.bind((self.SERVER_HOST, self.SERVER_PORT))

    def listen(self):
        # Listen for incoming connections
        self.s.listen(10)

    def accept(self):
        # Accept incoming connections from client
        # and return client socket and address.
        self.client_socket, self.address = self.s.accept()
        return (self.client_socket, self.address)

    def receive(self):
        # Receive message from client_socket
        # and return filename and filesize
        received = self.client_socket.recv(self.BUFFER_SIZE).decode()
        filename, filesize = received.split(self.SEPARATOR)
        self.filename = os.path.basename(filename)
        self.filesize = int(filesize)
        return self.filename, self.filesize

    def write(self):
        # Write file which is received from client_socket
        # in the same directory.
        with open(self.filename, "wb") as f:
            while True:
                # read 1024 bytes from the socket (receive)
                bytes_read = self.client_socket.recv(self.BUFFER_SIZE)
                if not bytes_read:
                    # nothing is received
                    # file transmitting is done
                    break
                # write to the file the bytes we just received
                f.write(bytes_read)

    def close(self):
        # close the client socket
        self.client_socket.close()
        # close the server socket
        self.s.close()

# Function to get Internal IP of device

def get_internal_ip():
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    s.connect(("8.8.8.8", 80))
    ip = s.getsockname()[0]
    s.close()
    return ip

# CHANGE SERVER HERE
SERVER = get_internal_ip()
# CHANGE SERVER HERE

# CHANGE PORT HERE
PORT = 8080
# CHANGE PORT HERE

ADDR = (SERVER, PORT)

# Check if port is not in use
# If port is in use, return True
# Else False
def is_port_in_use() -> bool:
    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
        return s.connect_ex((get_internal_ip(), PORT)) == 0

# Checks if -port argument is passed
try:
    if sys.argv[1] is None:
        pass
    else:
        if "-port" in sys.argv[1]:
            PORT = int(sys.argv[2])
            # print(PORT)
            ADDR = (SERVER, PORT)
except IndexError:
    pass

# If port 8080 is in use,
# End program with message informing that port is in use
if is_port_in_use() is True:
    print(
        f"[bold red][{constants.error}]: Port 8080 is already in use in your network. Choose another port instead using the '-port' parameter.[/]"
    )
    exit()

# Start server

try:
    client = Client(SERVER, PORT)

    print(
        f"[bold green][{constants.success}]:[/] Server is listening on {SERVER}:{PORT}"
    )
    print(
        f"[bold green][{constants.success}]:[/] Connection Key: {keymaker(str(SERVER + ':' + str(PORT)))}"
    )
    print(
        f"[bold green][{constants.success}]:[/] Waiting for a connection for files..."
    )

    client.listen()
    _, address = client.accept()

    print(
        f"[bold green][{constants.success}]:[/] Connection from {address[0]}:{address[1]} accepted."
    )
    print(f"[bold green][{constants.success}]:[/] Reading file data.....")

    try:
        client.receive()
    except ValueError as e:
        print(
            f"{repr(e)} \n\n [bold red][{constants.error}]: An error occured. Please try again in few seconds....[/]"
        )
        exit()

    print(
        f"[bold green][{constants.success}]:[/] Received file '[bold yellow]{client.filename}[/]', saving to current directory."
    )
    client.write()

    print(
        "[bold green][{constants.success}]:[/] File transferred successfully. Closing connection..."
    )
    client.close()

except KeyboardInterrupt:
    print(f"\n\n[bold red][{constants.error}]: KeyboardInterrupt[/]")
    exit()

except Exception as e:
    print(
        f"\n\n[bold red][{constants.error}]: {repr(e)} \n\n If the problem persists, contact the owner of the package at https://github.com/SblipDev/pyserved[/] \n"
    )
    exit()