from collections import namedtuple

from blivet.size import Size

from pyanaconda.anaconda_loggers import get_module_logger
from pyanaconda.core.i18n import CN_, CP_
from pyanaconda.modules.common.structures.storage import DeviceData
from pyanaconda.ui.lib.storage import apply_disk_selection, try_populate_devicetree, \
    filter_disks_by_names
from pyanaconda.modules.common.constants.objects import DISK_SELECTION, FCOE, ISCSI, DASD, \
    DEVICE_TREE
from pyanaconda.modules.common.constants.services import STORAGE

from pyanaconda.ui.gui.utils import timed_action, really_show, really_hide
from pyanaconda.ui.gui.spokes import NormalSpoke
from pyanaconda.ui.gui.spokes.advstorage.fcoe import FCoEDialog
from pyanaconda.ui.gui.spokes.advstorage.iscsi import ISCSIDialog
from pyanaconda.ui.gui.spokes.advstorage.zfcp import ZFCPDialog
from pyanaconda.ui.gui.spokes.advstorage.dasd import DASDDialog
from pyanaconda.ui.gui.spokes.advstorage.nvdimm import NVDIMMDialog
from pyanaconda.ui.gui.spokes.lib.cart import SelectedDisksDialog
from pyanaconda.ui.categories.system import SystemCategory

import gi
gi.require_version("Gtk", "3.0")
from gi.repository import Gtk

log = get_module_logger(__name__)

__all__ = ["FilterSpoke"]

PAGE_SEARCH = 0
PAGE_MULTIPATH = 1
PAGE_OTHER = 2
PAGE_NVDIMM = 3
PAGE_Z = 4

DiskStoreRow = namedtuple("DiskStoreRow", [
    "visible", "selected", "mutable",
    "name", "type", "model", "capacity",
    "vendor", "interconnect", "serial",
    "wwid", "paths", "port", "target",
    "lun", "ccw", "wwpn", "namespace", "mode"
])

def create_row(device_data, selected, mutable):
    """Create a disk store row for the given data.

    :param device_data: an instance of DeviceData
    :param selected: True if the device is selected, otherwise False
    :param mutable: False if the device is protected, otherwise True
    :return: an instance of DiskStoreRow
    """
    return DiskStoreRow(
        visible=True,
        selected=selected,
        mutable=mutable and not device_data.protected,
        name=device_data.name,
        type=device_data.type,
        model=device_data.attrs.get("model", ""),
        capacity=str(Size(device_data.size)),
        vendor=device_data.attrs.get("vendor", ""),
        interconnect=device_data.attrs.get("bus", ""),
        serial=device_data.attrs.get("serial", ""),
        wwid=device_data.attrs.get("path-id", ""),
        paths="\n".join(device_data.parents),
        port=device_data.attrs.get("port", ""),
        target=device_data.attrs.get("target", ""),
        lun=device_data.attrs.get("lun", "") or device_data.attrs.get("fcp-lun", ""),
        ccw=device_data.attrs.get("hba-id", ""),
        wwpn=device_data.attrs.get("wwpn", ""),
        namespace=device_data.attrs.get("namespace", ""),
        mode=device_data.attrs.get("mode", "")
    )

class FilterPage(object):
    """A FilterPage is the logic behind one of the notebook tabs on the filter
       UI spoke.  Each page has its own specific filtered model overlaid on top
       of a common model that holds all non-advanced disks.

       A Page is created once, when the filter spoke is initialized.  It is
       setup multiple times - each time the spoke is revisited.  When the Page
       is setup, it is given a complete view of all disks that belong on this
       Page.  This is because certain pages may require populating a combo with
       all vendor names, or other similar tasks.

       This class is just a base class.  One subclass should be created for each
       more specialized type of page.  Only one instance of each subclass should
       ever be created.
    """
    # Default value of a type combo.
    SEARCH_TYPE_NONE = 'None'

    def __init__(self, builder, model_name, combo_name):
        """Create a new FilterPage instance.

        :param builder: a instance of the Gtk.Builder
        :param model_name: a name of the filter model
        :param combo_name: a name of the type combo
        """
        self._builder = builder
        self._is_active = False

        self._model = self._builder.get_object(model_name)
        self._model.set_visible_func(self.visible_func)

        self._combo = self._builder.get_object(combo_name)

    @property
    def model(self):
        """The model."""
        return self._model

    @property
    def is_active(self):
        """Is the filter active?"""
        return self._is_active

    @is_active.setter
    def is_active(self, value):
        self._is_active = value

    def is_member(self, device_type):
        """Does device belong on this page?  This function should taken into
           account what kind of thing device is.  It should not be concerned
           with any sort of filtering settings.  It only determines whether
           device belongs.
        """
        return True

    def setup(self, store, disks, selected_names, protected_names):
        """Do whatever setup of the UI is necessary before this page can be
           displayed.  This function is called every time the filter spoke
           is revisited, and thus must first do any cleanup that is necessary.

           The setup function is passed a reference to the primary store, a list
           of names of disks the user has selected (either from a previous visit
           or via kickstart), and a list of all disk objects that belong on this
           page as determined from the is_member method.

           At the least, this method should add all the disks to the store.  It
           may also need to populate combos and other lists as appropriate.
        """
        pass

    def _setup_combo(self, combo, items):
        """Populate a given GtkComboBoxText instance with a list of items.

        The combo will first be cleared, so this method is suitable for calling
        repeatedly. The first item in the list will be empty to allow the combo
        box criterion to be cleared. The first non-empty item in the list will
        be selected by default.
        """
        combo.remove_all()
        combo.append_text('')

        # Remove duplicate and empty items and sort them.
        items = sorted(set(filter(None, items)))

        for i in items:
            combo.append_text(i)

        if items:
            combo.set_active(1)

    def _setup_search_type(self):
        """Set up the default search type."""
        self._combo.set_active_id(self.SEARCH_TYPE_NONE)
        self._combo.emit("changed")

    def clear(self):
        """Blank out any filtering-related fields on this page and return them
           to their defaults.  This is called when the Clear button is clicked.
        """
        pass

    def visible_func(self, model, itr, *args):
        """This method is called for every row (disk) in the store, in order to
           determine if it should be displayed on this page or not.  This method
           should take into account whether is_active is set, perhaps whether
           something in pyanaconda.flags is setup, and other settings to make
           a final decision.  Because filtering can be complicated, many pages
           will want to farm this decision out to another method.

           The return value is a boolean indicating whether the row is visible
           or not.
        """
        if not self._is_active:
            return True

        row = DiskStoreRow(*model[itr])
        if not self.is_member(row.type):
            return False

        log.debug("Filter %s with %s.", row.name, str(self))

        filter_by = self._combo.get_active_id()
        if filter_by == self.SEARCH_TYPE_NONE:
            return True

        return self._filter_func(filter_by, row)

    def _filter_func(self, filter_by, row):
        """Filter a row by the specified filter."""
        return True

    def __str__(self):
        """Get the name of the filter."""
        return self.__class__.__name__

class SearchPage(FilterPage):
    # Match these to searchTypeCombo ids in glade
    SEARCH_TYPE_PORT_TARGET_LUN = 'PTL'
    SEARCH_TYPE_WWID = 'WWID'

    def __init__(self, builder):
        super().__init__(builder, "searchModel", "searchTypeCombo")
        self._lun_entry = self._builder.get_object("searchLUNEntry")
        self._wwid_entry = self._builder.get_object("searchWWIDEntry")
        self._port_combo = self._builder.get_object("searchPortCombo")
        self._target_entry = self._builder.get_object("searchTargetEntry")

    def setup(self, store, disks, selected_names, protected_names):
        ports = set()

        for device_data in disks:
            ports.add(device_data.attrs.get("port"))

        self._setup_combo(self._port_combo, ports)
        self._setup_search_type()

    def clear(self):
        self._lun_entry.set_text("")
        self._port_combo.set_active(0)
        self._target_entry.set_text("")
        self._wwid_entry.set_text("")

    def _filter_func(self, filter_by, row):
        if filter_by == self.SEARCH_TYPE_PORT_TARGET_LUN:
            port = self._port_combo.get_active_text()
            if port and port != row.port:
                return False

            target = self._target_entry.get_text().strip()
            if target and target not in row.target:
                return False

            lun = self._lun_entry.get_text().strip()
            if lun and lun not in row.lun:
                return False

            return True

        if filter_by == self.SEARCH_TYPE_WWID:
            return self._wwid_entry.get_text() in row.wwid

        return False

class MultipathPage(FilterPage):
    # Match these to multipathTypeCombo ids in glade
    SEARCH_TYPE_VENDOR = 'Vendor'
    SEARCH_TYPE_INTERCONNECT = 'Interconnect'
    SEARCH_TYPE_WWID = 'WWID'

    def __init__(self, builder):
        super().__init__(builder, "multipathModel", "multipathTypeCombo")
        self._ic_combo = self._builder.get_object("multipathInterconnectCombo")
        self._vendor_combo = self._builder.get_object("multipathVendorCombo")
        self._wwid_entry = self._builder.get_object("multipathWWIDEntry")

    def is_member(self, device_type):
        return device_type == "dm-multipath"

    def setup(self, store, disks, selected_names, protected_names):
        vendors = set()
        interconnects = set()

        for device_data in disks:
            row = create_row(
                device_data,
                device_data.name in selected_names,
                device_data.name not in protected_names
            )

            store.append(list(row))
            vendors.add(device_data.attrs.get("vendor"))
            interconnects.add(device_data.attrs.get("bus"))

        self._setup_combo(self._vendor_combo, vendors)
        self._setup_combo(self._ic_combo, interconnects)
        self._setup_search_type()

    def clear(self):
        self._ic_combo.set_active(0)
        self._vendor_combo.set_active(0)
        self._wwid_entry.set_text("")

    def _filter_func(self, filter_by, row):
        if filter_by == self.SEARCH_TYPE_VENDOR:
            return row.vendor == self._vendor_combo.get_active_text()

        if filter_by == self.SEARCH_TYPE_INTERCONNECT:
            return row.interconnect == self._ic_combo.get_active_text()

        if filter_by == self.SEARCH_TYPE_WWID:
            return self._wwid_entry.get_text() in row.wwid

        return False

class OtherPage(FilterPage):
    # Match these to otherTypeCombo ids in glade
    SEARCH_TYPE_VENDOR = 'Vendor'
    SEARCH_TYPE_INTERCONNECT = 'Interconnect'
    SEARCH_TYPE_ID = 'ID'

    def __init__(self, builder):
        super().__init__(builder, "otherModel", "otherTypeCombo")
        self._ic_combo = self._builder.get_object("otherInterconnectCombo")
        self._id_entry = self._builder.get_object("otherIDEntry")
        self._vendor_combo = self._builder.get_object("otherVendorCombo")

    def is_member(self, device_type):
        return device_type == "iscsi" or device_type == "fcoe"

    def setup(self, store, disks, selected_names, protected_names):
        vendors = set()
        interconnects = set()

        for device_data in disks:
            row = create_row(
                device_data,
                device_data.name in selected_names,
                device_data.name not in protected_names
            )

            store.append([*row])
            vendors.add(device_data.attrs.get("vendor"))
            interconnects.add(device_data.attrs.get("bus"))

        self._setup_combo(self._vendor_combo, vendors)
        self._setup_combo(self._ic_combo, interconnects)
        self._setup_search_type()

    def clear(self):
        self._ic_combo.set_active(0)
        self._id_entry.set_text("")
        self._vendor_combo.set_active(0)

    def _filter_func(self, filter_by, row):
        if filter_by == self.SEARCH_TYPE_VENDOR:
            return self._vendor_combo.get_active_text() == row.vendor

        if filter_by == self.SEARCH_TYPE_INTERCONNECT:
            return self._ic_combo.get_active_text() == row.interconnect

        if filter_by == self.SEARCH_TYPE_ID:
            return self._id_entry.get_text().strip() in row.wwid

        return False

class ZPage(FilterPage):
    # Match these to zTypeCombo ids in glade
    SEARCH_TYPE_CCW = 'CCW'
    SEARCH_TYPE_WWPN = 'WWPN'
    SEARCH_TYPE_LUN = 'LUN'

    def __init__(self, builder):
        super().__init__(builder, "zModel", "zTypeCombo")
        self._ccw_entry = self._builder.get_object("zCCWEntry")
        self._wwpn_entry = self._builder.get_object("zWWPNEntry")
        self._lun_entry = self._builder.get_object("zLUNEntry")

    def clear(self):
        self._lun_entry.set_text("")
        self._ccw_entry.set_text("")
        self._wwpn_entry.set_text("")

    def is_member(self, device_type):
        return device_type == "zfcp" or device_type == "dasd"

    def setup(self, store, disks, selected_names, protected_names):
        """ Set up our Z-page, but only if we're running on s390x. """
        for device_data in disks:
            if device_data.type != "zfcp":
                continue

            row = create_row(
                device_data,
                device_data.name in selected_names,
                device_data.name not in protected_names
            )

            store.append([*row])

        self._setup_search_type()

    def _filter_func(self, filter_by, row):
        if filter_by == self.SEARCH_TYPE_CCW:
            return self._ccw_entry.get_text() in row.ccw

        if filter_by == self.SEARCH_TYPE_WWPN:
            return self._wwpn_entry.get_text() in row.wwpn

        if filter_by == self.SEARCH_TYPE_LUN:
            return self._lun_entry.get_text() in row.lun

        return False

class NvdimmPage(FilterPage):
    # Match these to nvdimmTypeCombo ids in glade
    SEARCH_TYPE_NAMESPACE = 'Namespace'
    SEARCH_TYPE_MODE = 'Mode'

    def __init__(self, builder):
        super().__init__(builder, "nvdimmModel", "nvdimmTypeCombo")
        self._tree_view = self._builder.get_object("nvdimmTreeView")
        self._mode_combo = self._builder.get_object("nvdimmModeCombo")
        self._namespace_entry = self._builder.get_object("nvdimmNamespaceEntry")

    def is_member(self, device_type):
        return device_type == "nvdimm"

    def setup(self, store, disks, selected_names, protected_names):
        modes = set()

        for device_data in disks:
            mode = device_data.attrs.get("mode")
            row = create_row(
                device_data,
                device_data.name in selected_names and mode == "sector",
                device_data.name not in protected_names or mode != "sector",
            )

            store.append([*row])
            modes.add(mode)

        self._setup_combo(self._mode_combo, modes)
        self._setup_search_type()

    def clear(self):
        self._mode_combo.set_active(0)
        self._namespace_entry.set_text("")

    def _filter_func(self, filter_by, row):
        if filter_by == self.SEARCH_TYPE_MODE:
            return self._mode_combo.get_active_text() == row.mode

        if filter_by == self.SEARCH_TYPE_NAMESPACE:
            return self._namespace_entry.get_text().strip() in row.namespace

        return False

    def get_selected_namespaces(self):
        namespaces = []
        selection = self._tree_view.get_selection()
        store, path_list = selection.get_selected_rows()

        for path in path_list:
            store_row = DiskStoreRow(*store[store.get_iter(path)])
            namespaces.append(store_row.namespace)

        return namespaces

class FilterSpoke(NormalSpoke):
    """
       .. inheritance-diagram:: FilterSpoke
          :parts: 3
    """
    builderObjects = ["diskStore", "filterWindow",
                      "searchModel", "multipathModel", "otherModel", "zModel", "nvdimmModel"]
    mainWidgetName = "filterWindow"
    uiFile = "spokes/advanced_storage.glade"
    category = SystemCategory
    title = CN_("GUI|Spoke", "_Installation Destination")

    @staticmethod
    def get_screen_id():
        """Return a unique id of this UI screen."""
        return "storage-advanced-configuration"

    def __init__(self, *args):
        super().__init__(*args)
        self.applyOnSkip = True

        self._pages = {}
        self._ancestors = []
        self._disks = []
        self._selected_disks = []
        self._protected_disks = []

        self._storage_module = STORAGE.get_proxy()
        self._device_tree = STORAGE.get_proxy(DEVICE_TREE)
        self._disk_selection = STORAGE.get_proxy(DISK_SELECTION)

        self._notebook = self.builder.get_object("advancedNotebook")
        self._store = self.builder.get_object("diskStore")
        self._reconfigure_nvdimm_button = self.builder.get_object("reconfigureNVDIMMButton")

    @property
    def indirect(self):
        return True

    # This spoke has no status since it's not in a hub
    @property
    def status(self):
        return None

    def apply(self):
        apply_disk_selection(self._selected_disks)

    def initialize(self):
        super().initialize()
        self.initialize_start()

        self._pages = {
            PAGE_SEARCH: SearchPage(self.builder),
            PAGE_MULTIPATH: MultipathPage(self.builder),
            PAGE_OTHER: OtherPage(self.builder),
            PAGE_NVDIMM: NvdimmPage(self.builder),
            PAGE_Z: ZPage(self.builder),
        }

        if not STORAGE.get_proxy(DASD).IsSupported():
            self._notebook.remove_page(PAGE_Z)
            self._pages.pop(PAGE_Z)

            self.builder.get_object("addZFCPButton").destroy()
            self.builder.get_object("addDASDButton").destroy()

        if not STORAGE.get_proxy(FCOE).IsSupported():
            self.builder.get_object("addFCOEButton").destroy()

        if not STORAGE.get_proxy(ISCSI).IsSupported():
            self.builder.get_object("addISCSIButton").destroy()

        # The button is sensitive only on NVDIMM page
        self._reconfigure_nvdimm_button.set_sensitive(False)

        # report that we are done
        self.initialize_done()

    def refresh(self):
        super().refresh()

        # Reset the scheduled partitioning if any to make sure that we
        # are working with the current system's storage configuration.
        # FIXME: Change modules and UI to work with the right device tree.
        self._storage_module.ResetPartitioning()

        self._disks = self._disk_selection.GetUsableDisks()
        self._selected_disks = self._disk_selection.SelectedDisks
        self._protected_disks = self._disk_selection.ProtectedDevices
        self._ancestors = self._device_tree.GetAncestors(self._disks)

        # Now all all the non-local disks to the store.  Everything has been set up
        # ahead of time, so there's no need to configure anything.  We first make
        # these lists of disks, then call setup on each individual page.  This is
        # because there could be page-specific setup to do that requires a complete
        # view of all the disks on that page.
        self._store.clear()

        disks_data = DeviceData.from_structure_list([
            self._device_tree.GetDeviceData(device_name)
            for device_name in self._disks
        ])

        for page in self._pages.values():
            disks = [
                d for d in disks_data
                if page.is_member(d.type)
            ]

            page.setup(
                self._store,
                disks,
                self._selected_disks,
                self._protected_disks
            )

        self._update_summary()

    def _update_summary(self):
        summary_button = self.builder.get_object("summary_button")
        label = self.builder.get_object("summary_button_label")

        # We need to remove ancestor devices from the count.  Otherwise, we'll
        # end up in a situation where selecting one multipath device could
        # potentially show three devices selected (mpatha, sda, sdb for instance).
        count = len([
            disk for disk in self._selected_disks
            if disk not in self._ancestors
        ])

        summary = CP_(
            "GUI|Installation Destination|Filter",
            "{} _storage device selected",
            "{} _storage devices selected",
            count
        ).format(count)

        if count > 0:
            really_show(summary_button)
            label.set_text(summary)
            label.set_use_underline(True)
        else:
            really_hide(summary_button)

    def on_back_clicked(self, button):
        self.skipTo = "StorageSpoke"
        super().on_back_clicked(button)

    def on_summary_clicked(self, button):
        disks = filter_disks_by_names(
            self._disks, self._selected_disks
        )
        dialog = SelectedDisksDialog(
            self.data, disks, show_remove=False, set_boot=False
        )

        with self.main_window.enlightbox(dialog.window):
            dialog.refresh()
            dialog.run()

    def on_clear_icon_clicked(self, entry, icon_pos, event):
        if icon_pos == Gtk.EntryIconPosition.SECONDARY:
            entry.set_text("")

    def on_page_switched(self, notebook, new_page, new_page_num, *args):
        # Disable all filters.
        for page in self._pages.values():
            page.is_active = False

        # Set up the new page.
        page = self._pages[new_page_num]
        page.is_active = True
        page.model.refilter()

        log.debug("Show the page %s.", str(page))

        # Set up the UI.
        notebook.get_nth_page(new_page_num).show_all()
        self._reconfigure_nvdimm_button.set_sensitive(new_page_num == 3)

    def on_row_toggled(self, button, path):
        if not path:
            return

        page_index = self._notebook.get_current_page()
        filter_model = self._pages[page_index].model
        model_itr = filter_model.get_iter(path)
        itr = filter_model.convert_iter_to_child_iter(model_itr)
        self._store[itr][1] = not self._store[itr][1]

        if self._store[itr][1] and self._store[itr][3] not in self._selected_disks:
            self._selected_disks.append(self._store[itr][3])
        elif not self._store[itr][1] and self._store[itr][3] in self._selected_disks:
            self._selected_disks.remove(self._store[itr][3])

        self._update_summary()

    @timed_action(delay=50, threshold=100)
    def on_refresh_clicked(self, widget, *args):
        log.debug("Refreshing...")
        try_populate_devicetree()
        self.refresh()

    def on_add_iscsi_clicked(self, widget, *args):
        log.debug("Add a new iSCSI device.")
        dialog = ISCSIDialog(self.data)
        self._run_dialog_and_refresh(dialog)

    def on_add_fcoe_clicked(self, widget, *args):
        log.debug("Add a new FCoE device.")
        dialog = FCoEDialog(self.data)
        self._run_dialog_and_refresh(dialog)

    def on_add_zfcp_clicked(self, widget, *args):
        log.debug("Add a new zFCP device.")
        dialog = ZFCPDialog(self.data)
        self._run_dialog_and_refresh(dialog)

    def on_add_dasd_clicked(self, widget, *args):
        log.debug("Add a new DASD device.")
        dialog = DASDDialog(self.data)
        self._run_dialog_and_refresh(dialog)

    def on_reconfigure_nvdimm_clicked(self, widget, *args):
        log.debug("Reconfigure a NVDIMM device.")
        namespaces = self._pages[PAGE_NVDIMM].get_selected_namespaces()
        dialog = NVDIMMDialog(self.data, namespaces)
        self._run_dialog_and_refresh(dialog)

    def _run_dialog_and_refresh(self, dialog):
        # Run the dialog.
        with self.main_window.enlightbox(dialog.window):
            dialog.refresh()
            dialog.run()

        # We now need to refresh so any new disks picked up by adding advanced
        # storage are displayed in the UI.
        self.refresh()

    @timed_action(delay=1200, busy_cursor=False)
    def on_filter_changed(self, *args):
        self._refilter_current_page()

    def on_search_type_changed(self, combo):
        self._set_notebook_page("searchTypeNotebook", combo.get_active())
        self._refilter_current_page()

    def on_multipath_type_changed(self, combo):
        self._set_notebook_page("multipathTypeNotebook", combo.get_active())
        self._refilter_current_page()

    def on_other_type_combo_changed(self, combo):
        self._set_notebook_page("otherTypeNotebook", combo.get_active())
        self._refilter_current_page()

    def on_nvdimm_type_combo_changed(self, combo):
        self._set_notebook_page("nvdimmTypeNotebook", combo.get_active())
        self._refilter_current_page()

    def on_z_type_combo_changed(self, combo):
        self._set_notebook_page("zTypeNotebook", combo.get_active())
        self._refilter_current_page()

    def _set_notebook_page(self, notebook_name, page_index):
        notebook = self.builder.get_object(notebook_name)
        notebook.set_current_page(page_index)
        self._refilter_current_page()

    def _refilter_current_page(self):
        index = self._notebook.get_current_page()
        page = self._pages[index]
        page.model.refilter()


from fastapi import APIRouter, Depends, status, HTTPException, Response
from sqlalchemy.orm import Session

from .. import model, schemas, database, oauth2

router = APIRouter(prefix="/vote", tags=["Vote"])

@router.post("/", status_code=status.HTTP_201_CREATED)
def vote(
    vote: schemas.Vote,
    db: Session = Depends(database.get_db),
    user_id: schemas.TokenData = Depends(oauth2.get_current_user),
):
    vote_query = db.query(model.Vote).filter(
        model.Vote.post_id == vote.post_id, model.Vote.user_id == user_id.id
    )
    found_vote = vote_query.first()
    if vote.dir == 1:
        if found_vote:
            raise HTTPException(
                status_code=status.HTTP_409_CONFLICT, detail="Already voted"
            )
        new_vote = model.Vote(post_id=vote.post_id, user_id=user_id.id)
        db.add(new_vote)
        db.commit()
        return {"message": "successfully added vote"}
    if not found_vote:
        raise HTTPException(
            status_code=status.HTTP_409_CONFLICT, detail="vote not exists"
        )
    vote_query.delete(synchronize_session=False)
    db.commit()
    return {"message": "successfully deleted vote"}


# 90%대에서 틀렸습니다. 예외 케이스 존재

import sys
from collections import deque
n, m = map(int, sys.stdin.readline().split())

a = [list(map(int, sys.stdin.readline().split())) for i in range(n)]
d = [[0]*m for i in range(n)]
q = deque()
q.append((a[0][0], 0, 0))
d[0][0] = a[0][0]
while q:
    candy, r, c = q.popleft()
    for dr, dc in ((1,0),(0,1),(1,1)):
        nr = r + dr
        nc = c + dc
        if nr < 0 or nc < 0 or nr >= n or nc >= m: continue
        if d[r][c] + a[nr][nc] > d[nr][nc]:
            d[nr][nc] = d[r][c] + a[nr][nc]
            q.append((d[nr][nc], nr, nc))

print(d[n-1][m-1])


import gi
gi.require_version('Gtk', '3.0')
import requests
import json
import threading
import os
from gi.repository import Gtk, GdkPixbuf, GLib, Pango
# Import the modularized components
from ACDcomponents import ThumbnailBox, ImageButton, AmiiboFilterBox
from ACDutils import fetch_amiibo_data, load_image_data

class AmiiboApp(Gtk.Box):
    def __init__(self):
        super(AmiiboApp, self).__init__(orientation=Gtk.Orientation.VERTICAL)

        self.amiibo_images = []
        self.selected_images = []
        self.image_thumbnail_size = 100

        main_box = self  # Now, AmiiboApp itself is the main box

        # Start - Logo and Label code
        logo_label_box = Gtk.Box(orientation=Gtk.Orientation.HORIZONTAL, spacing=10)
        main_box.pack_start(logo_label_box, False, False, 10)

        logo_pixbuf = GdkPixbuf.Pixbuf.new_from_file("Amiibo_logo.png")
        logo_image = Gtk.Image.new_from_pixbuf(logo_pixbuf)
        logo_label_box.pack_start(logo_image, False, False, 0)

        title_label = Gtk.Label("Cards Downloader")
        title_label.set_size_request(300, 0)

        font_desc = title_label.get_pango_context().get_font_description()
        font_desc.set_size(18 * Pango.SCALE)
        title_label.modify_font(font_desc)

        logo_label_box.pack_start(title_label, False, False, 0)

        filter_box = Gtk.Box(orientation=Gtk.Orientation.HORIZONTAL)
        main_box.pack_start(filter_box, False, False, 5)

        self.search_entry = Gtk.Entry()
        self.search_entry.set_placeholder_text("Search")
        self.search_entry.connect("changed", self.apply_filters)
        filter_box.pack_start(self.search_entry, False, False, 5)

        self.amiibo_series_filter = Gtk.ComboBoxText()
        self.amiibo_series_filter.connect("changed", self.apply_filters)
        filter_box.pack_start(self.amiibo_series_filter, False, False, 5)

        content_box = Gtk.Box(orientation=Gtk.Orientation.HORIZONTAL)
        main_box.pack_start(content_box, True, True, 0)

        scrolled_window = Gtk.ScrolledWindow()
        self.image_listbox = Gtk.ListBox()
        scrolled_window.add(self.image_listbox)
        content_box.pack_start(scrolled_window, True, True, 5)

        details_box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL)
        self.large_image = Gtk.Image()

        details_box.pack_start(self.large_image, True, True, 5)
        self.info_label = Gtk.Label.new("0/9 images selected")
        details_box.pack_start(self.info_label, False, False, 5)
        self.download_button = Gtk.Button(label="Download")
        self.download_button.connect("clicked", self.download_selected_images)
        details_box.pack_start(self.download_button, False, False, 5)
        self.deselect_button = Gtk.Button(label="Deselect All")
        self.deselect_button.connect("clicked", self.deselect_all_images)
        details_box.pack_start(self.deselect_button, False, False, 5)

        content_box.pack_start(details_box, False, False, 5)
        self.show_all()

        threading.Thread(target=self.load_images, daemon=True).start()

    def load_images(self):
        response = requests.get('https://www.amiiboapi.com/api/amiibo/')
        amiibo_data = json.loads(response.text)
        self.amiibo_images = amiibo_data['amiibo']

        amiibo_series = set()
        for amiibo in self.amiibo_images:
            amiibo_series.add(amiibo.get('amiiboSeries', 'Unknown'))

        GLib.idle_add(self.amiibo_series_filter.append_text, "Select Amiibo Series")
        for series in sorted(amiibo_series):
            GLib.idle_add(self.amiibo_series_filter.append_text, series)

        GLib.idle_add(self.amiibo_series_filter.set_active, 0)

    def apply_filters(self, *args):
        search_query = self.search_entry.get_text().strip().lower()
        amiibo_series_filter = self.amiibo_series_filter.get_active_text()

        if amiibo_series_filter == "Select Amiibo Series":
            amiibo_series_filter = None

        for row in self.image_listbox.get_children():
            row.destroy()

        for index, amiibo in enumerate(self.amiibo_images):
            if search_query and search_query not in amiibo['name'].lower():
                continue

            if amiibo_series_filter and amiibo_series_filter not in amiibo['amiiboSeries']:
                continue

            thumbnail_url = amiibo['image']
            name = amiibo['name']
            series = amiibo['amiiboSeries']
            character = amiibo['character']
            amiibo_type = amiibo['type']

            def create_row(image_data, name, series, character, amiibo_type, url):
                thumbnail = ThumbnailBox(image_data, name, series, character, amiibo_type)
                image_button = ImageButton(thumbnail, url)
                image_button.connect("clicked", self.select_image, amiibo)
                self.image_listbox.add(image_button)
                image_button.show_all()

            threading.Thread(target=self.load_image_thumbnail, args=(thumbnail_url, name, series, character, amiibo_type, create_row), daemon=True).start()

    def load_image_thumbnail(self, url, name, series, character, amiibo_type, callback):
        image_data = self.load_image_data(url, self.image_thumbnail_size)
        GLib.idle_add(callback, image_data, name, series, character, amiibo_type, url)

    def load_image_data(self, url, size=None):
        response = requests.get(url)
        loader = GdkPixbuf.PixbufLoader()
        loader.write(response.content)
        loader.close()

        pixbuf = loader.get_pixbuf()
        if size:
            aspect_ratio = pixbuf.get_height() / pixbuf.get_width()
            new_height = size * aspect_ratio
            pixbuf = pixbuf.scale_simple(size, int(new_height), GdkPixbuf.InterpType.BILINEAR)

        return pixbuf

    def select_image(self, button, amiibo):
        thumbnail_box = button.get_thumbnail_widget()
        image_widget = thumbnail_box.get_image()
        url = button.get_url()

        if len(self.selected_images) >= 9 and image_widget.get_opacity() == 1:
            return

        if image_widget.get_opacity() < 1:
            image_widget.set_opacity(1)
            self.selected_images.remove(url)
        else:
            image_widget.set_opacity(0.3)
            self.selected_images.append(url)
            large_image_data = self.load_image_data(url)
            self.large_image.set_from_pixbuf(large_image_data)
            self.large_image.set_size_request(300, -1)
            self.large_image.props.icon_size = Gtk.IconSize.DIALOG

        self.update_info_label()
        self.download_button.set_sensitive(len(self.selected_images) == 9)

    def deselect_all_images(self, button):
        for row in self.image_listbox.get_children():
            image_button = row.get_children()[0]
            thumbnail_box = image_button.get_thumbnail_widget()
            image_widget = thumbnail_box.get_image()
            image_widget.set_opacity(1)

        self.selected_images.clear()
        self.update_info_label()
        self.download_button.set_sensitive(False)

    def download_selected_images(self, button):
        amiibo_folder = "amiibo"
        if not os.path.exists(amiibo_folder):
            os.makedirs(amiibo_folder)

        for index, url in enumerate(self.selected_images):
            image_data = self.load_image_data(url)
            file_path = os.path.join(amiibo_folder, f"image{index+1}.png")
            image_data.savev(file_path, 'png', [], [])

        print(f"{len(self.selected_images)} images downloaded to {amiibo_folder}")

    def update_info_label(self):
        self.info_label.set_text(f"{len(self.selected_images)}/9 images selected")


#!/usr/bin/env python
'''
An export method producing for GDML.

http://lcgapp.cern.ch/project/simu/framework/GDML/doc/GDMLmanual.pdf
http://gdml1.web.cern.ch/gdml1/doc/g4gdml_howto.html
http://geant4.in2p3.fr/2007/prog/GiovanniSantin/GSantin_Geant4_Paris07_Materials_v08.pdf

http://lxml.de/tutorial.html
http://lxml.de/objectify.html
http://stackoverflow.com/questions/2850823/multiple-xml-namespaces-in-tag-with-lxml

'''

import os

schema_dir = os.path.join(os.path.dirname(__file__), 'schema')
schema_file = os.path.join(schema_dir, 'gdml.xsd')

from lxml import etree
from gegede import Quantity
from gegede.iter import ascending
from gegede.export import pod

def qtus(q,unit):
    'Quantity to unit string'
    return str(q.to(unit).magnitude)

def nt_qunit2xmldict(nt, unit):
    ret = dict(unit=unit)
    for k,v in zip(nt._fields,nt):
        if Quantity == type(v):
            v = str(v.to(unit).magnitude)
        else:
            v = str(v)
        ret[k] = v
    return ret

def D(obj):
    return str(obj.density.to('g/cc').magnitude)
def Atom(obj):
    return str(obj.a.to('g/mole').magnitude)
def Symbol(obj):
    return obj.symbol or ""

# def wash(geom):
#     '''Return a new geom object with its contents "washed" of any things
#     that will tickle GDML limitations.
#     '''
#     import gegede.construct
#     newg = gegede.construct.Geometry(geom.schema)
#     for store_name in geom.store._fields:
#         store = getattr(geom.store,store_name)
#         newstore = getattr(newg.store,store_name)
#         for name, obj in store.items():
#             newobj = type(obj)(obj.name.replace(' ','_'), *obj[1:])
#             newstore[name.replace(' ','_')] = newobj
#     return newg

def make_material_node(obj):
    '''
    Return an lxml.etree.Element made from the matter object.
    '''

    typename = type(obj).__name__
    node = None

    if typename == 'Element':
        node = etree.Element('element', name=obj.name, Z=str(obj.z), formula=Symbol(obj))
        node.append(etree.Element('atom', value=Atom(obj)))

    if typename == 'Isotope':
        node = etree.Element('isotope', name=obj.name, Z=str(obj.z), N=str(obj.ia))
        node.append(etree.Element('atom', type="A", value=Atom(obj)))

    if typename == 'Composition':
        node = etree.Element('element', name=obj.name)
        for isoname, isofrac in obj.isotopes:
            node.append(etree.Element('fraction', ref=isoname, n=str(isofrac)))

    if typename == 'Amalgam':
        node = etree.Element('material', name=obj.name, Z=str(float(obj.z)))
        # fixme: units???
        node.append(etree.Element('D', value=D(obj)))
        node.append(etree.Element('atom', value=Atom(obj)))
        for propname, propvect in obj.properties:
            node.append(etree.Element('property',
                                      name=propname,
                                      ref=obj.name+'_'+propname+'_VALUE'))

    if typename == 'Molecule':
        node = etree.Element('material', name=obj.name, formula=Symbol(obj))
        node.append(etree.Element('D', value=D(obj)))
        for elename, elenum in obj.elements:
            node.append(etree.Element('composite', ref=elename, n=str(elenum)))
        for propname, propvect in obj.properties:
            node.append(etree.Element('property',
                                      name=propname,
                                      ref=obj.name+'_'+propname+'_VALUE'))

    if typename == 'Mixture':
        node = etree.Element('material', name=obj.name, formula=Symbol(obj))
        node.append(etree.Element('D', value=D(obj)))
        for compname, compfrac in obj.components:
            node.append(etree.Element('fraction', ref=compname, n=str(compfrac)))
        for propname, propvect in obj.properties:
            node.append(etree.Element('property',
                                      name=propname,
                                      ref=obj.name+'_'+propname+'_VALUE'))

    return node

def make_shape_node(shape):
    '''
    Return an lxml.etree.Element made from the <shape>
    '''
    lunit = 'cm'
    aunit = 'radian'
    typename = type(shape).__name__

    def unitify(unit, quant, scale=1.0):
        quant = quant * scale
        return str(quant.to(unit).magnitude)
    def dsize(quant):
        return unitify(lunit, quant, 2.0)
    def rsize(quant):
        return unitify(lunit, quant, 1.0)
    def ang(quant):
        return unitify(aunit, quant, 1.0)

    if typename == 'Box':
        dat = dict(name=shape.name, lunit=lunit,
                   x=dsize(shape.dx), y=dsize(shape.dy), z=dsize(shape.dz))
        return etree.Element('box', **dat)

    if typename == 'TwistedBox':
        dat = dict(name=shape.name, lunit=lunit,
                   x=dsize(shape.dx), y=dsize(shape.dy), z=dsize(shape.dz), PhiTwist=ang(shape.phitws))
        return etree.Element('twistedbox', **dat)

    if typename == 'Tubs':
        dat = dict(name=shape.name, lunit=lunit, aunit=aunit,
                   rmin=rsize(shape.rmin), rmax=rsize(shape.rmax), z=dsize(shape.dz),
                   startphi=ang(shape.sphi), deltaphi=ang(shape.dphi))
        return etree.Element('tube', **dat)

    if typename == 'Sphere':
        dat = dict(name=shape.name, lunit=lunit, aunit=aunit,
                   rmin=rsize(shape.rmin), rmax=rsize(shape.rmax),
                   startphi=ang(shape.sphi), deltaphi=ang(shape.dphi),
                   starttheta=ang(shape.stheta), deltatheta=ang(shape.dtheta))
        return etree.Element('sphere', **dat)

    if typename == 'Cone':
        dat = dict(name=shape.name, lunit=lunit, aunit=aunit,
                    rmin1=rsize(shape.rmin1), rmax1=rsize(shape.rmax1),
                    rmin2=rsize(shape.rmin2), rmax2=rsize(shape.rmax2), z=dsize(shape.dz),
                    startphi=ang(shape.sphi), deltaphi=ang(shape.dphi))
        return etree.Element('cone', **dat)

    if typename == 'Trapezoid':
        dat = dict(name=shape.name, lunit=lunit, aunit=aunit,
                    x1=dsize(shape.dx1), x2=dsize(shape.dx2),
                    y1=dsize(shape.dy1), y2=dsize(shape.dy2), z=dsize(shape.dz))
        return etree.Element('trd', **dat)

    if typename == 'TwistedTrap':
        dat = dict(name=shape.name, lunit=lunit, aunit=aunit,
                    x1=dsize(shape.dx1), x2=dsize(shape.dx2), x3=dsize(shape.dx3), x4=dsize(shape.dx4),
                    y1=dsize(shape.dy1), y2=dsize(shape.dy2), z=dsize(shape.dz),
                    Theta=ang(shape.dtheta), Phi=ang(shape.dphi), Alph=ang(shape.dalpha), PhiTwist=ang(shape.phitws))
        return etree.Element('twistedtrap', **dat)

    if typename == 'TwistedTrd':
        dat = dict(name=shape.name, lunit=lunit, aunit=aunit,
                    x1=dsize(shape.dx1), x2=dsize(shape.dx2),
                    y1=dsize(shape.dy1), y2=dsize(shape.dy2), z=dsize(shape.dz),
                    PhiTwist=ang(shape.phitws))
        return etree.Element('twistedtrd', **dat)

    if typename == 'Arb8':
        # print("arb8ivert",shape.ivert[0][0])
        dat = dict(name=shape.name, lunit=lunit, aunit=aunit,
                   v1x=dsize(shape.v1x),v1y=dsize(shape.v1y),
                   v2x=dsize(shape.v2x),v2y=dsize(shape.v2y),
                   v3x=dsize(shape.v3x),v3y=dsize(shape.v3y),
                   v4x=dsize(shape.v4x),v4y=dsize(shape.v4y),
                   v5x=dsize(shape.v5x),v5y=dsize(shape.v5y),
                   v6x=dsize(shape.v6x),v6y=dsize(shape.v6y),
                   v7x=dsize(shape.v7x),v7y=dsize(shape.v7y),
                   v8x=dsize(shape.v8x),v8y=dsize(shape.v8y),
                    dz=dsize(shape.dz))
        return etree.Element('arb8', **dat)

    if typename == 'Paraboloid':
        dat = dict(name=shape.name, lunit=lunit, aunit=aunit,
                    rlo=dsize(shape.drlo), rhi=dsize(shape.drhi),
                    dz=dsize(shape.ddz))
        return etree.Element('paraboloid', **dat)

    if typename == 'Ellipsoid':
        dat = dict(name=shape.name, lunit=lunit, aunit=aunit,
                    ax=dsize(shape.dax), by=dsize(shape.dby), cz=dsize(shape.dcz),
                    zcut1=dsize(shape.dzcut1), zcut2=dsize(shape.dzcut2))
        return etree.Element('ellipsoid', **dat)

    if typename == 'PolyhedraRegular':
        dat = dict(name=shape.name, lunit=lunit, aunit=aunit,
                    startphi=ang(shape.sphi), deltaphi=ang(shape.dphi), numsides=str(int(shape.numsides)))
        ele = etree.Element('polyhedra', **dat)
        zpos = shape.dz / 2
        zneg = - shape.dz / 2
        ele.append(etree.Element('zplane', rmin=rsize(shape.rmin), rmax=rsize(shape.rmax), z=dsize(zpos)))
        ele.append(etree.Element('zplane', rmin=rsize(shape.rmin), rmax=rsize(shape.rmax), z=dsize(zneg)))
        return ele

    if typename == 'Boolean':
        ele = etree.Element(shape.type, name=shape.name)
        # the rest are sub nodes.  why?  because, don't ask questions!
        ele.append(etree.Element('first', ref=shape.first))
        ele.append(etree.Element('second', ref=shape.second))
        ele.append(etree.Element('positionref', ref = shape.pos or 'center'))
        ele.append(etree.Element('rotationref', ref = shape.rot or 'identity'))
        return ele

    if typename == 'EllipticalTube':
        dat = dict(name=shape.name, lunit=lunit, aunit=aunit,
                    dx=dsize(0.5*shape.dx), dy=dsize(0.5*shape.dy), dz=dsize(0.5*shape.dz))
        return etree.Element('eltube', **dat)

    if typename == 'Torus':
        dat = dict(name=shape.name, lunit=lunit, aunit=aunit,
                    rmin=rsize(shape.rmin), rmax=rsize(shape.rmax), rtor=rsize(shape.rtor),
                    startphi=ang(shape.startphi), deltaphi=ang(shape.deltaphi))
        return etree.Element('torus', **dat)

    # etc....  Grow this as gegede.schema.Schema.shapes grows....

    return

def make_volume_node(vol, store):
    #print ('VOL',vol)
    node_type = 'volume'
    if vol.material is None and vol.shape is None:
        node = etree.Element('assembly', name=vol.name)
    else:
        assert vol.material and vol.shape
        node = etree.Element('volume', name=vol.name)
        node.append(etree.Element('materialref', ref=vol.material))
        node.append(etree.Element('solidref', ref=vol.shape))
    for placename in vol.placements or []:
        place = store[placename]
        if place.copynumber:
            pvol = etree.Element('physvol', copynumber=str(place.copynumber))
        else:
            pvol = etree.Element('physvol')
        node.append(pvol)
        pvol.append(etree.Element('volumeref', ref=place.volume))
        if place.pos:
            pvol.append(etree.Element('positionref', ref=place.pos))
        else:
            pvol.append(etree.Element('positionref', ref='center'))
        if place.rot:
            pvol.append(etree.Element('rotationref', ref=place.rot))
        else:
            pvol.append(etree.Element('rotationref', ref='identity'))
    for parname, parval in vol.params or []:
        pnode = etree.Element('auxiliary', auxtype=parname, auxvalue=parval)
        node.append(pnode)
    return node

def convert(geom):
    '''
    Return an lxml.etree formed from the geometry
    '''

    gdml_node = etree.Element("gdml")

    # I have no idea what this means but it reproduces what other GDML
    # files have and w/out it Geant4 complains.
    gdml_node.set('{http://www.w3.org/2001/XMLSchema-instance}noNamespaceSchemaLocation',
                  'http://service-spi.web.cern.ch/service-spi/app/releases/GDML/schema/gdml.xsd')

    # <define>
    define_node = etree.Element('define')
    gdml_node.append(define_node)
    center = identity = None
    for name, obj in geom.store.structure.items():
        typename = type(obj).__name__.lower()
        node = None
        if typename == 'position':
            node = etree.Element('position', **nt_qunit2xmldict(obj, 'cm'))
            if obj.name == 'center':
                center = obj
        if typename == 'rotation':
            node = etree.Element('rotation', **nt_qunit2xmldict(obj, 'degree'))
            if obj.name == 'identity':
                identity = obj
        if node is not None:
            define_node.append(node)
        continue
    for name, obj in geom.store.matter.items():
        typename = type(obj).__name__.lower()
        if typename=='mixture' or typename=='molecule' or typename=='amalgam':
            for prop, val in obj.properties:
                vals = str(val[0])
                for v in val[1:]: vals += ' ' + str(v)
                define_node.append(etree.Element('matrix',
                                                 name=name+'_'+prop+'_VALUE',
                                                 coldim=str(len(val)),
                                                 values=vals))
                continue
            continue
        continue
    if center is None:
        define_node.append(etree.Element('position', name='center'))
    if identity is None:
        define_node.append(etree.Element('rotation', name='identity'))

    # <materials>
    materials_node = etree.Element('materials')
    gdml_node.append(materials_node)
    for name, obj in geom.store.matter.items():
        node = make_material_node(obj)
        if node is not None:
            materials_node.append(node)

    # <solids>
    solids_node = etree.Element('solids')
    gdml_node.append(solids_node)
    for obj in geom.store.shapes.values():
        node = make_shape_node(obj)
        if node is not None:
            solids_node.append(node)

    # <structure>
    structure_node = etree.Element('structure')
    gdml_node.append(structure_node)
    for vol in ascending(geom.store.structure, geom.world):
        assert vol
        node = make_volume_node(vol, geom.store.structure)
        if node is not None:
            structure_node.append(node)

    # <setup>
    setup_node = etree.Element('setup', name="Default", version="0")
    gdml_node.append(setup_node)

    world_node = etree.Element('world', ref=geom.world)
    setup_node.append(world_node)

    return gdml_node

def validate(text):
    try:
        from StringIO import StringIO
        sio = StringIO(text)
    except ImportError:         # python3
        from io import BytesIO
        sio = BytesIO(text)

    xsd_doc = etree.parse(schema_file)
    xsd = etree.XMLSchema(xsd_doc)
    xml = etree.parse(sio)
    okay = xsd.validate(xml)
    if not okay:
        print (xsd.error_log)
        raise ValueError('Invalid GDML')
    return True

def validate_object(obj):
    return validate(dumps(obj))

#def validate_output(obj, filename):
#    return False

def dumps(obj):
    '''
    Return a string representation of the object returned by convert.
    '''
    xml = etree.tostring(obj, pretty_print = True, xml_declaration = True)
    xml = xml.replace(b"'",b'"')  # work around ROOT GDML import bug....
    # don't validate here
    return xml

def output(obj, filename):
    '''
    Save to file
    '''
    open(filename,'wb').write(dumps(obj))


from ..import model
from ...lib import config
import logging, jwt
_LOGGER = logging.getLogger()

class Notification:
    def __init__(self):
        _LOGGER.info("[init] Notification")
        self.model = model.model('notification')

    def _mapOne(self, rDocument):
        rDocument["_id"] = str(rDocument["_id"])
        return rDocument

    def _map(self, rDocuments):
        for (i, doc) in enumerate(rDocuments):
            rDocuments[i]["_id"] = str(doc["_id"])
        return rDocuments

    def get(self, doc, ifNone={}):
        _LOGGER.info("getting... ")
        _LOGGER.info(f"{doc}")
        rDocument = [self.model.find_one(doc)]
        _LOGGER.info(rDocument)
        if(rDocument == [None]): return ifNone
        return list(map(self._mapOne, rDocument))[0]

    def getMany(self, doc):
        res = [self.model.find(doc)]
        _LOGGER.info(res)
        return list(map(self._map, res))

    # def new(self,user,doc):
    def new(self, doc):
        _LOGGER.info("inserting... ")
        _LOGGER.info(f"{doc}")
        # res = self.model.update_one(user, {"$set": doc})
        res = self.model.insert_one(doc)
        _LOGGER.info(res)
        return 0
        # return str(res.documentIds[0])

    def redact(self, _id, doc):
        _LOGGER.info("redacting... ")
        _LOGGER.info(f"{doc}")
        res = self.model.update_one({"_id": _id}, {"$set": doc})
        _LOGGER.info(res)
        return 0
        # return str(res.documentIds[0])

    # - RESERVED FOR FUTER UPDATE
    # def delete(self,doc):
    #     _LOGGER.info("deleting... ")
    #     _LOGGER.info(f"{doc}")
    #     res = self.model.delete_one(doc)
    #     _LOGGER.info(res)
    #     return 0


import gym
import numpy as np
from rl.lib.cross_entropy.cem import CrossEntropyAgent

NUM_EPISODES = 10000000000
MAX_ITERATIONS = 10**6
REFIT_IT = 300

def main():
  env = gym.make('CartPole-v0')

  cem_agent = CrossEntropyAgent(4, 2)

  curr_rewards = []

  for episode in range(1, NUM_EPISODES):
      observation = env.reset()
      theta = cem_agent.sample_theta()

      total_rewards = 0

      for iteration in range(MAX_ITERATIONS):
          action = cem_agent.get_action(observation, theta)

          observation, reward, done, info = env.step(action)
          if done:
              break
          total_rewards += reward

          if done:
              # print('Episode {}, iterations: {}'.format(
              #     episode,
              #     iteration
              # ))
              break

      cem_agent.report_reward(total_rewards)
      curr_rewards.append(total_rewards)

      if episode % REFIT_IT == 0:
          print('Current rewards: ', np.mean(curr_rewards))
          curr_rewards = []
          cem_agent.refit()

if __name__ == '__main__':
    main()


# -*- codeing = utf-8 -*-
# @Time :2022/6/27 9:00
# @Author:Voyage
# @File : createdata.py
# @Software: PyCharm
#创建数据表录入各年份各专业各学校的数据库
import os

import MysqlConfig
from pyspark import Row
from pyspark.sql import SparkSession
import pandas as pd
os.environ["PYSPARK_PYTHON"] = r"C:\Users\61X\miniconda3\envs\py36\python.exe"

spark = SparkSession \
    .builder \
    .appName("Python Spark SQL basic example") \
    .config("spark.some.config.option", "some-value") \
    .getOrCreate()
# sc = spark.sparkContext

#将文件上传
df = pd.read_excel(r"C:\Users\61X\MyUniverse\SourceCode\PythonProjects\practice\intelligent-recommendations-system-of-college-choosing\compute\data\DivByMajor\2021四川理科高校分专业数据_region.xlsx")
df["year"]=2021 #pandas新增一列，加在最后
col = ['min1','max1','min_section','year']
df[col] = df[col].fillna(0.0)
df = df.replace(pd.NA,'')#空值替换
df = df.replace('-',0)#空值替换
# df = df.fillna(0)
# df = df.replace(pd.NA,'')#空值替换
df= df.astype(dtype={'min1':'float','max1':'float','min_section':'int','year':'int'})

df_spark_excel = spark.createDataFrame(df)  # 转换为spark格式
conn_param = {}
conn_param['user'] = MysqlConfig.MYSQL_USER
conn_param['password'] = MysqlConfig.MYSQL_PWD
conn_param['driver'] = MysqlConfig.MYSQL_DRIVER
df_spark_excel.write.jdbc(MysqlConfig.MYSQL_CONN, 'div_by_major', 'append', conn_param)##第一次用下面的overwrite之后用append
# df_spark_excel.write.jdbc(MysqlConfig.MYSQL_CONN, 'div_by_major', 'overwrite', conn_param)
print("执行完毕")


#!/usr/bin/env python

from __future__ import division, print_function, absolute_import

import numpy as np
from numpy.testing import run_module_suite, assert_allclose, assert_

import pywt

def test_upcoef_docstring():
    data = [1, 2, 3, 4, 5, 6]
    (cA, cD) = pywt.dwt(data, 'db2', 'smooth')
    rec = pywt.upcoef('a', cA, 'db2') + pywt.upcoef('d', cD, 'db2')
    expect = [-0.25, -0.4330127, 1., 2., 3., 4., 5.,
              6., 1.78589838, -1.03108891]
    assert_allclose(rec, expect)
    n = len(data)
    rec = (pywt.upcoef('a', cA, 'db2', take=n) +
           pywt.upcoef('d', cD, 'db2', take=n))
    assert_allclose(rec, data)

def test_upcoef_reconstruct():
    data = np.arange(3)
    a = pywt.downcoef('a', data, 'haar')
    d = pywt.downcoef('d', data, 'haar')

    rec = (pywt.upcoef('a', a, 'haar', take=3) +
           pywt.upcoef('d', d, 'haar', take=3))
    assert_allclose(rec, data)

def test_downcoef_multilevel():
    rstate = np.random.RandomState(1234)
    r = rstate.randn(16)
    nlevels = 3
    # calling with level=1 nlevels times
    a1 = r.copy()
    for i in range(nlevels):
        a1 = pywt.downcoef('a', a1, 'haar', level=1)
    # call with level=nlevels once
    a3 = pywt.downcoef('a', r, 'haar', level=nlevels)
    assert_allclose(a1, a3)

def test_compare_downcoef_coeffs():
    rstate = np.random.RandomState(1234)
    r = rstate.randn(16)
    # compare downcoef against wavedec outputs
    for nlevels in [1, 2, 3]:
        for wavelet in pywt.wavelist():
            wavelet = pywt.Wavelet(wavelet)
            max_level = pywt.dwt_max_level(r.size, wavelet.dec_len)
            if nlevels <= max_level:
                a = pywt.downcoef('a', r, wavelet, level=nlevels)
                d = pywt.downcoef('d', r, wavelet, level=nlevels)
                coeffs = pywt.wavedec(r, wavelet, level=nlevels)
                assert_allclose(a, coeffs[0])
                assert_allclose(d, coeffs[1])

def test_upcoef_multilevel():
    rstate = np.random.RandomState(1234)
    r = rstate.randn(4)
    nlevels = 3
    # calling with level=1 nlevels times
    a1 = r.copy()
    for i in range(nlevels):
        a1 = pywt.upcoef('a', a1, 'haar', level=1)
    # call with level=nlevels once
    a3 = pywt.upcoef('a', r, 'haar', level=nlevels)
    assert_allclose(a1, a3)

def test_wavelet_repr():
    from pywt._extensions import _pywt
    wavelet = _pywt.Wavelet('sym8')

    repr_wavelet = eval(wavelet.__repr__())

    assert_(wavelet.__repr__() == repr_wavelet.__repr__())

def test_dwt_max_level():
    assert_(pywt.dwt_max_level(16, 2) == 4)
    assert_(pywt.dwt_max_level(16, 8) == 1)
    assert_(pywt.dwt_max_level(16, 9) == 1)
    assert_(pywt.dwt_max_level(16, 10) == 0)
    assert_(pywt.dwt_max_level(16, 18) == 0)

if __name__ == '__main__':
    run_module_suite()


from __future__ import annotations

import re
from typing import List

from docinpy import Struct

"""
This module is for parse and extract comments
"""

# three_quote_pattern is multi-line comment pattern
three_quote_pattern = re.compile(r'(^|\n)\s*?"""([^(""")]*?)"""', re.DOTALL)

def prepare_raw_comment_struct(parent_src: str) -> List[Struct]:
    min_pos_for_line = get_min_pos_for_line(parent_src)
    min_line_no = 1
    comment_struct_list = []
    matches = three_quote_pattern.finditer(parent_src)
    for match in matches:
        comment_content = match.group(2)
        start = match.group(1)
        start_pos, end_pos = match.span()
        start_pos += len(start)
        start_line_no = find_line_no(start_pos, min_line_no, min_pos_for_line)
        end_line_no = find_line_no(end_pos, start_line_no, min_pos_for_line)
        min_line_no = end_line_no
        comment_pos = (start_line_no, end_line_no)
        comment_struct_list.append(Struct("raw_comment", comment_content, comment_pos))
    return comment_struct_list

def get_min_pos_for_line(src: str) -> List[int]:
    src_lines = src.split("\n")
    min_pos_for_line = [0]
    for line in src_lines:
        min_pos_for_line.append(min_pos_for_line[-1] + len(line) + 1)
    return min_pos_for_line

def find_line_no(pos: int, min_line_no, min_pos_for_line: List[int]) -> int:
    for i in range(min_line_no - 1, len(min_pos_for_line)):
        if pos < min_pos_for_line[i]:
            return i

section_pattern = re.compile(r"(\n\s*?#+ .*)")

def parse_raw_comments(root_struct: Struct):
    new_children = []
    for i, struct in enumerate(root_struct.children):
        if struct.struct_type == "raw_comment":
            comment_content = struct.obj
            comment_structs = process_raw_comment_content(comment_content)
            new_children.extend(comment_structs)
        else:
            new_children.append(struct)
            if struct.struct_type == "class":
                parse_raw_comments(struct)
    root_struct.children = new_children

def process_raw_comment_content(comment_content: str) -> List[Struct]:
    min_pos_for_line = get_min_pos_for_line(comment_content)
    min_line_no = 1

    structs_in_comment = []
    # Find all sections which should start with one or more # followed by a space
    section_matches = section_pattern.finditer(comment_content)
    last_section_end = 0
    for section_match in section_matches:
        section_markdown = section_match.group(1)[1:].lstrip()
        # find first non-# character
        section_title = section_markdown.lstrip("#")
        section_level = len(section_markdown) - len(section_title)
        section_title = section_title.strip()
        section_start, section_end = section_match.span()
        start_line_no = find_line_no(section_start, min_line_no, min_pos_for_line)
        end_line_no = find_line_no(section_end, start_line_no, min_pos_for_line)
        min_line_no = end_line_no
        comment_text_before_section = comment_content[last_section_end + 1:section_start]
        last_section_end = section_end
        if len(comment_text_before_section) > 0:
            structs_in_comment.append(
                Struct("comment", comment_text_before_section, (0, start_line_no)))
        structs_in_comment.append(
            Struct("section", (section_title, section_level),
                   (start_line_no, end_line_no)))
    if last_section_end < len(comment_content):
        remaining_text = comment_content[last_section_end:].strip()
        if len(remaining_text) > 0:
            last_section_end_line_no = find_line_no(last_section_end, min_line_no, min_pos_for_line)
            structs_in_comment.append(Struct("comment", remaining_text,
                                         (last_section_end_line_no, len(min_pos_for_line)-1)))
    return structs_in_comment


from django.shortcuts import render

# Create your views here.
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.shortcuts import render
from .models import Product

def all_products(request): # pagination requesting 6 products only on a page
    products = Product.objects.all()

    page = request.GET.get('page', 1)

    paginator = Paginator(products, 6)

    try:

        products = paginator.page(page)

    except PageNotAnInteger:

        products = paginator.page(1)

    except EmptyPage:

        products = paginator.page(paginator.num_pages)
    return render(request, "products/products.html", {"products": products})


#! /usr/bin/env python
from __future__ import print_function
try:
    import httplib
except ImportError:
    import http.client as httplib

import sys
import os
import optparse
import time
import json

def change_priority(url, workflow, priority, cert, key, retry):
    data = {'RequestPriority': priority}
    headers = {'Content-type': 'application/json',
               'Accept': 'application/json'}

    for _ in range(retry):
        conn = httplib.HTTPSConnection(url, cert_file=cert, key_file=key)
        conn.request('PUT', '/reqmgr2/data/request/%s' % workflow, json.dumps(data), headers)
        response = conn.getresponse()
        status, res = response.status, response.read()
        conn.close()
        if status == 200:
            return json.loads(res).get('result', [])[0].get(workflow, '').lower() == 'ok'

        print('Status: %s, response: %s' % (status, res))
        time.sleep(1)

    return False

def main():
    parser = optparse.OptionParser()
    parser.add_option('-u', '--url',
                      help='Base url to send request to',
                      dest='url',
                      default='cmsweb.cern.ch')
    parser.add_option('-c', '--cert',
                      help='Cert file location',
                      dest='cert',
                      default=os.getenv('X509_USER_PROXY'))
    parser.add_option('-k', '--key',
                      help='Key file location',
                      dest='key',
                      default=os.getenv('X509_USER_PROXY'))
    parser.add_option('-r', '--retry',
                      help='Number of retries',
                      dest='retry',
                      default=1)
    options, args = parser.parse_args()
    if len(args) < 2:
        print('usage: wmpriority.py <workflowname> <priority> [options]')
        sys.exit(1)

    workflow = args[0]
    priority = int(args[1])
    res = change_priority(options.url, workflow, priority, options.cert, options.key, options.retry)
    print('%s: %s' % (workflow, res))

if __name__ == "__main__":
    main()


import os
import rospy

MANIFEST_FILE = 'manifest.xml'
PACKAGE_FILE = 'package.xml'

try:
    from catkin_pkg.package import parse_package
    CATKIN_SUPPORTED = True
except ImportError:
    CATKIN_SUPPORTED = False

PACKAGE_CACHE = {}

def utf8(s, errors='replace'):
    if isinstance(s, (str, buffer)):
        return unicode(s, "utf-8", errors=errors)
    elif not isinstance(s, unicode):
        return unicode(str(s))
    return s

def get_ros_home():
    '''
    Returns the ROS HOME depending on ROS distribution API.
    @return: ROS HOME path
    @rtype: C{str}
    '''
    try:
        import rospkg.distro
        distro = rospkg.distro.current_distro_codename()
        if distro in ['electric', 'diamondback', 'cturtle']:
            import roslib.rosenv
            return roslib.rosenv.get_ros_home()
        else:
            from rospkg import get_ros_home
            return get_ros_home()
    except:
        from roslib import rosenv
        return rosenv.get_ros_home()

def masteruri_from_ros():
    '''
    Returns the master URI depending on ROS distribution API.
    @return: ROS master URI
    @rtype: C{str}
    '''
    try:
        import rospkg.distro
        distro = rospkg.distro.current_distro_codename()
        if distro in ['electric', 'diamondback', 'cturtle']:
            import roslib.rosenv
            return roslib.rosenv.get_master_uri()
        else:
            import rosgraph
            return rosgraph.rosenv.get_master_uri()
    except:
        return os.environ['ROS_MASTER_URI']

def get_rosparam(param, masteruri):
    if masteruri:
        try:
            master = rospy.msproxy.MasterProxy(masteruri)
            return master[param]  # MasterProxy does all the magic for us
        except KeyError:
            return {}

def delete_rosparam(param, masteruri):
    if masteruri:
        try:
            master = rospy.msproxy.MasterProxy(masteruri)
            del master[param]  # MasterProxy does all the magic for us
        except Exception:
            pass

def get_packages(path):
    result = {}
    if os.path.isdir(path):
        fileList = os.listdir(path)
        if MANIFEST_FILE in fileList:
            return {os.path.basename(path): path}
        if CATKIN_SUPPORTED and PACKAGE_FILE in fileList:
            try:
                pkg = parse_package(path)
                return {pkg.name: path}
            except:
                pass
            return {}
        for f in fileList:
            ret = get_packages(os.path.join(path, f))
            result = dict(ret.items() + result.items())
    return result

def resolve_paths(text):
    '''
    Searches in text for $(find ...) statements and replaces it by the package path.
    @return: text with replaced statements.
    '''
    result = text
    startIndex = text.find('$(')
    if startIndex > -1:
        endIndex = text.find(')', startIndex + 2)
        script = text[startIndex + 2: endIndex].split()
        if len(script) == 2 and (script[0] == 'find'):
            pkg = ''
            try:
                from rospkg import RosPack
                rp = RosPack()
                pkg = rp.get_path(script[1])
            except:
                import roslib
                pkg = roslib.packages.get_pkg_dir(script[1])
            return result.replace(text[startIndex: endIndex + 1], pkg)
    return result

def to_url(path):
    '''
    Searches the package name for given path and create an URL starting with pkg://
    '''
    result = path
    pkg, pth = package_name(os.path.dirname(path))
    if pkg is not None:
        result = "pkg://%s%s" % (pkg, path.replace(pth, ''))
    return result

def package_name(path):
    '''
    Returns for given directory a tuple of package name and package path or None values.
    The results are cached!
    @rtype: C{(name, path)}
    '''
    if not (path is None) and path and path != os.path.sep and os.path.isdir(path):
        if path in PACKAGE_CACHE:
            return PACKAGE_CACHE[path]
        package = os.path.basename(path)
        fileList = os.listdir(path)
        for f in fileList:
            if f == MANIFEST_FILE:
                PACKAGE_CACHE[path] = (package, path)
                return (package, path)
            if CATKIN_SUPPORTED and f == PACKAGE_FILE:
                try:
                    pkg = parse_package(os.path.join(path, f))
                    PACKAGE_CACHE[path] = (pkg.name, path)
                    return (pkg.name, path)
                except:
                    return (None, None)
        PACKAGE_CACHE[path] = package_name(os.path.dirname(path))
        return PACKAGE_CACHE[path]
    return (None, None)

def is_package(file_list):
    return (MANIFEST_FILE in file_list or (CATKIN_SUPPORTED and PACKAGE_FILE in file_list))


""" Reconciliation module """
from typing import List
from model.payment import payment
from model.currency import CurrencyConverter
from web.api.payment_status import PaymentStatusAPI

class ReconciliationAPI():
    """ Reconciliation API """
    def __init__(self):
        self._out = {}
        self._company_dict = {}
        self._pay_stat_api = PaymentStatusAPI()
        self._curr = CurrencyConverter()

    def get_result(self, company_names: List[str]) -> dict:
        """ Returns payment status """
        payment.generate_high_time_recurrences()

        self._out = {"reconciliations": []}

        for company_name in company_names:
            self._company_dict = {"header": {"company": company_name,
                                             "inc_sum": 0,
                                             "out_sum": 0,
                                             "balance": 0},
                                  "incoming": [],
                                  "outgoing": []}

            self._process_open_payments(company_name)
            self._out["reconciliations"].append(self._company_dict)

        return self._out

    def _process_open_payments(self, company_name):
        open_payments = payment.get_open_payments_of_company(company_name)

        for open_payment in open_payments:
            open_payment_dict = self._pay_stat_api.get_result(open_payment.guid)
            open_amt = self._curr.convert_to_local_currency(
                open_payment_dict["summary"]["open_amount"],
                open_payment_dict["summary"]["currency"])

            if open_payment.direction == payment.DIRECTION_IN:
                self._company_dict["incoming"].append(open_payment_dict)
                self._company_dict["header"]["inc_sum"] += open_amt
                self._company_dict["header"]["balance"] += open_amt
            elif open_payment.direction == payment.DIRECTION_OUT:
                self._company_dict["outgoing"].append(open_payment_dict)
                self._company_dict["header"]["out_sum"] += open_amt
                self._company_dict["header"]["balance"] -= open_amt


import logging
import os
import sys
import dotenv
import openai
from tenacity.stop import stop_after_attempt
from tenacity.wait import wait_random_exponential

dotenv.load_dotenv()
openai.organization = os.getenv("OPENAI_ORGANIZATION", None)
openai.api_key = os.getenv("OPENAI_API_KEY")

logging.basicConfig(stream=sys.stderr, level=logging.INFO)
logger = logging.getLogger(__name__)

BASE_CACHE_DIR = ".cache"

def log_after_retry(logger, level):
    def log(retry_state):
        logger.log(
            level,
            "Retrying %s, attempt %s after exception %s",
            retry_state.fn,
            retry_state.attempt_number,
            retry_state.outcome.exception(),
        )

    return log

# kwargs used for exponential backoff
RETRY_KWARGS = dict(
    wait=wait_random_exponential(min=3, max=60),
    stop=stop_after_attempt(6),
    after=log_after_retry(logger, logging.INFO),
)

def get_base_model_name(model_name) -> str:
    if ":" in model_name:
        return model_name.split(":")[0]
    else:
        return model_name

def get_cost_per_token(model_name, training=False):
    # source: https://openai.com/pricing
    base_inference_price_dict_1k = {
        "ada": 0.0004,
        "babbage": 0.0005,
        "curie": 0.0020,
        "davinci": 0.02,
        "code-davinci-002": 0,
        "code-cushman-001": 0,
        "text-ada-001": 0.0004,
        "text-babbage-001": 0.0005,
        "text-curie-001": 0.0020,
        "text-davinci-001": 0.02,
        "text-davinci-002": 0.02,
        "text-davinci-003": 0.02,
        "gpt-3.5-turbo": 0.002,
        # They charge 2x that per output token, so this metric is a bit off
        "gpt-4": 0.03,
    }

    training_price_dict_1k = {
        "ada": 0.0004,
        "babbage": 0.0006,
        "curie": 0.0030,
        "davinci": 0.03,
    }

    ft_inference_price_dict_1k = {
        "ada": 0.0016,
        "babbage": 0.0024,
        "curie": 0.0120,
        "davinci": 0.12,
    }

    if training:
        price_1k = training_price_dict_1k.get(get_base_model_name(model_name), 0)
    elif ":" in model_name:
        price_1k =  ft_inference_price_dict_1k.get(get_base_model_name(model_name), 0)
    else:
        price_1k = base_inference_price_dict_1k.get(model_name, 0)

    return price_1k / 1000


from pid import PID
from lowpass import LowPassFilter
from yaw_controller import YawController
import rospy

GAS_DENSITY = 2.858
ONE_MPH = 0.44704

class Controller(object):
    def __init__(self, vehicle_mass, fuel_capacity, brake_deadband, decel_limit,
                 accel_limit, wheel_radius, wheel_base, steer_ratio, max_lat_accel,max_steer_angle):

        # set steering pid controller with minimum speed 0.1
        self.yaw_controller = YawController(wheel_base, steer_ratio, 0.1, max_lat_accel, max_steer_angle)

        # set throttling pid controller with min. 0, max. 0.2
        kp = 0.3
        ki = 0.1
        kd = 0.
        mn = 0.
        mx = 0.2
        self.throttle_controller = PID(kp, ki, kd, mn, mx)

        # set lowpath filter
        tau = 0.5
        ts  = .02
        self.vel_lpf = LowPassFilter(tau, ts)

        self.vehicle_mass    = vehicle_mass
        self.fuel_capacity   = fuel_capacity
        self.brake_deadband  = brake_deadband
        self.decel_limit     = decel_limit
        self.accel_limit     = accel_limit
        self.wheel_radius    = wheel_radius

        self.last_time = rospy.get_time()

    def control(self, current_vel, dbw_enabled, linear_vel, angular_vel):

        # if dbw_enabled is disable, pid controller.
        if not dbw_enabled:
            self.throttle_controller.reset()
            return 0., 0., 0.# Return throttle, brake, steer

        current_vel = self.vel_lpf.filt(current_vel)

        # get steering pid controller's output
        steering = self.yaw_controller.get_steering(linear_vel, angular_vel, current_vel)

        vel_error = linear_vel - current_vel
        self.last_vel = current_vel

        current_time = rospy.get_time()
        sample_time  = current_time - self.last_time
        self.last_time = current_time

        throttle = self.throttle_controller.step(vel_error, sample_time)
        brake    = 0

        if linear_vel == 0. and current_vel < 0.1:
            throttle = 0
            brake    = 700 #400

        # if car is faster than the goal, vel_error is neg.
        elif throttle < .1 and vel_error < 0:
            throttle = 0
            # decel_limit is -5
            decel = max(vel_error, self.decel_limit)
            # brake unit is in torque(N*m)
            brake = abs(decel)*self.vehicle_mass*self.wheel_radius

        return throttle, brake, steering


import io
import logging
import os
import shutil
from pathlib import Path
from typing import List, Tuple

import pdfminer.pdfdocument
import wikipedia
from pdfminer.converter import TextConverter
from pdfminer.layout import LAParams
from pdfminer.pdfinterp import PDFPageInterpreter
from pdfminer.pdfinterp import PDFResourceManager
from pdfminer.pdfpage import PDFPage

log = logging.getLogger(__name__)

def cleanup_data(root_dir: str, target_dir: str, extensions: List[str]):
    """ Copy all files in all subdirectories of root_dir, to target_dir that are in extension.

    :param root_dir: root directory
    :param target_dir: target directory
    :param extensions: extensions to not remove
    :return: bool determining success
    """
    # if target dir doesn't exist create
    Path(target_dir).mkdir(parents=True, exist_ok=True)

    # check subtree of root
    for root, directories, filenames in os.walk(root_dir, topdown=False):
        # iterate over files
        for filename in filenames:
            # copy files with correct extension to target directory
            filepath = os.path.join(root, filename).replace("\\", "/")
            try:
                ext = filename.split(".")[1]
                if ext in extensions:
                    shutil.copyfile(filepath, f"{target_dir}/{filename}")
            except IndexError:
                log.info(f"Couldn't get extension of file {filepath}. Skipping...")
            except FileNotFoundError:
                log.info(f"Couldn't find file {filepath}. Skipping...")

    return True

def pdf_to_string(path):
    """ Converts a pdf file to a string. String contains

    :param path: Path to pdf file
    :return: String
    """
    resource_manager = PDFResourceManager()
    fake_file_handle = io.StringIO()
    converter = TextConverter(resource_manager, fake_file_handle, laparams=LAParams())
    page_interpreter = PDFPageInterpreter(resource_manager, converter)

    with open(path, 'rb') as fh:
        for page in PDFPage.get_pages(fh,
                                      caching=True,
                                      check_extractable=True):
            page_interpreter.process_page(page)

        text = fake_file_handle.getvalue()

    # close open handles
    converter.close()
    fake_file_handle.close()

    return text

def replace_cid_codes(string):
    """ Takes a string and replaces relevant cid codes

    :param string: string with cid codes
    :return: string with relevant cid codes replaced
    """
    # letters
    string = string.replace('(cid:228)', 'ä')
    string = string.replace('(cid:246)', 'ö')
    string = string.replace('(cid:252)', 'ü')

    string = string.replace('(cid:214)', 'Ö')
    string = string.replace('(cid:220)', 'Ü')
    string = string.replace('(cid:223)', 'ß')

    string = string.replace('\n', ' ')
    string = string.replace('\r', '')

    return string

def convert_pdfs_to_text(pdf_root_dir: str, _) -> List[str]:
    """ Convert all pdf files to a list of texts.

    :param pdf_root_dir: root directory containing pdf files
    :param _: placeholder to ensure correct pipeline execution order
    :return: Dict with filenames as keys and data as value
    """
    dataset = []

    # iterate over files in pdf_root_dir
    for root, directories, filenames in os.walk(pdf_root_dir, topdown=False):
        # convert and preprocess text file
        for filename in filenames:
            filepath = os.path.join(root, filename)
            try:
                text = pdf_to_string(filepath)
                processed_text = replace_cid_codes(text)

                # add to dataset
                dataset.append(processed_text)
                log.info(f"Done with processing file {filename}!")
            except FileNotFoundError:
                log.info(f"Couldn't find file {filepath}. Skipping...")
            except pdfminer.pdfdocument.PDFTextExtractionNotAllowed:
                log.info(f"Couldn't extract text from {filepath}. Skipping...")

    return dataset

def add_element_to_dict(dictionary, element):
    """ Adds an element to a dictionary. If not in dictionary, adds a new key to dictionary.

    :param dictionary: dictionary
    :param element: string
    :return: updated dictionary
    """

    if len(element) == 1:
        return dictionary

    if element not in dictionary.keys():
        dictionary[element] = 1
    else:
        dictionary[element] += 1

    return dictionary

def get_tokens(spacy_model, text):
    """ Extracts tokens from a text and returns a dictionary with the tokens and the number of appearances and
    a dictionary with the lemmas of the tokens.

    :param spacy_model: spacy model
    :param text: Text to extract the tokens from
    :return: word_dict, lemma_dict
    """
    word_dict = {}
    lemma_dict = {}

    doc = spacy_model(text)
    for token in doc:
        if token.is_alpha:
            add_element_to_dict(word_dict, token.text)
            add_element_to_dict(lemma_dict, token.lemma_)

    # sort dictionaries descending by appearance of tokens
    word_dict = {k: v for k, v in sorted(word_dict.items(), key=lambda item: item[1], reverse=True)}
    lemma_dict = {k: v for k, v in sorted(lemma_dict.items(), key=lambda item: item[1], reverse=True)}

    return word_dict, lemma_dict

def count_token_appearances(texts: List[str], spacy_model) -> Tuple[List, List]:
    """ Convert each text into a Dict containing the tokens and their number of appearances (same with lemmas).

    :param texts: list of texts
    :param spacy_model: spacy model
    :return: dict with counts for tokens, dict with counts for lemmas of tokens
    """
    token_count = []
    lemma_count = []

    # iterate over texts
    for text in texts:
        tokens, lemmas = get_tokens(spacy_model, text)
        token_count.append(tokens)
        lemma_count.append(lemmas)

    return token_count, lemma_count

def get_wikipedia_articles(num_pages: int, language: str) -> List[str]:
    """ Get a list of the content of random wikipedia articles.

    :param num_pages: number of wikipedia pages to load
    :param language: language of articles
    :return: list of article content (texts)
    """
    wikipedia.set_lang(language)

    articles = []

    # find valid random articles
    counter = 0
    while counter < num_pages:
        try:
            articles.append(wikipedia.page(title=wikipedia.random()).content)
            counter += 1
            log.info(f"{counter}/{num_pages}: Downloading {articles[-1].title}")
        except wikipedia.DisambiguationError:
            # skip if DisambiguationError appears and decrement counter
            log.info('DisambiguationError! Skip...')
            continue
        except wikipedia.PageError:
            # skip if PageError appears and decrement counter
            log.info('PageError! Skip...')
            continue

    return articles


from __future__ import print_function

def getKey(data, key, default="N/A"):
   try:
      for member in key.split('.'):
         member = int(member) if member.isdigit() else member
         data = data[member]
   except Exception: # pylint: disable=broad-except
      data = default
   return data

def tupleFmt(data, fmt, *args):
   return fmt % tuple(getKey(data, k) for k in args)

class Row():
   def __init__(self, fmt, *args):
      self.fmt = fmt
      self.args = args

   def render(self, data, indent=0):
      spacer = ' ' * indent
      print('%s%s' % (spacer, tupleFmt(data, self.fmt, *self.args)))

class List():
   def __init__(self, title=None, attr=None, header=None, tree=None):
      self.title = title
      self.attr = attr
      self.header = header
      self.tree = tree or []

   def renderItem(self, item, indent, spacing):
      spacer = ' ' * indent
      print('%s%s' % (spacer, tupleFmt(item, *self.header)))
      indent += spacing

      spacer = ' ' * indent
      for row in self.tree:
         if isinstance(row, List):
            row.render(item, indent) # XXX
         else:
            row.render(item, indent=indent)

   def render(self, data, indent=0, spacing=2, newline=False):
      if self.title:
         print('%s%s' % (' ' * indent, self.title))
         indent += spacing

      if self.attr is not None:
         data = getKey(data, self.attr)

      for item in data:
         self.renderItem(item, indent, spacing)

      if data and newline:
          print()

class Col():
   def __init__(self, name, attr, size):
      self.name = name
      self.attr = attr
      self.size = size

class Table():
   def __init__(self, columns):
      self.columns = columns

   def render(self, data, newline=False):
      if not data:
         return

      fmt = ' '.join('%%-%ds' % c.size for c in self.columns)
      print(fmt % tuple(c.name for c in self.columns))
      print(fmt % tuple('-' * c.size for c in self.columns))
      for item in data:
         print(fmt % tuple(getKey(item, c.attr) for c in self.columns))

      if newline:
         print()

class Renderer():

   NAME = None

   def __init__(self, name=None):
      self.name = name or self.NAME
      self.data_ = {}

   def data(self, show):
      data = self.data_.get(show)
      if data is None:
         data = self.getData(show)
         self.data_[show] = data
      return data

   def getData(self, show):
      '''Output for JSON, recommended to use as source for renderText'''
      raise NotImplementedError

   def renderText(self, show):
      '''Textual output for CLI'''
      raise NotImplementedError

class Show():

   TXT = 'text'
   JSON = 'json'

   def __init__(self, outputFormat=None, args=None):
      self.outputFormat = outputFormat
      self.inventories = []
      self.platforms = []
      self.args = args

   def addInventory(self, inventory, **metadata):
      self.inventories.append((inventory, metadata))

   def addPlatform(self, platform):
      self.platforms.append(platform)

   def renderText(self, *renderers):
      for r in renderers:
         r.renderText(self)

   def renderJson(self, *renderers):
      data = {
         "version": 1,
         "renderers": {
            r.name : r.data(self) for r in renderers
         },
      }

      import json
      if self.args.pretty:
         print(json.dumps(data, indent=3, separators=(',', ': ')))
      else:
         print(json.dumps(data))

   def render(self, *renderers):
      if self.outputFormat == self.TXT:
         self.renderText(*renderers)
      elif self.outputFormat == self.JSON:
         self.renderJson(*renderers)


"""DataPlaneAPIOpsManager Class"""
import re
import sys
import subprocess
from urllib import request
import zipfile
import shutil
from pathlib import Path
import requests

class LokiManager:
    """
    Mangages the Loki service, such as installing, configuring, etc.

    This class should work independently from juju, such as that it can
    be tested without lauching a full juju environment.
    """

    def __init__(self):
        self.loki_home = Path('/opt/loki')
        self.loki = Path('/opt/loki/loki-linux-amd64')
        self.loki_cfg = self.loki_home.joinpath('loki-local-config.yaml')
        self.loki_unitfile = Path('/etc/systemd/system/loki.service')

    def _prepareOS(self):
        """ sudo mkdir /opt/loki """
        try:
            subprocess.run(['mkdir', '-p', self.loki_home], check = True)
            print(f"Prepared OS for loki installation {self.loki_home}")
        except:
            print(f"Error preparing OS for loki installation {self.loki_home}")

    def _install_from_resource(self, resource_path):
        """
        Install from a resource.
        """
        # Remove the loki home dir if exists
        if self.loki_home.exists():
            shutil.rmtree(self.loki_home)

        # Unzip the juju resource into the build dir
        with zipfile.ZipFile(resource_path,"r") as zip_ref:
            zip_ref.extractall(self.loki_home)

        try:
            subprocess.run(['chmod','a+x',self.loki], check = True)
        except:
            print("Error installing loki binary")
            sys.exit(1)

    def _install_config(self):
        """
        Install the config from template.
        """
        if self.loki_cfg.exists():
            self.loki_cfg.unlink()
        lokiconfig_tmpl = Path('templates/loki-local-config.yaml.tmpl').read_text()
        self.loki_cfg.write_text(lokiconfig_tmpl)

    def _install_systemd_unitfile(self):
        """ Install the systemd unit file."""
        if self.loki_unitfile.exists():
            self.loki_unitfile.unlink()
        systemdunitfile_tmpl = Path('templates/loki.service.tmpl').read_text()
        self.loki_unitfile.write_text(systemdunitfile_tmpl)

    def stop_loki(self):
        """Stop loki"""
        try:
            subprocess.run(['systemctl','stop','loki'], check = True)
        except Exception as e:
            print("Error stopping loki", str(e))

    def start_loki(self):
        """Start loki"""
        try:
            subprocess.run(['systemctl','start','loki'], check = True)
        except Exception as e:
            print("Error starting loki", str(e))

    def restart_loki(self):
        """Restart loki"""
        try:
            subprocess.run(['systemctl','restart','loki'], check = True)
        except Exception as e:
            print("Error starting loki", str(e))

    def install(self, resource_file):
        """ Installs from a supplied zip file resource """
        self._prepareOS()
        self._install_from_resource(resource_file)
        self._install_config()
        self._install_systemd_unitfile()

    def loki_version(self):
        """ Return the version of loki as a string or None"""
        try:
            r = subprocess.run(
                [
                    self.loki.resolve(),
                    '-config.file', self.loki_cfg.resolve(),
                    '-version'
                ],capture_output=True
            ).stdout.decode()
            ver = re.search(r'version\s*([\d.]+)', r).group(1)
            return ver
        except Exception as e:
            print("Error getting version from loki", e)
            return None

    def verify_config(self, filename=None):
        """ Use loki to verify a loki config. E.g. look for msg="config is valid" """
        if filename:
            filetocheck = Path(filename)
        else:
            filetocheck = self.loki_cfg
        try:
            r = subprocess.run(
                [
                    self.loki.resolve(),
                    '-config.file', filetocheck.resolve(),
                    '-verify-config'
                ],capture_output=True
            )
            s = r.stderr.decode()
            return re.search(r'config is valid', s)

        except Exception as e:
            print("Error verifying config", e)
            return None

    def is_ready(self):
        """
        Checks the status of loki service by calling the api on localhost.
        Manually: curl -G -s http://localhost:3100/ready
        """
        url = "http://localhost:3100/ready"
        r = requests.get(url, timeout=2.50)
        return r.text.strip() == "ready"

    def _purge(self):
        """ Whipes the installation and remove all traces of Loki """
        try:
            subprocess.run(['rm', self.loki], check = True)
            print("Success removing loki bin", self.loki)
            subprocess.run(['rm', self.loki_unitfile], check = True)
            print("Success removing loki unitfile", self.loki_unitfile)
            subprocess.run(['rm', '-rf', self.loki_home], check = True)
            print("Success purging loki home dir", self.loki_home)
        except:
            print("Error purging loki")


import json
from time import time
import copy

from django.http import HttpResponse, HttpRequest

from configuration import Configuration
from utilities.similar_words_dictionary_parser import SimilarWordsDictionaryParser
from utilities.stats_calculator import StatsCalculator

number_of_words_in_dictionary, similar_words_dictionary = \
    SimilarWordsDictionaryParser.parse(Configuration.get_path_to_dictionary_file())

def similar(request: HttpRequest) -> HttpResponse:
    begin = time()
    if request.method == 'GET':
        if 'word' in request.GET:
            # Fetched received parameter and format is
            query_word = request.GET['word']
            clean_query_word = query_word.rstrip('\n').rstrip('\r').strip().lower()

            # Sorted word will be used as the information key
            sorted_word = ''.join(sorted(clean_query_word))

            ret_dict = {'similar': []}  # Default response in case there aren't any similar words in the dictionary
            if sorted_word in similar_words_dictionary:
                ret_dict = similar_words_dictionary[sorted_word]

                # Omit the queried word if present
                if clean_query_word in ret_dict['similar']:
                    ret_dict = copy.deepcopy(ret_dict)
                    ret_dict['similar'].remove(clean_query_word)

            json_string = json.dumps(ret_dict)
            total_time = time() - begin
            StatsCalculator.add_request_stat(total_time)
            return HttpResponse(json_string)

    # Default response
    return HttpResponse('')

def stats(request: HttpRequest) -> HttpResponse:
    if request.method == 'GET':
        # Fetch Statistics
        stats_dict = StatsCalculator.get_stats()

        # Add to the statistics the totalWords information
        stats_dict['totalWords'] = number_of_words_in_dictionary
        json_string = json.dumps(stats_dict)
        return HttpResponse(json_string)

    # Default response
    return HttpResponse('')


from django.urls import path
from .views import (
    UserListAndCreate,
    UserDelete,
    UserUpdate,
    ArticleList,
    ArticleCreate,
    ArticleUpdate,
    ArticleDelete,
    # RevokeToken
)

app_name = 'api'
urlpatterns = [
    path('user-list', UserListAndCreate.as_view(), name='user_list'),
    path('user-update/<int:pk>', UserUpdate.as_view(), name='user_update'),
    path('user-delete/<int:pk>', UserDelete.as_view(), name='user_delete'),
    path('article-list', ArticleList.as_view(), name='article_list'),
    path('article-create', ArticleCreate.as_view(), name='article_create'),
    path('article-update/<int:pk>', ArticleUpdate.as_view(), name='article_update'),
    path('article-delete/<int:pk>', ArticleDelete.as_view(), name='article_delete'),
    # path('revoke-token', RevokeToken.as_view(), name='revoke_token')
]


import cv2
import numpy as np

def absSobelThreshFilter(image, orient='x', sobelKernel=6, thresh=(0, 255)):
    # Use lightness channel of HLS color space for better performance over grayscale
    hls = cv2.cvtColor(image, cv2.COLOR_RGB2HLS)
    l_channel = hls[:, :, 1]
    # Take the derivative in x or y given orient = 'x' or 'y'
    if orient == 'x':
        sobel = cv2.Sobel(l_channel, cv2.CV_64F, 1, 0, ksize=sobelKernel)
    else:
        sobel = cv2.Sobel(l_channel, cv2.CV_64F, 0, 1)
    # Get absolute value, direction is of no importance
    absSobel = np.absolute(sobel)
    # Scale to max value and apply threshold filter
    binaryImg = scaleAndApplyThresholdsOnAbsoluteSobel(absSobel, thresh)
    return binaryImg

def magSobelThreshFilter(image, sobelKernel=6, magThresh=(0, 255)):
    # Use lightness channel of HLS color space for better performance over grayscale
    hls = cv2.cvtColor(image, cv2.COLOR_RGB2HLS)
    lChannel = hls[:, :, 1]
    # Apply Sobel in both directions
    sobelX = cv2.Sobel(lChannel, cv2.CV_64F, 1, 0, ksize=sobelKernel)
    sobelY = cv2.Sobel(lChannel, cv2.CV_64F, 0, 1, ksize=sobelKernel)
    # Get absolute sum out of both, we are targeting for the magnitude,
    # no filtering for either direction
    absSobel = np.sqrt(np.square(sobelX) + np.square(sobelY))
    # Scale to max value and apply threshold filter
    binaryImg = scaleAndApplyThresholdsOnAbsoluteSobel(absSobel, magThresh)

    return binaryImg

def scaleAndApplyThresholdsOnAbsoluteSobel(absoluteSobel, thresh=(0, 255)):
    # Scale to max found value and convert to resolution 8bit
    scaledSobel = np.uint8(255 * absoluteSobel / np.max(absoluteSobel))
    # Convert to binary image
    binaryImg = convertToBinaryFilter(scaledSobel, thresh)

    return binaryImg

def convertToBinaryFilter(img, thresh=(0, 255)):
    # Prepare empty binary image
    binaryImg = np.zeros_like(img)
    # Only keep those pixels that are between the threshold
    binaryImg[(img >= thresh[0]) & (img <= thresh[1])] = 1
    return binaryImg

def sChannelThreshFilter(img, thresh=(0,255)):
    hls = cv2.cvtColor(img, cv2.COLOR_RGB2HLS)
    sChannel = hls[:, :, 2]
    # Convert to binary image
    binaryImg = convertToBinaryFilter(sChannel, thresh)
    return binaryImg

def mergeWithAnd(binImg1 , binImg2):
    mergedBinImg = np.zeros_like(binImg1)
    mergedBinImg[((binImg1 == 1)) & ((binImg2 == 1))] = 1
    return mergedBinImg

def mergeWithOr(binImg1, binImg2):
    mergedBinImg = np.zeros_like(binImg1)
    mergedBinImg[((binImg1 == 1)) | ((binImg2 == 1))] = 1
    return mergedBinImg

def blurFilter(img, blurKernel=5):
    return cv2.blur(img, (blurKernel,blurKernel))

if __name__ == '__main__':
    import misc

    testImages = misc.getFileRelativeFilepathsInDirectory('test_images')
    for imageName in testImages:
        img = cv2.imread(imageName)
        # Setup hyper parameters
        blurKernelSize = 5
        sobelKernelSize = 5
        absSobelXThresh = (15, 200)
        magSobelThresh = (38, 124)
        sChannelThresh = (103, 200)
        # Blur image
        blurImg = blurFilter(img, blurKernelSize)
        # Get gradient image via combination of x-dir abs. sobel and magnitude filter
        xGradImg = absSobelThreshFilter(blurImg,'x',sobelKernelSize, absSobelXThresh)
        magGradImg = magSobelThreshFilter(blurImg, sobelKernelSize, magSobelThresh)
        gradImg = mergeWithAnd(xGradImg, magGradImg)
        # Filter via s-channel filter esp. for yellow this is important
        sChannelImg = sChannelThreshFilter(blurImg, sChannelThresh)
        # Merge gradient and s-channel images via or to get best out of both worlds
        finalImg = mergeWithOr(gradImg, sChannelImg)
        # Show before after
        #misc.showBeforeAfter(cv2.cvtColor(img, cv2.COLOR_BGR2RGB), "Original", cv2.cvtColor(finalImg*255,cv2.COLOR_BGR2RGB), "Gradient and color threshold filter")
        #misc.showImageInNewFigure(cv2.cvtColor(finalImg*255,cv2.COLOR_BGR2RGB), "Gradient and color threshold filter")
        cv2.imwrite('debugOutput/' + misc.getFilenameFromPath(imageName), cv2.cvtColor(finalImg*255,cv2.COLOR_BGR2RGB))