#!/usr/bin/env python3

import argparse
import asyncio
import atexit
from dynaconf import Dynaconf, Validator
from dynaconf.base import Settings
from dynaconf.utils.boxing import DynaBox
import json
import logging
from .message_mapping import map_bundle_to_sightings
import signal
from stix2 import parse, Indicator, Sighting
from stix_shifter.stix_translation import stix_translation
from stix_shifter.stix_transmission import stix_transmission
import sys
from threatbus.logger import setup as setup_logging_threatbus
import warnings
import zmq

# Ignore warnings about SSL configuration in the user configs.
warnings.filterwarnings("ignore")

logger_name = "stix-shifter-threatbus"
logger = logging.getLogger(logger_name)
# List of all running async tasks of the bridge.
async_tasks = []
# The p2p topic sent back by Threat Bus upon successful subscription.
p2p_topic = None
# Boolean flag indicating that the user has issued a SIGNAL (e.g., SIGTERM).
user_exit = False

### --------------------------- Application helpers ---------------------------

def setup_logging_with_config(config: DynaBox):
    """
    Sets up the global logger as configured in the `config` object.
    @param config The user-defined logging configuration
    """
    global logger
    logger = setup_logging_threatbus(config, logger_name)
    logging.getLogger("stix-shifter-utils").propagate = False

def validate_config(config: Settings):
    """
    Validates the given Dynaconf object. Throws if the config is invalid.
    """
    validators = [
        Validator("logging.console", is_type_of=bool, required=True, eq=True)
        | Validator("logging.file", is_type_of=bool, required=True, eq=True),
        Validator(
            "logging.console_verbosity",
            is_in=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
            required=True,
            when=Validator("logging.console", eq=True),
        ),
        Validator(
            "logging.file_verbosity",
            is_in=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
            required=True,
            when=Validator("logging.file", eq=True),
        ),
        Validator(
            "logging.filename", required=True, when=Validator("logging.file", eq=True)
        ),
        Validator("threatbus", required=True),
        Validator("snapshot", is_type_of=int, required=True),
        Validator("modules", is_type_of=dict, required=True),
    ]

    if "modules" in config:
        for mod in config.modules.keys():
            validators.append(
                Validator(f"modules.{mod}.max_results", is_type_of=int, required=True)
            )
            validators.append(
                Validator(f"modules.{mod}.connection", is_type_of=dict, required=True)
            )
            validators.append(
                Validator(f"modules.{mod}.data_source", is_type_of=dict, required=True)
            )
            validators.append(
                Validator(f"modules.{mod}.data_source.type", required=True)
            )
            validators.append(
                Validator(f"modules.{mod}.data_source.name", required=True)
            )
            validators.append(Validator(f"modules.{mod}.data_source.id", required=True))
            validators.append(
                Validator(f"modules.{mod}.transmission", is_type_of=dict, default={})
            )  # default to empty config
            validators.append(
                Validator(f"modules.{mod}.translation", is_type_of=dict, default={})
            )  # default to empty config

    config.validators.register(*validators)
    config.validators.validate()

async def cancel_async_tasks():
    """
    Cancels all async tasks.
    """
    global async_tasks
    for task in async_tasks:
        if task is not asyncio.current_task():
            task.cancel()
            del task
    async_tasks = []
    return await asyncio.gather(*async_tasks)

async def stop_signal():
    """
    Implements Python's asyncio eventloop signal handler
    https://docs.python.org/3/library/asyncio-eventloop.html
    Cancels all running tasks and exits the app.
    """
    global user_exit
    user_exit = True
    await cancel_async_tasks()

### --------------- ZeroMQ communication / management functions ---------------

def send_manage_message(endpoint: str, action: dict, timeout: int = 5):
    """
    Sends a 'management' message, following the threatbus-zmq protocol to
    either subscribe or unsubscribe this application to/from Threat Bus.
    @param endpoint A host:port string to connect to via ZeroMQ
    @param action The message to send as JSON
    @param timeout The period after which the connection attempt is aborted
    """
    context = zmq.Context()
    socket = context.socket(zmq.REQ)
    socket.setsockopt(zmq.LINGER, 0)
    socket.connect(f"tcp://{endpoint}")
    socket.send_json(action)
    poller = zmq.Poller()
    poller.register(socket, zmq.POLLIN)

    reply = None
    if poller.poll(timeout * 1000):
        reply = socket.recv_json()
    socket.close()
    context.term()
    return reply

def reply_is_success(reply: dict):
    """
    Predicate to check if `reply` is a dict and contains the key-value pair
    "status" = "success"
    @param reply A python dict
    @return True if the dict contains "status" = "success"
    """
    return (
        reply
        and type(reply) is dict
        and reply.get("status", None)
        and reply["status"] == "success"
    )

def subscribe(endpoint: str, topic: str, snapshot: int, timeout: int = 5):
    """
    Subscribes this app to Threat Bus for the given topic. Requests an optional
    snapshot of historical indicators.
    @param endpoint The ZMQ management endpoint of Threat Bus ('host:port')
    @param topic The topic to subscribe to
    @param snapshot An integer value to request n days of historical IoC items
    @param timeout The period after which the connection attempt is aborted
    """
    global logger
    logger.info(f"Subscribing to topic '{topic}'...")
    action = {"action": "subscribe", "topic": topic, "snapshot": snapshot}
    return send_manage_message(endpoint, action, timeout)

def unsubscribe(endpoint: str, topic: str, timeout: int = 5):
    """
    Unsubscribes this app from Threat Bus for the given topic.
    @param endpoint The ZMQ management endpoint of Threat Bus
    @param topic The topic to unsubscribe from
    @param timeout The period after which the connection attempt is aborted
    """
    global logger
    logger.info(f"Unsubscribing from topic '{topic}' ...")
    action = {"action": "unsubscribe", "topic": topic}
    reply = send_manage_message(endpoint, action, timeout)
    if not reply_is_success(reply):
        logger.warning("Unsubscription failed")
        return
    logger.info("Unsubscription successful")

async def heartbeat(endpoint: str, p2p_topic: str, interval: int = 5):
    """
    Sends heartbeats to Threat Bus periodically to check if the given p2p_topic
    is still valid at the Threat Bus host. Cancels all async tasks of this app
    when the heartbeat fails and stops the heartbeat.
    @param endpoint The ZMQ management endpoint of Threat Bus
    @param p2p_topic The topic string to include in the heartbeat
    @param timeout The period after which the connection attempt is aborted
    """
    global logger
    action = {"action": "heartbeat", "topic": p2p_topic}
    while True:
        reply = send_manage_message(endpoint, action, interval)
        if not reply_is_success(reply):
            logger.error("Subscription with Threat Bus host became invalid")
            return await cancel_async_tasks()
        await asyncio.sleep(interval)

### --------------------------- The actual app logic ---------------------------

async def start(zmq_endpoint: str, snapshot: int, modules_config: dict):
    """
    Starts the STIX-Shifter Threat Bus app.
    @param zmq_endpoint The ZMQ management endpoint of Threat Bus ('host:port')
    @param snapshot An integer value to request n days of historical IoC items
    @param modules_config User-provided configuration for STIX-Shifter modules
    """
    global logger, async_tasks, p2p_topic
    # needs to be created inside the same eventloop where it is used
    logger.debug(f"Calling Threat Bus management endpoint {zmq_endpoint}")
    reply = subscribe(zmq_endpoint, "stix2/indicator", snapshot)
    if not reply_is_success(reply):
        logger.error("Subscription failed")
        return
    pub_port = reply.get("pub_port", None)
    sub_port = reply.get("sub_port", None)
    topic = reply.get("topic", None)
    if not pub_port or not sub_port or not topic:
        logger.error("Subscription failed")
        return
    zmq_host = zmq_endpoint.split(":")[0]
    pub_endpoint = f"{zmq_host}:{pub_port}"
    sub_endpoint = f"{zmq_host}:{sub_port}"

    logger.info(f"Subscription successful. New p2p_topic: {topic}")
    if p2p_topic:
        # The 'start' function is called as result of a restart
        # Unsubscribe the old topic as soon as we get a working connection
        logger.info("Cleaning up old p2p_topic subscription ...")
        unsubscribe(zmq_endpoint, p2p_topic)
        atexit.unregister(unsubscribe)
    p2p_topic = topic
    atexit.register(unsubscribe, zmq_endpoint, topic)

    # Set task exception handler.
    loop = asyncio.get_event_loop()

    def exception_handler(loop, context):
        logger.error(f"Error in async task: {context}")

    loop.set_exception_handler(exception_handler)

    # Start a heartbeat task so we notice when the Threat Bus host goes away.
    async_tasks.append(
        asyncio.create_task(heartbeat(zmq_endpoint, p2p_topic, interval=5))
    )

    # Create queues (channels) for passing indicators and sightings
    # asynchronously between the routines that make up this app.
    indicator_queue = asyncio.Queue()
    sightings_queue = asyncio.Queue()

    # Start a receive task to retrieve real-time updates from Threat Bus.
    async_tasks.append(
        asyncio.create_task(receive(pub_endpoint, p2p_topic, indicator_queue))
    )
    async_tasks.append(
        asyncio.create_task(
            process_indicators(indicator_queue, sightings_queue, modules_config)
        )
    )

    # Start a publisher task to send back sightings to Threat Bus.
    async_tasks.append(
        asyncio.create_task(report_sightings(sub_endpoint, sightings_queue))
    )

    loop = asyncio.get_event_loop()
    for s in [signal.SIGHUP, signal.SIGTERM, signal.SIGINT]:
        loop.add_signal_handler(s, lambda: asyncio.create_task(stop_signal()))
    return await asyncio.gather(*async_tasks)

async def receive(pub_endpoint: str, topic: str, indicator_queue: asyncio.Queue):
    """
    Starts a zmq subscriber on the given endpoint and listens for new messages
    that are published on the given topic (zmq prefix matching). Depending on
    the topic suffix, Indicators are enqueued to the indicator_queue.
    @param pub_endpoint A host:port string to connect to via zmq
    @param topic The topic prefix to subscribe to intelligence items
    @param indicator_queue The queue to put arriving IoCs into
    """
    global logger
    socket = zmq.Context().socket(zmq.SUB)
    socket.connect(f"tcp://{pub_endpoint}")
    socket.setsockopt(zmq.SUBSCRIBE, topic.encode())
    poller = zmq.Poller()
    poller.register(socket, zmq.POLLIN)
    logger.info(f"Receiving via ZMQ on topic {pub_endpoint}/{topic}")
    while True:
        socks = dict(poller.poll(timeout=100))  # Smaller timeouts increase CPU load
        if socket in socks and socks[socket] == zmq.POLLIN:
            try:
                topic, msg = socket.recv().decode().split(" ", 1)
            except Exception as e:
                logger.error(f"Error decoding message: {e}")
                continue
            # The topic is suffixed with the message type. Use it for filtering
            if not topic.endswith("indicator"):
                logger.debug(f"Skipping unsupported message: {msg}")
                continue
            # Put the message into the queue for incoming intel items, so they
            # can be processed asynchronously
            await indicator_queue.put(msg)
        else:
            await asyncio.sleep(0.01)  # Free event loop for other tasks

async def process_indicators(
    indicator_queue: asyncio.Queue, sightings_queue: asyncio.Queue, modules_config: dict
):
    """
    Translates STIX-2 pattern and queries all configured modules via
    STIX-Shifter.
    @param indicator_queue The queue to read indicators from
    @param sightings_queue The queue to put sightings into
    @param modules_config User-provided configuration for STIX-Shifter modules
    """
    while True:
        msg = await indicator_queue.get()
        try:
            indicator = parse(msg, allow_custom=True)
        except Exception as e:
            logger.error(
                f"Error parsing indicator from Threat Bus. Expected STIX-2 Indicator: {msg}, {e}"
            )
            indicator_queue.task_done()
            continue
        logger.debug(
            f"Converting indicator from Threat Bus to module-specific query: {indicator}"
        )
        for module, opts in modules_config.items():
            asyncio.create_task(
                query_indicator(indicator, module, opts, sightings_queue)
            )
        indicator_queue.task_done()

async def query_indicator(
    indicator: Indicator, module: str, opts: dict, sightings_queue: asyncio.Queue
):
    """
    Translates an indicator into a module-specific query and executes it. E.g.,
    if the module is `splunk`, the indicator's pattern is first translated into
    a valid Splunk query and then executed via the Splunk REST API.
    @param indicator The indicator to translate and query
    @param module The module's name, e.g., `splunk`
    @param opts The module configuration directly taken from the user-defined
        configuration file `config.yaml` with which this app was started
    @param sightings_queue The queue to put sightings into
    """
    max_results = opts["max_results"]
    connection_opts = opts["connection"]
    transmission_opts = opts.get("transmission", {})
    translation_opts = opts.get("translation", {})
    data_source = opts["data_source"]

    ## Translate the pattern to a module-specific query.
    translation = stix_translation.StixTranslation()
    dsl = translation.translate(
        module, "query", indicator, indicator.pattern, translation_opts
    )
    if not dsl.get("queries", None):
        logger.error(
            f"Failed to translate STIX-2 indicator with ID '{indicator.id}' to query for module '{module}': {dsl}"
        )
        return
    logger.debug(f"Translated pattern to {module} query: {dsl}")

    ## Run the query against the configured endpoint for this module.
    transmission = stix_transmission.StixTransmission(
        module, connection_opts, transmission_opts
    )
    query_results = []
    for query in dsl["queries"]:
        search_result = transmission.query(query)
        if not search_result["success"]:
            logger.error(str(search_result))
            continue

        search_id = search_result["search_id"]

        if transmission.is_async():
            status = transmission.status(search_id)
            if not status.get("success", None):
                logger.error(f"Fetching query status failed for module '{module}'")
                return
            while status["progress"] < 100 and status["status"] == "RUNNING":
                status = transmission.status(search_id)
                await asyncio.sleep(0.05)
        result = transmission.results(search_id, 0, max_results)
        if result["success"]:
            # Collect all results
            query_results += result["data"]
        else:
            logger.error(f"Fetching results failed for module '{module}': {result}")

    ## Translate query_results to STIX.
    if not query_results:
        return

    stix_results = translation.translate(
        module,
        "results",
        json.dumps(data_source),
        json.dumps(query_results),
        translation_opts,
    )
    ## Parse output and report back sightings to Threat Bus
    ## The stix_results is always a self-made bundle with at least an `objects`
    ## field present. The bundle may be invalid STIX though, so we cannot simply
    ## invoke `parse()`. See this link for details on the bundle stucture:
    ## https://github.com/opencybersecurityalliance/stix-shifter/blob/3.4.5/stix_shifter_utils/stix_translation/src/json_to_stix/json_to_stix_translator.py#L12
    objs = stix_results.get("objects", None)
    if objs is None:
        logger.error(
            f"Received STIX bundle without `objects` field, cannot generate sightings: {stix_results}"
        )
        return
    for sighting in map_bundle_to_sightings(indicator, objs):
        await sightings_queue.put(sighting)

async def report_sightings(sub_endpoint: str, sightings_queue: asyncio.Queue):
    """
    Starts a ZeroMQ publisher on the given endpoint and publishes sightings from
    the sightings_queue to Threat Bus.
    @param sub_endpoint A host:port string to connect to via ZeroMQ
    @param sightings_queue The queue to receive sightings from
    """
    socket = zmq.Context().socket(zmq.PUB)
    socket.connect(f"tcp://{sub_endpoint}")
    topic = "stix2/sighting"
    logger.info(f"Forwarding sightings to Threat Bus at {sub_endpoint}/{topic}")
    while True:
        sighting = await sightings_queue.get()
        if type(sighting) is not Sighting:
            logger.warning(
                f"Ignoring unknown message type, expected Sighting: {type(sighting)}"
            )
            continue
        socket.send_string(f"{topic} {sighting.serialize()}")
        sightings_queue.task_done()
        logger.debug(f"Reported sighting: {sighting}")

def main():
    ## Default list of settings files for Dynaconf to parse.
    settings_files = ["config.yaml", "config.yml"]
    parser = argparse.ArgumentParser()
    parser.add_argument("--config", "-c", help="path to a configuration file")
    args = parser.parse_args()
    if args.config:
        if not args.config.endswith("yaml") and not args.config.endswith("yml"):
            sys.exit("Please provide a `yaml` or `yml` configuration file.")
        ## Allow users to provide a custom config file that takes precedence.
        settings_files = [args.config]

    config = Dynaconf(
        settings_files=settings_files,
        load_dotenv=True,
        envvar_prefix="STIX_SHIFTER_THREATBUS",
    )

    try:
        validate_config(config)
    except Exception as e:
        sys.exit(ValueError(f"Invalid config: {e}"))

    setup_logging_with_config(config.logging)

    while True:
        try:
            asyncio.run(
                start(
                    config.threatbus,
                    config.snapshot,
                    config.modules,
                )
            )
        except (KeyboardInterrupt, SystemExit):
            return
        except asyncio.CancelledError:
            if user_exit:
                # Tasks were cancelled because the user stopped the app.
                return
            logger.info("Restarting stix-shifter-threatbus ...")

if __name__ == "__main__":
    main()


import os
import re
import json
import time
import shutil
import pickle
import logging
from functools import reduce
from itertools import chain
from urllib.parse import urlparse
from datetime import datetime, timedelta
from mimetypes import guess_extension, guess_type
from collections import Counter
from django.utils import timezone
from django.utils.text import slugify
from django.shortcuts import render, redirect
from django.http import HttpResponse, FileResponse, StreamingHttpResponse
from django.views import View
from django.views.decorators.csrf import csrf_exempt
from django.contrib.auth.models import User
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger

from django.contrib.auth.decorators import login_required
from django.conf import settings

from vinanti import Vinanti
from bs4 import BeautifulSoup

from .models import Library, Tags, URLTags, UserSettings
from .forms import AddDir, RenameDir, RemoveDir, AddURL
from .custom_read import CustomRead as cread
from .dbaccess import DBAccess as dbxs
from .summarize import Summarizer
from .utils import ImportBookmarks
from django.db.models import Q

logger = logging.getLogger(__name__)

@login_required
def dashboard(request, username=None, directory=None):
    usr = request.user
    if username and username != usr.username:
        return redirect('/'+usr.username)
    if request.method == 'POST':
        form = AddDir(request.POST)
        if form.is_valid():
            form.clean_and_save_data(usr)
    form = AddDir()
    usr_list = Library.objects.filter(usr=usr).only('directory').order_by('directory')
    usr_list = [i.directory for i in usr_list if i.directory and '/' not in i.directory]
    usr_list = Counter(usr_list)
    nlist = []
    index = 1
    for key, value in usr_list.items():
        base_dir = '{}/{}/{}'.format(settings.ROOT_URL_LOCATION, usr, key)
        base_remove = base_dir + '/remove'
        base_rename = base_dir + '/rename'
        nlist.append([index, key, value-1, base_dir, base_rename, base_remove])
        index += 1
    response = render(
                    request, 'home.html',
                    {
                        'usr_list': nlist, 'form':form,
                        'root':settings.ROOT_URL_LOCATION
                    }
                )
    return response

@login_required
def rename_operation(request, username, directory):
    usr = request.user
    if username and usr.username != username:
        return redirect('logout')
    elif directory:
        if request.method == 'POST':
            form = RenameDir(request.POST)
            if form.is_valid():
                form.clean_and_rename(usr, directory)
            else:
                logger.debug('invalid values {}'.format(request.POST))
            return redirect(request.path_info.rsplit("/", 2)[0])
        else:
            form = RenameDir()
            if '/' in directory:
                base_dir = '{}/{}/subdir/{}'.format(settings.ROOT_URL_LOCATION, usr, directory)
            else:
                base_dir = '{}/{}/{}'.format(settings.ROOT_URL_LOCATION, usr, directory)
            base_remove = base_dir + '/remove'
            base_rename = base_dir + '/rename'
            nlist = [[1, directory, 'N/A', base_dir, base_rename, base_remove]]
            return render(
                        request, 'home.html',
                        {
                            'usr_list': nlist, 'form':form,
                            'root':settings.ROOT_URL_LOCATION
                        }
                    )
    else:
        return redirect('logout')

@login_required
def remove_operation(request, username, directory):
    usr = request.user
    if username and usr.username != username:
        return redirect('logout')
    elif directory:
        if request.method == 'POST':
            form = RemoveDir(request.POST)
            if form.is_valid():
                form.check_and_remove_dir(usr, directory)
            else:
                logger.debug('invalid values {}'.format(request.POST))
            return redirect(request.path_info.rsplit("/", 2)[0])
        else:
            form = RemoveDir()
            if '/' in directory:
                base_dir = '{}/{}/subdir/{}'.format(settings.ROOT_URL_LOCATION, usr, directory)
            else:
                base_dir = '{}/{}/{}'.format(settings.ROOT_URL_LOCATION, usr, directory)
            base_remove = base_dir + '/remove'
            base_rename = base_dir + '/rename'
            nlist = [[1, directory, 'N/A', base_dir, base_rename, base_remove]]
            return render(
                        request, 'home.html',
                        {
                            'usr_list': nlist, 'form':form,
                            'root':settings.ROOT_URL_LOCATION
                        }
                    )
    else:
        return redirect('logout')

@login_required
def get_resources(request, username, directory, url_id):
    usr = request.user
    logger.info(request.path_info)
    if username and usr.username != username:
        return HttpResponse('Not Allowed')
    elif directory and url_id:
        if request.method == 'GET':
            resource_dir = os.path.join(settings.ARCHIVE_LOCATION, 'resources', str(url_id))
            loc = request.path_info.rsplit('/', 1)[1]
            if loc.endswith('.css'):
                content_type = 'text/css'
            else:
                content_type = 'image/jpeg'
            resource_loc = os.path.join(resource_dir, loc)
            logger.debug('resource-loc: {}'.format(resource_loc))
            if os.path.exists(resource_loc):
                response = FileResponse(open(resource_loc, 'rb'))
                response['content-type'] = content_type
                response['content-length'] = os.stat(resource_loc).st_size
                return response
            else:
                logger.debug('resource: {} not available'.format(resource_loc))
    return HttpResponse('Not Found')

@login_required
def get_relative_resources(request, username, directory, url_id, rel_path):
    usr = request.user
    logger.info(request.path_info)
    if directory and url_id:
        if request.method == 'GET':
            resource_loc = os.path.join(settings.ARCHIVE_LOCATION, 'PDF', str(url_id), rel_path)
            if resource_loc.endswith('.css'):
                content_type = 'text/css'
            elif resource_loc.endswith("png"):
                content_type = 'image/png'
            else:
                content_type = 'image/jpeg'
            logger.debug('resource-loc: {}'.format(resource_loc))
            if os.path.exists(resource_loc):
                response = FileResponse(open(resource_loc, 'rb'))
                response['content-type'] = content_type
                response['content-length'] = os.stat(resource_loc).st_size
                return response
            else:
                logger.debug('resource: {} not available'.format(resource_loc))
    return HttpResponse('Not Found')

@login_required
def perform_epub_operation(request, username, directory, url_id=None, opt=None, meta_path=None):
    usr = request.user
    logger.info(request.path_info)
    if username and usr.username != username:
        return HttpResponse('Not Allowed')
    elif directory and url_id:
        if meta_path:
            return cread.read_epub(usr, url_id, mode='read-epub-meta', req=request, rel_path=meta_path)
        else:
            url = request.path_info.rsplit("/", 2)[0]
            return redirect(url)

@login_required
def perform_link_operation(request, username, directory, url_id=None):
    usr = request.user
    logger.info(request.path_info)
    if username and usr.username != username:
        return HttpResponse('Not Allowed')
    elif directory and url_id:
        if request.method == 'POST':
            if request.path_info.endswith('remove'):
                rem_lnk = request.POST.get('remove_url', '')
                logger.debug('{} , {}'.format(url_id, request.POST))
                if rem_lnk == 'yes':
                    dbxs.remove_url_link(usr, url_id=url_id)
                return HttpResponse('Deleted')
            elif request.path_info.endswith('move-bookmark'):
                msg = dbxs.move_bookmarks(usr, request, url_id)
                return HttpResponse(msg)
            elif request.path_info.endswith('edit-bookmark'):
                msg = dbxs.edit_bookmarks(usr, request, url_id)
                return HttpResponse(msg)
            elif request.path_info.endswith('archived-note-save'):
                return cread.save_customized_note(usr, url_id, mode='save-note', req=request)
            else:
                return HttpResponse('Wrong command')
        elif request.method == 'GET':
            if request.path_info.endswith('archive'):
                return cread.get_archived_file(usr, url_id, mode='archive', req=request)
            elif request.path_info.endswith('archived-note'):
                return cread.read_customized_note(usr, url_id, mode='read-note', req=request)
            elif request.path_info.endswith('read'):
                return cread.read_customized(usr, url_id, mode='read', req=request)
            elif request.path_info.endswith('pdf-annot'):
                return cread.read_customized(usr, url_id, mode='pdf-annot', req=request)
            elif request.path_info.endswith('read-dark'):
                return cread.read_customized(usr, url_id, mode='read-dark', req=request)
            elif request.path_info.endswith('read-light'):
                return cread.read_customized(usr, url_id, mode='read-light', req=request)
            elif request.path_info.endswith('read-gray'):
                return cread.read_customized(usr, url_id, mode='read-gray', req=request)
            elif request.path_info.endswith('read-default'):
                return cread.read_customized(usr, url_id, mode='read-default', req=request)
            elif request.path_info.endswith('read-pdf'):
                return cread.get_archived_file(usr, url_id, mode='pdf', req=request)
            elif request.path_info.endswith('read-png'):
                return cread.get_archived_file(usr, url_id, mode='png', req=request)
            elif request.path_info.endswith('read-html'):
                return cread.get_archived_file(usr, url_id, mode='html', req=request)
        else:
            return HttpResponse('Method not Permitted')
    elif directory and request.method == 'POST':
        msg = 'nothing'
        if request.path_info.endswith('move-bookmark-multiple'):
            msg = dbxs.move_bookmarks(usr, request, single=False)
        elif request.path_info.endswith('archive-bookmark-multiple'):
            msg = dbxs.group_links_actions(usr, request, directory, mode='archive')
        elif request.path_info.endswith('merge-bookmark-with'):
            msg = dbxs.group_links_actions(usr, request, directory, mode='merge')
        elif request.path_info.endswith('edit-tags-multiple'):
            msg = dbxs.group_links_actions(usr, request, directory, mode='tags')
        return HttpResponse(msg)
    else:
        return HttpResponse('What are you trying to accomplish!')

@login_required
def default_dest(request):
    return redirect('home')

def public_profile(request, username):
    qlist = User.objects.filter(username=username)
    if qlist:
        usr = qlist[0]
        qlist = UserSettings.objects.filter(usrid=usr)
        if qlist:
            public_dir = qlist[0].public_dir
            if public_dir:
                usr_list = dbxs.get_rows_by_directory(usr, directory=public_dir)
                nlist = dbxs.populate_usr_list(usr, usr_list)
                base_dir = '{}/{}/{}'.format(settings.ROOT_URL_LOCATION, usr, public_dir)
                return render(
                            request, 'public.html',
                            {
                                'usr_list': nlist, 'form':"",
                                'base_dir':base_dir, 'dirname':public_dir,
                                'root': settings.ROOT_URL_LOCATION
                            }
                        )
    return HttpResponse('No Public Profile Available')

@login_required
def group_profile(request, username):
    group_usr = request.user
    qlist = User.objects.filter(username=username)
    if qlist:
        usr = qlist[0]
        qlist = UserSettings.objects.filter(usrid=usr)
        if qlist:
            buddy_list = qlist[0].buddy_list
            group_dir = qlist[0].group_dir
            if buddy_list and group_dir:
                nbuddy = [i.strip() for i in buddy_list.split(',') if i.strip()]
                if group_usr.username in nbuddy:
                    usr_list = dbxs.get_rows_by_directory(usr, directory=group_dir)
                    nlist = dbxs.populate_usr_list(usr, usr_list)
                    base_dir = '{}/{}/{}'.format(settings.ROOT_URL_LOCATION, usr, group_dir)
                    api_url = '{}/{}/api/request'.format(settings.ROOT_URL_LOCATION, usr)
                    return render(
                                request, 'home_dir.html',
                                {
                                    'usr_list': nlist, 'form':"",
                                    'base_dir':base_dir, 'dirname':group_dir,
                                    'refresh': 'no', 'root': settings.ROOT_URL_LOCATION,
                                    'api_url': api_url, 'dir_list':[("active", group_dir, base_dir)]
                                }
                            )
    return HttpResponse('No Group Profile Available')

def record_epub_loc(request, epub_loc):
    path_info, epub_url_cfi = request.path_info.split("/epub-bookmark/", 1)
    url_id, cfi = epub_loc.split("/", 1)
    row = Library.objects.filter(usr=request.user, id=int(url_id)).first()
    media_path = row.media_path
    logger.debug(media_path)
    media_path_dir, _ = os.path.split(media_path)
    epub_loc = os.path.join(media_path_dir, "epub_loc.txt")
    with open(epub_loc, 'w') as f:
        f.write(cfi)
    return redirect(path_info)

@login_required
def navigate_directory(request, username, directory=None, tagname=None, epub_loc=None):

    usr = request.user
    base_dir = '{}/{}/{}'.format(settings.ROOT_URL_LOCATION, usr, directory)
    usr_list = []
    if username and usr.username != username:
        return redirect(settings.ROOT_URL_LOCATION+'/'+usr.username)
    add_url = 'no'
    if directory or tagname:
        if epub_loc:
            return record_epub_loc(request, epub_loc)

        place_holder = 'Enter URL'
        if request.method == 'POST' and directory:
            form = AddURL(request.POST)
            url_name = request.POST.get('add_url', '')
            if (form.is_valid() or (url_name and url_name.startswith('md:'))
                    or (url_name and url_name.startswith('note:'))):
                row = UserSettings.objects.filter(usrid=usr)
                dbxs.add_new_url(usr, request, directory, row)
                add_url = 'yes'
            else:
                place_holder = 'Wrong Input, Enter URL'
        form = AddURL()
        form.fields['add_url'].widget.attrs['placeholder'] = place_holder
        if directory:
            usr_list = dbxs.get_rows_by_directory(usr, directory=directory)
        else:
            directory = 'tag'
            usr_list = dbxs.get_rows_by_tag(usr, tagname)
            if usr_list is None:
                return redirect('home')
        nlist = dbxs.populate_usr_list(usr, usr_list)

        page = request.GET.get('page', 1)
        row = UserSettings.objects.filter(usrid=usr)
        if row and row[0].pagination_value:
            paginator = Paginator(nlist, row[0].pagination_value)
        else:
            paginator = Paginator(nlist, 100)
        try:
            dirlist = paginator.page(page)
        except PageNotAnInteger:
            dirlist = paginator.page(1)
        except EmptyPage:
            dirlist = paginator.page(paginator.num_pages)
        if '/' in directory:
            base_dir = '{}/{}/subdir/{}'.format(settings.ROOT_URL_LOCATION, usr, directory)
        else:
            base_dir = '{}/{}/{}'.format(settings.ROOT_URL_LOCATION, usr, directory)
        dir_list = [("", "Home", settings.ROOT_URL_LOCATION+'/'+username)]
        if '/' in directory:
            dir_split = directory.split('/')
            for i, j in enumerate(dir_split):
                if i == 0:
                    nbase = '{}/{}/{}'.format(settings.ROOT_URL_LOCATION, username, j)
                elif i == 1:
                    is_active, dn, path = dir_list[-1]
                    last = path.split('/')[-1]
                    nbase = '{}/{}/subdir/{}/{}'.format(settings.ROOT_URL_LOCATION, username, last, j)
                else:
                    is_active, dn, path = dir_list[-1]
                    nbase = path + '/' + j
                if i == len(dir_split) - 1:
                    dir_list.append(("active",j, nbase))
                else:
                    dir_list.append(("",j, nbase))
        else:
            dir_list.append(("active", directory, base_dir))
        api_url = '{}/{}/api/request'.format(settings.ROOT_URL_LOCATION, username)
        return render(
                    request, 'home_dir.html',
                    {
                        'usr_list': dirlist, 'form':form,
                        'base_dir':base_dir, 'dirname':directory,
                        'refresh':add_url, 'root':settings.ROOT_URL_LOCATION,
                        'api_url': api_url, 'dir_list': dir_list
                    }
                )
    else:
        return redirect('home')

@login_required
def record_reading_position(request, username, directory=None, url_id=None, html_pos=None, mode=None):
    prev_loc, url_id, read_type = request.path_info.rsplit('/', 2)
    usr = request.user
    if username and usr.username != username:
        HttpResponse(status=401)
    if request.method == 'POST' and directory and html_pos and mode:
        row = Library.objects.filter(usr=request.user, id=int(url_id)).first()
        media_path = row.media_path
        logger.debug(media_path)
        media_path_dir, _ = os.path.split(media_path)
        if mode == "readhtml":
            loc = os.path.join(media_path_dir, "html_original_loc.txt")
        elif mode == "readcustom":
            loc = os.path.join(media_path_dir, "html_custom_loc.txt")
        elif mode in ["readpdf", "pdf-annotpdf"]:
            loc = os.path.join(media_path_dir, "pdf_loc.txt")
        if html_pos.startswith("-"):
            html_pos = html_pos[1:]
        with open(loc, 'w') as f:
            f.write(html_pos)
        return HttpResponse('OK')
    else:
        HttpResponse(status=404)

@login_required
def navigate_subdir(request, username, directory=None, epub_loc=None):
    ops = set([
        "archive", "remove", "read", "read-pdf", "read-png",
        "read-html", "read-dark", "read-light", "read-default",
        "read-gray", "edit-bookmark", "move-bookmark",
        "archived-note", "archived-note-save", "pdf-annot"
    ])
    if directory:
        if epub_loc:
            return record_epub_loc(request, epub_loc)
        link_split = directory.split('/')
        op = link_split[-1]
        if len(link_split) >= 2:
            link_id = link_split[-2]
        else:
            link_id = None
        if op in ops and link_id and link_id.isnumeric():
            return perform_link_operation(request, username, directory, int(link_id))
        elif op == "remove":
            dirn, _ = directory.rsplit('/', 1)
            return remove_operation(request, username, dirn)
        elif op == "rename":
            dirn, _ = directory.rsplit('/', 1)
            return rename_operation(request, username, dirn)
        else:
            return navigate_directory(request, username, directory)
    else:
        return redirect('home')

def get_archived_video_link(request, username, video_id):
    if video_id and '-' in video_id:
        _, video_id = video_id.rsplit('-', 1)
    if os.path.isfile(cread.CACHE_FILE):
        with open(cread.CACHE_FILE, 'rb') as fd:
            cread.VIDEO_ID_DICT = pickle.load(fd)
    return cread.get_archived_video(request, username, video_id)

@login_required
def get_archived_playlist(request, username, directory, playlist_id):
    plfile = os.path.join(settings.TMP_LOCATION, playlist_id)
    pls_txt = ''
    if os.path.isfile(plfile):
        with open(plfile, 'rb') as fd:
            pls_txt = pickle.load(fd)
        os.remove(plfile)
    response = HttpResponse()
    response['Content-Type'] = 'audio/mpegurl'
    response['Content-Disposition'] = 'attachment; filename={}.m3u'.format(directory)
    response.write(bytes(pls_txt, 'utf-8'))
    return response

def create_annotations(request):
    req_body = json.loads(request.body)
    uri = req_body.get("uri")
    logger.debug(uri)
    annot_file = get_annot_file(uri, request.user)
    id_created = "0"
    if os.path.isfile(annot_file):
        with open(annot_file) as fl:
            data = json.load(fl)
        total = data.get("total")
        index = data.get("index")
        rows = data.get("rows")
        id_created = str(index + 1)
        req_body.update({"id": id_created})
        rows.append(req_body.copy())
        js = {"total": len(rows), "index": int(id_created), "rows":rows}
    else:
        req_body.update({"id": id_created})
        js = {"total": 1, "index": int(id_created), "rows":[req_body.copy()]}
    with open(annot_file, "w") as fl:
        fl.write(json.dumps(js, ensure_ascii=False))
    return HttpResponse(json.dumps(req_body))

def annotation_root(request):
    return HttpResponse(status=200)

def get_annot_file(uri, usr):
    url = uri
    offset = None
    if "#" in uri:
        uri, offset = uri.split("#")
    url_id = uri.split('/')[-2]
    logger.debug(url_id)
    row = Library.objects.filter(usr=usr, id=int(url_id)).first()
    media_path = row.media_path
    logger.debug(media_path)
    media_path_dir, _ = os.path.split(media_path)
    logger.debug(media_path_dir)
    mode = uri.rsplit('/', 1)[-1]
    slug = None
    if offset:
        slug = slugify(offset, allow_unicode=True)
        slug = slug.replace("-", "_")
    if slug:
        annot_file = os.path.join(media_path_dir, "annot_custom_{}.json".format(slug))
    else:
        if mode in ["read", "read-dark", "read-light", "read-default","read-gray"]:
            annot_file = os.path.join(media_path_dir, "annot_custom.json")
        else:
            annot_file = os.path.join(media_path_dir, "annot_original.json")
    return annot_file

def search_annotations(request):
    uri = request.GET["uri"]
    annot_file = get_annot_file(uri, request.user)
    if not os.path.exists(annot_file):
        return HttpResponse(json.dumps({"total":0, "index": -1}))
    else:
        with open(annot_file, "rb") as fl:
            data = fl.read()
        return HttpResponse(data)

def modify_annotations(request, annot_id):
    req_body = json.loads(request.body)
    uri = req_body.get("uri")
    logger.debug(uri)
    annot_file = get_annot_file(uri, request.user)
    if request.method == "PUT":
        if os.path.isfile(annot_file):
            with open(annot_file) as fl:
                data = json.load(fl)
            rows = data.get("rows")
            req_index = get_annot_index(annot_id, rows)
            if req_index is not None:
                rows[req_index] = req_body
            js = {"total": data.get("total"), "index": data.get("index"), "rows":rows}
            with open(annot_file, "w") as fl:
                fl.write(json.dumps(js, ensure_ascii=False))
        return HttpResponse(json.dumps(req_body))
    elif request.method == "DELETE":
        if os.path.isfile(annot_file):
            with open(annot_file) as fl:
                data = json.load(fl)
            rows = data.get("rows")
            index = data.get("index")
            total = data.get("total")
            req_index = get_annot_index(annot_id, rows)
            if req_index is not None:
                del rows[req_index]
            logger.info(rows)
            js = {"total": len(rows), "index": data.get("index"), "rows":rows}
            with open(annot_file, "w") as fl:
                fl.write(json.dumps(js, ensure_ascii=False))
            return HttpResponse(status=204)

def get_annot_index(annot_id, rows):
    del_index = str(annot_id)
    req_index = None
    for counter, row in enumerate(rows):
        if row.get("id") == del_index:
            req_index = counter
            logger.info("--deleting---{}".format(row))
            break
    return req_index

@login_required
def api_points(request, username):
    usr = request.user
    default_dict = {'status':'none'}
    if username and usr.username != username:
        return redirect(settings.ROOT_URL_LOCATION+'/'+usr.username)
    if request.method == 'POST':
        req_list = request.POST.get('listdir', '')
        req_search = request.POST.get('search', '')
        req_archive = request.POST.get('archive', '')
        req_get_settings = request.POST.get('get_settings', '')
        req_set_settings = request.POST.get('set_settings', '')
        req_summary = request.POST.get('req_summary', '')
        req_import = request.POST.get('import-bookmark', '')
        req_upload = request.POST.get('upload-binary', '')
        req_chromium_backend = request.POST.get('chromium-backend', '')
        req_media_path = request.POST.get('get-media-path', '')
        req_media_playlist = request.POST.get('generate-media-playlist', '')
        req_subdir = request.POST.get('create_subdir', '')
        logger.debug(req_import)
        logger.debug(request.FILES)
        if req_list and req_list == 'yes':
            q_list = Library.objects.filter(usr=usr)
            dir_list = set()
            for i in q_list:
                dirn = i.directory
                if dirn:
                    dir_list.add(dirn)
            dir_list = list(dir_list)
            dir_list.sort()
            dir_dict = {'dir':dir_list}
            return HttpResponse(json.dumps(dir_dict))
        elif req_subdir and req_subdir == "yes":
            pdir = request.POST.get('parent_dir', '')
            subdir = request.POST.get('subdir_name', '')
            if pdir and subdir:
                dirname = re.sub(r'/|:|#|\?|\\\\|\%', '-', subdir)
                if dirname:
                    dirname = pdir+'/'+dirname
                    qdir = Library.objects.filter(usr=usr, directory=dirname)
                    if not qdir:
                        Library.objects.create(usr=usr, directory=dirname, timestamp=timezone.now()).save()
                        qlist = Library.objects.filter(usr=usr, directory=pdir, url__isnull=True).first()
                        if qlist:
                            if qlist.subdir:
                                slist = qlist.subdir.split('/')
                                if subdir not in slist:
                                    qlist.subdir = '/'.join(slist + [subdir])
                                    qlist.save()
                            else:
                                qlist.subdir = subdir
                                qlist.save()
                return HttpResponse('OK')
        elif req_media_path and req_media_path == 'yes':
            url_id = request.POST.get('url_id', '')
            if url_id and url_id.isnumeric():
                return_path = cread.get_archived_file(usr, url_id, mode='archive',
                                                      req=request, return_path=True)
                return HttpResponse(json.dumps({'link':return_path}))
        elif req_media_playlist and req_media_playlist == 'yes':
            directory = request.POST.get('directory', '')
            ip = request.POST.get('ip', '')
            logger.debug('{} {}'.format(directory, ip))
            pls_path = cread.generate_archive_media_playlist(ip, usr, directory)
            return HttpResponse(pls_path)
        elif req_import and req_import == 'yes':
            req_file = request.FILES.get('file-upload', '')
            if req_file:
                filename = req_file.name
                logger.info(filename)
                mime_type = guess_type(filename)[0]
                logger.info(mime_type)
                if mime_type in ['text/html', 'text/htm']:
                    content = req_file.read().decode('utf-8')
                    qlist = UserSettings.objects.filter(usrid=usr)
                    if qlist:
                        settings_row = qlist[0]
                    else:
                        settings_row = None
                    ImportBookmarks.import_bookmarks(usr, settings_row,
                                                     content, mode='content')
            return HttpResponse('OK')
        elif req_upload and req_upload == 'yes':
            dirname = request.POST.get('dirname', '')
            if not dirname:
                dirname = 'Uploads'
                qdir = Library.objects.filter(usr=usr, directory=dirname)
                if not qdir:
                    Library.objects.create(usr=usr, directory=dirname, timestamp=timezone.now()).save()
            dbxs.save_in_binary_format(usr, request, dirname)
            return HttpResponse('OK')
        elif req_search and len(req_search) > 2:
            tag_list = []
            if req_search.startswith('tag:'):
                search_term = req_search.split(':', 1)[1]
                mode = 'tag'
            elif req_search.startswith('url:'):
                search_term = req_search.split(':', 1)[1]
                mode = 'url'
            elif req_search.startswith('dir:'):
                search_term = req_search.split(':', 1)[1]
                mode = 'dir'
            elif req_search.startswith('sum:'):
                search_term = req_search.split(':', 1)[1]
                mode = 'summary'
            elif req_search.startswith('tag-wall:'):
                qlist = Library.objects.filter(usr=usr)
                tagset = set()
                mode = 'tag-wall'
                if qlist:
                    tags = [row.tags.split(',') for row in qlist if row.tags]
                    fun = lambda *x : set(chain(*x))
                    tag_list = list(reduce(fun, tags))
                    tag_list.sort()
            else:
                search_term = req_search
                mode = 'title'
            if mode == 'tag-wall':
                usr_list = [('Total Tags', '', '', timezone.now(), tag_list, 'Tag-Wall', '', '')]
            else:
                logger.info('{}->{}'.format(mode, search_term))
                usr_list = dbxs.get_rows_by_directory(usr, search_mode=mode, search=search_term)
            ndict = dbxs.populate_usr_list(usr, usr_list, create_dict=True)
            ndict.update({'root':settings.ROOT_URL_LOCATION})
            return HttpResponse(json.dumps(ndict))
        elif req_summary and req_summary == 'yes':
            ndict = {}
            reqid = request.POST.get('url_id', '')
            summary = 'not available'
            if reqid and reqid.isnumeric():
                qlist = Library.objects.filter(usr=usr, id=int(reqid))
                if qlist:
                    row = qlist[0]
                    summary = row.summary
                    if not summary:
                        media_path = row.media_path
                        if media_path and os.path.exists(media_path):
                            content = ''
                            with open(media_path, mode='r', encoding='utf-8') as fd:
                                content = fd.read()
                            if content:
                                summary, tags_list = Summarizer.get_summary_and_tags(content, 2)
                    if not summary:
                        summary = ('Automatic Summary not generated!\
                                    First enable automatic summary generation\
                                    from settings. Remove this link and add it again.')
                        summary = re.sub(' +', ' ', summary)
            ndict.update({'summary':summary})
            return HttpResponse(json.dumps(ndict))
        elif req_summary and req_summary == 'modify':
            reqid = request.POST.get('url_id', '')
            summary = request.POST.get('modified_summary', '')
            msg = "no modied"
            logger.debug(summary)
            if reqid and reqid.isnumeric() and summary:
                qlist = Library.objects.filter(usr=usr, id=int(reqid)).update(summary=summary)
                msg = 'modified summary'
            return HttpResponse(msg)
        elif req_get_settings and req_get_settings == 'yes':
            qlist = UserSettings.objects.filter(usrid=usr)
            if qlist:
                row = qlist[0]
                if row.public_dir:
                    public_dir = row.public_dir
                else:
                    public_dir = ""
                if row.group_dir:
                    group_dir = row.group_dir
                else:
                    group_dir = ""
                ndict = {
                    'autotag': row.autotag,
                    'auto_summary':row.auto_summary,
                    'total_tags': row.total_tags,
                    'public_dir': public_dir,
                    'group_dir': group_dir,
                    'save_pdf': row.save_pdf,
                    'save_png': row.save_png,
                    'png_quality': row.png_quality,
                    'auto_archive': row.auto_archive,
                    'pagination_value': row.pagination_value,
                    'download_manager': row.download_manager,
                    'media_streaming': row.media_streaming
                }
                if row.buddy_list:
                    ndict.update({'buddy':row.buddy_list})
                else:
                    ndict.update({'buddy':''})
            else:
                ndict = {
                    'autotag': False,
                    'auto_summary': False,
                    'total_tags': 5,
                    'buddy': "",
                    'public_dir': "",
                    'group_dir': "",
                    'save_pdf': False,
                    'save_png': False,
                    'png_quality': 85,
                    'auto_archive': False,
                    'pagination_value': 100,
                    'download_manager': 'wget {iurl} -O {output}',
                    'media_streaming': False
                }
            return HttpResponse(json.dumps(ndict))
        elif req_set_settings and req_set_settings == 'yes':
            autotag = request.POST.get('autotag', 'false')
            auto_summary = request.POST.get('auto_summary', 'false')
            total_tags = request.POST.get('total_tags', '5')
            buddy_list = request.POST.get('buddy_list', '')
            public_dir = request.POST.get('public_dir', '')
            group_dir = request.POST.get('group_dir', '')
            save_pdf = request.POST.get('save_pdf', '')
            save_png = request.POST.get('save_png', '')
            png_quality = request.POST.get('png_quality', '')
            auto_archive = request.POST.get('auto_archive', '')
            pagination_value = request.POST.get('pagination_value', '100')
            media_streaming = request.POST.get('media_streaming', 'false')
            dm_str = 'wget {iurl} -O {output}'
            download_manager = request.POST.get('download_manager', dm_str)
            if autotag == 'true':
                autotag = True
            else:
                autotag = False
            if auto_summary == 'true':
                auto_summary = True
            else:
                auto_summary = False
            if total_tags.isnumeric():
                total_tags = int(total_tags)
            else:
                total_tags = 5
            if pagination_value.isnumeric():
                pagination_value = int(pagination_value)
            else:
                pagination_value = 100
            if save_pdf == 'true':
                save_pdf = True
            else:
                save_pdf = False
            if save_png == 'true':
                save_png = True
            else:
                save_png = False
            if auto_archive == 'true':
                auto_archive = True
            else:
                auto_archive = False
            if media_streaming == 'true':
                media_streaming = True
            else:
                media_streaming = False
            if png_quality and png_quality.isnumeric():
                png_quality = int(png_quality) if int(png_quality) in range(0, 101) else 85
            else:
                png_quality = 85
            if buddy_list:
                buddy_list = [i.strip() for i in buddy_list.split(',') if i.strip()]
                buddy_list = ','.join(buddy_list)
            qlist = UserSettings.objects.filter(usrid=usr)
            if qlist:
                row = qlist[0]
                row.autotag = autotag
                row.auto_summary = auto_summary
                row.total_tags = total_tags
                row.buddy_list = buddy_list
                row.public_dir = public_dir
                row.group_dir = group_dir
                row.save_pdf = save_pdf
                row.save_png = save_png
                row.png_quality = png_quality
                row.auto_archive = auto_archive
                row.pagination_value = pagination_value
                row.media_streaming = media_streaming
                row.download_manager = download_manager
                row.save()
            else:
                row = UserSettings.objects.create(usrid=usr, autotag=autotag,
                                                  auto_summary=auto_summary,
                                                  total_tags=total_tags,
                                                  buddy_list=buddy_list,
                                                  public_dir=public_dir,
                                                  group_dir=group_dir,
                                                  save_pdf=save_pdf,
                                                  save_png=save_png,
                                                  png_quality=png_quality,
                                                  auto_archive=auto_archive,
                                                  pagination_value=pagination_value,
                                                  media_streaming=media_streaming,
                                                  download_manager=download_manager)
                row.save()
            if (autotag or auto_summary) and not os.path.exists(settings.NLTK_DATA_PATH):
                dbxs.vnt_task.function(Summarizer.check_data_path)
            ndict = {'status':'ok'}
            return HttpResponse(json.dumps(ndict))
        elif req_chromium_backend and req_chromium_backend == 'yes':
            url_id = request.POST.get('url_id', '')
            mode = request.POST.get('mode', '')
            if mode and mode in ['pdf', 'dom']:
                qlist = UserSettings.objects.filter(usrid=usr)
                if qlist:
                    settings_row = qlist[0]
                else:
                    settings_row = None
                qset = Library.objects.filter(usr=usr, id=url_id)
                if qset:
                    row = qset[0]
                    if row.media_path:
                        media_path_parent, _ = os.path.split(row.media_path)
                        dbxs.convert_html_pdf_with_chromium(media_path_parent,
                                                            settings_row, row,
                                                            row.url, row.media_path,
                                                            mode=mode)
                return HttpResponse('OK')
        elif req_archive and req_archive in ['yes', 'force']:
            url_id = request.POST.get('url_id', '')
            dirname = request.POST.get('dirname', '')
            logger.debug('{}, {}'.format(url_id, dirname))
            if url_id and url_id.isnumeric() and dirname:
                qset = Library.objects.filter(usr=usr, id=url_id)
                if qset:
                    row = qset[0]
                    media_path = row.media_path
                    url_ar = '{}/{}/{}/{}/archive'.format(settings.ROOT_URL_LOCATION,
                                                          usr.username, dirname, url_id)
                    dict_val = {'url': url_ar}
                    if media_path and os.path.exists(media_path) and req_archive == 'yes':
                        dict_val.update({'status':'already archived'})
                    else:
                        qlist = UserSettings.objects.filter(usrid=usr)
                        if qlist:
                            set_row = qlist[0]
                        else:
                            set_row = None
                        dbxs.process_add_url(usr, row.url, dirname,
                                             archive_html=True, row=row,
                                             settings_row=set_row)
                        dict_val.update({'status':'ok'})
                    return HttpResponse(json.dumps(dict_val))

    return HttpResponse(json.dumps(default_dict))


import logging

from lib import script
from lib import states

"""
This script will take nodes that are active but don't have an instance
assigned to them and tear down them down, then decommission them.

Nodes can get in this state due to bugs in Nova resulting from running
HA nova-computes. Nodes can get rescheduled, but have already attempted
a provision on Ironic. Ironic will continue, but never get an instance
UUID, which means the node doesn't have a customer on it.
"""

logger = logging.getLogger(__name__)

class FixActiveNoInstance(script.TeethScript):
    use_ironic = True

    def __init__(self):
        super(FixActiveNoInstance, self).__init__(
            'Utility for fixing nodes that are active without an instance'
            'UUID.')

    def _is_active_no_instance(self, node):
        return (node['instance_uuid'] is None
                and node['provision_state'] == states.ACTIVE)

    def _reverify(self, node):
        # Returns True if node is still active no instance, False otherwise
        node = self.ironic_client.get_node(node['uuid'])
        if not self._is_active_no_instance(node):
            logger.error(
                'Node {uuid} in unexpected state: instance_uuid: '
                '{instance_uuid}; provision_state: '
                '{provision_state}.'.format(**node))
            return False
        return True

    def run(self):
        nodes = self.ironic_client.list_nodes()
        for node in nodes:
            if self._is_active_no_instance(node):
                if not self._reverify(node):
                    continue
                self.ironic_client.set_target_state(node, states.DELETED)

if __name__ == "__main__":
    FixActiveNoInstance().run()


from nltk.tokenize import sent_tokenize, word_tokenize
import sys

def tokenize(paragraphs):
    paras = []
    for p in paragraphs:
        for i, s in enumerate(sent_tokenize(p)):
            token = word_tokenize(s)
            token.append({"length": len(token), "score": 0, "full": 0})
            if len(paras) > i:
                paras[i].append(token)
            else:
                paras.append([token])

    for i in range(len(paras)):
        p = paras[i]
        for base_index in range(len(p)):
            for compare_index in range(len(p)):
                if compare_index != base_index:
                    last_compared = -float("inf")
                    for word in p[compare_index][:-1]:
                        if word in p[base_index] and p[base_index].index(word) > last_compared:
                            p[base_index][-1]["score"] += 1
                            if word == p[compare_index][-2]:
                                p[base_index][-1]["full"] += 1
                        else:
                            break

    results = []
    for i in range(len(paras)):
        p = paras[i]
        m_score = -1
        m_s = []
        for sen in p:
            scores = sen[-1]
            c_score = (scores["score"]/scores["length"] +
                       scores["full"]) * scores["length"]
            if c_score > m_score:
                m_s = sen[:-1]
                m_score = c_score
        if len(results) > i:
            results[i].append(m_s)
        else:
            results.append([m_s])

    op = ""
    for sentence in results:
        op += " ".join(sentence[0][:-1])
        op += sentence[0][-1] + " "
    return op

def main(*paragraphs):
    print(tokenize(paragraphs))
    sys.stdout.flush()

if __name__ == "__main__":
    main(sys.argv[1:])


from flask_restx import Api, Resource, fields
from flask_jwt import jwt_required, current_identity
from models.api_meta_class import MetaAPI
from flask import request

from models.api_response import APIResponse, EAPIResponseCode
from api import module_api
from config import ConfigClass
from resources.utils import get_project_permissions
from services.permissions_service.decorators import permissions_check
import requests

api_ns_report = module_api.namespace(
    'Announcement', description='Announcement API', path='/v1')

class APIAnnouncement(metaclass=MetaAPI):
    def api_registry(self):
        api_ns_report.add_resource(
            self.AnnouncementRestful, '/announcements')

    class AnnouncementRestful(Resource):
        @jwt_required()
        @permissions_check("announcement", "*", "view")
        def get(self):
            api_response = APIResponse()
            data = request.args
            if not data.get("project_code"):
                api_response.set_error_msg("Missing project code")
                api_response.set_code(EAPIResponseCode.bad_request)
                return api_response.to_dict, api_response.code

            response = requests.get(ConfigClass.NOTIFY_SERVICE + "/v1/announcements", params=data)
            if response.status_code != 200:
                api_response.set_error_msg(response.json())
                return api_response.to_dict, response.status_code
            api_response.set_result(response.json())
            return api_response.to_dict, api_response.code

        @jwt_required()
        @permissions_check("announcement", "*", "create")
        def post(self):
            api_response = APIResponse()
            data = request.get_json()
            if not data.get("project_code"):
                api_response.set_error_msg("Missing project code")
                api_response.set_code(EAPIResponseCode.bad_request)
                return api_response.to_dict, api_response.code

            # Get Dataset
            response = requests.post(ConfigClass.NEO4J_SERVICE + "nodes/Container/query", json={"code": data["project_code"]})
            if not response.json():
                api_response.set_error_msg("Dataset not found")
                api_response.set_code(EAPIResponseCode.not_found)
                return api_response.to_dict, api_response.code
            dataset_id = response.json()[0]["id"]

            data["publisher"] = current_identity["username"]
            response = requests.post(ConfigClass.NOTIFY_SERVICE + "/v1/announcements", json=data)
            if response.status_code != 200:
                api_response.set_error_msg(response.json()["error_msg"])
                response_dict = api_response.to_dict
                response_dict["code"] = response.status_code
                return response_dict, response.status_code
            api_response.set_result(response.json())
            return api_response.to_dict, api_response.code


import datetime

'''
将shopid.txt文件的内容格式化到shopid.yaml并保存
'''
with open('./all_shopid.yaml', 'a', encoding='utf-8') as f:
    with open('shopid.txt', "r", encoding="utf-8") as file_obj:
        f.write("shop_id:")
        content = file_obj.read()
        a = content.split("\n")
        print(a)
        for index in a:
            text = f'\n- \'{index}\''
            f.write(text)
        f.write(f"\nupdate_time: {str(datetime.date.today())}")


import pstats, cProfile, io

def profile(fnc):
    """
    An implementation of a function decorator that wraps a function in
    a code that profiles it.
    """
    def inner(*args, **kwargs):
        pr = cProfile.Profile()
        pr.enable()

        # wrapped function starts
        retval = fnc(*args, **kwargs) #fnc is whatever function has the @profile tag
        # wrapped function ends

        pr.disable()
        s = io.StringIO()
        sortby = pstats.SortKey.CALLS
        ps = pstats.Stats(pr, stream=s).strip_dirs().sort_stats(sortby)
        ps.print_stats()
        print(s.getvalue())
        return retval

    return inner

#profile function is called first, passing in heavy_function as its parameter
@profile
def heavy_function():
    for x in range(0,1000):
        for y in range(0,100):
            for z in range(0,100):
                z += x + y

#profile function is called first, passing in heavy_function_2 as its parameter
@profile
def heavy_function_2():
    for a in range(0,1000):
        for b in range(0,100):
            for c in range(0,100):
                c += a * b

def main():
    heavy_function() #when heavy_function is executed it's first wrapped in the profile function, then executed
    pass

if __name__ == '__main__':
    main()


"""Tubes URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/3.1/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path, include
from user.views import *
from rest_framework import routers

router = routers.DefaultRouter()
router.register('ujian', userUjianViewSet)
router.register('jadwal', userJadwalViewSet)
router.register('tugas', userTugasViewSet)
router.register('mahasiswa', kelasViewSet)
router.register('presensi', presensiViewSet)
router.register('persentase', userPresensiViewSet)
router.register('submisi', userSubmisiViewSet)
router.register('cekpresensi', cekPresensiViewSet)
router.register('cekpertemuan', cekPertemuanViewSet)
router.register('ceksubmisi', cekSubmisiViewSet)
router.register('history', historyViewSet)

urlpatterns = [
    path('api/', include(router.urls)),
    path('admin/', admin.site.urls),
    path('api/account/', include('account.api.urls', 'account_api')),
    path('api/time/', timeView.as_view()),
]


# -*- coding: utf-8 -*-
#写一个通用的 日志 函数，返回一个日志实例
#函数的作用就是通过传递参数来定制个性化的 日志实例，精细的控制日志行为来满足需要
#比如多久日志清空一次，日志的路径等等
#默认参数包括日志的名称和路径，是否启用文件日志，日志标志符等
#当然，是否可以通过自定义 logging.Logger 类,来实现同样的功能，测试一下看看
#记得还有 装饰器 的用法来着

import logging
from logging import getLogger
from logging import FileHandler
from logging import StreamHandler
from logging import Formatter

#code for log
#Using:     logger.info( 'Some information' ) to replace print( 'Some information' )
logger = getLogger( 'SF:' )
logger.setLevel( logging.DEBUG)
#logging.basicConfig( level = logging.INFO )

#LOG_FILE = 'Log_test.log'

fh = FileHandler( 'Log-test.log' )
fh.setLevel( logging.INFO )

ch = StreamHandler()
#ch.setLevel( logging.WARN)
ch.setLevel( logging.DEBUG)

formatter = Formatter( '%(asctime)s - %(name)s - %(levelname)s:\n%(message)s\n' )
formatter_ch = Formatter( '%(name)s:     %(message)s' )
fh.setFormatter( formatter )
ch.setFormatter( formatter_ch )

logger.addHandler(fh)
logger.addHandler(ch)

"""
The code below is a test for module logging.
"""
tts = '0'
ttn = int(tts)
logger.info( 'ttn = %d ' % ttn )
logger.debug( 'Debug:ttn = %d ' % ttn )
ldebug = logger.debug
linfo = logger.info
ldebug( 'LDB\n\n rename loggger.debug to ldebug!!' )
try:
    print( 10 / ttn )
except Exception as e:
    ldebug( e )


# -*- coding: utf-8 -*-
import feedparser
import urllib
import json
import pprint

s = "国内"
s_quote = urllib.parse.quote(s)
url = "https://news.google.com/news/rss/search/section/q/" + s_quote + "/" + s_quote + "?ned=jp&hl=ja&gl=JP"
d = feedparser.parse(url)
news = list()

for i, entry in enumerate(d.entries, 1):

    p = entry.published_parsed
    sortkey = "%04d%02d%02d%02d%02d%02d" % (p.tm_year, p.tm_mon, p.tm_mday, p.tm_hour, p.tm_min, p.tm_sec)

    tmp = {
        "no": i,
        "title": entry.title,
        "summary": entry.summary,
        "link": entry.link,
        "published": entry.published,
        "sortkey": sortkey
    }

    news.append(tmp)

pprint.pprint(news[0:1])


import csv
import argparse
from dicomanonymizer import *
import hashlib
import uuid
import os

# ''.join(['%x' % a for a in os.urandom(16)]).encode('utf-8')
SALT = b'9693a0733ae7d13df59d4c5cbb123a45'

def bytes_to_digits(b,l):
    num = int.from_bytes(b,'big')
    out_str = ""
    for i in range(0, l):
        (num, r) = divmod(num, 10)
        if i == 0 and r == 0:
            r = 1
        out_str += str(r)
    return out_str

def bytes_to_uid(b):
    return '2.25.'+str(uuid.UUID(bytes=b[:16], version=4).int)

def hash_it_str(s):
    return bytes_to_digits(hashlib.sha1(SALT+s.encode('utf-8')).digest(),9)

def hash_it(dataset, tag):
    element = dataset.get(tag)
    if element is not None:
        element.value = bytes_to_digits(hashlib.sha1(SALT+element.value.encode('utf-8')).digest(),9)

def mrn_hash_it(dataset, tag):
    element = dataset.get(tag)
    if element is not None:
        element.value = 'PH'+bytes_to_digits(hashlib.sha1(SALT+element.value.encode('utf-8')).digest(),9)

def hash_uid(dataset, tag):
    element = dataset.get(tag)
    if element is not None:
        hash_bytes = hashlib.sha1(SALT+element.value.encode('utf-8')).digest()
        element.value = bytes_to_uid(hash_bytes)

rules = {}
for tag in U_TAGS:
    rules[tag] = hash_uid

rules[(0x0008, 0x0050)] = hash_it
rules[(0x0010, 0x0020)] = mrn_hash_it


from commutative_ring import verify_ring_properties
from lib.test_helpers import assert_equal, assert_str, run_test
from math_helper import MathHelper
from elephant import Elephant
from poly import SingleVarPoly
from poly_elephant import ElephantIntegerPoly

@run_test
def test_basics():
    x = ElephantIntegerPoly.x
    assert_equal(x.type_string, "SingleVarPoly.Elephant")

    two = ElephantIntegerPoly.const(2)
    three = ElephantIntegerPoly.const(3)
    p = (x + two) * (x + three)
    assert_str(p, "x**2+(5)*x+6")
    n = p.eval(Elephant.make(4))
    assert n.val == 42
    assert_str(
        n.history,
        "(((0 + ((2 * 3) * (4)**0)) + (((2 * 1) + (1 * 3)) * (4)**1)) + ((1 * 1) * (4)**2))",
    )

@run_test
def test_axioms():
    x = ElephantIntegerPoly.x
    one = ElephantIntegerPoly.one
    two = ElephantIntegerPoly.const(2)
    three = ElephantIntegerPoly.const(3)
    samples = [
        (x + two) ** 3,
        (x + one),
        three,
        two * three,
    ]

    math = MathHelper(
        value_type=SingleVarPoly,
        zero=ElephantIntegerPoly.zero,
        one=ElephantIntegerPoly.one,
    )
    verify_ring_properties(math, samples)


# Simple Linear Regression

# Importing the libraries
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
import os
# Importing the dataset
# changed the namde

import sys, os

# obtain real path
pathname = os.path.dirname(sys.argv[0])

dataset = pd.read_csv(os.path.abspath(pathname) + '\Salary_Data.csv')

X = dataset.iloc[:, :-1].values
y = dataset.iloc[:, 1].values

# Splitting the dataset into the Training set and Test set
# 1/3 of the set goes to test .. means 10 obsevations
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=1/3, random_state=0)

# Feature Scaling
"""from sklearn.preprocessing import StandardScaler
sc_X = StandardScaler()
X_train = sc_X.fit_transform(X_train)
X_test = sc_X.transform(X_test)
sc_y = StandardScaler()
y_train = sc_y.fit_transform(y_train)"""

# Fitting Simple Linear Regression to the Training set
regressor = LinearRegression()
regressor.fit(X_train, y_train)

# Predicting the Test set results
y_pred = regressor.predict(X_test)

# Visualising the Training set results
plt.scatter(X_train, y_train, color = 'red')
plt.plot(X_train, regressor.predict(X_train), color = 'blue')
plt.title('Salary vs Experience (Training set)')
plt.xlabel('Years of Experience')
plt.ylabel('Salary')
plt.show()

# Visualising the Test set results
plt.scatter(X_test, y_test, color = 'red')
# do not use the X_test because it is going to be the same as the result of the prediction
plt.plot(X_train, regressor.predict(X_train), color = 'blue')
plt.title('Salary vs Experience (Test set)')
plt.xlabel('Years of Experience')
plt.ylabel('Salary')
plt.show()

Part 2 - Regression/Section 4 - Simple Linear Regression/simple_linear_regression.py


import argparse
import sys
import tempfile
import uuid

from typing import Iterator, List, Optional, Tuple

import torch
from pyre_extensions import none_throws
from torch import distributed as dist
from torch.distributed import launcher
from torch.distributed.optim.apply_optimizer_in_backward import (
    _apply_optimizer_in_backward,
)
from torch.utils.data import DataLoader
from torcheval.metrics.classification.auroc import BinaryAUROC
from torcheval.metrics.toolkit import sync_and_compute
from torchrec import EmbeddingBagCollection
from torchrec.datasets.criteo import DEFAULT_CAT_NAMES, DEFAULT_INT_NAMES
from torchrec.datasets.random import RandomRecDataset
from torchrec.distributed import TrainPipelineSparseDist
from torchrec.distributed.comm import get_local_size
from torchrec.distributed.model_parallel import (
    DistributedModelParallel,
    get_default_sharders,
)
from torchrec.distributed.planner import EmbeddingShardingPlanner, Topology
from torchrec.distributed.planner.storage_reservations import (
    HeuristicalStorageReservation,
)
from torchrec.models.dlrm import DLRM, DLRMTrain
from torchrec.modules.embedding_configs import EmbeddingBagConfig
from torchrec.optim.keyed import KeyedOptimizerWrapper
from torchrec.optim.optimizers import in_backward_optimizer_filter
from torchtnt.framework.callbacks import TQDMProgressBar
from torchtnt.framework.fit import fit
from torchtnt.framework.state import State

from torchtnt.framework.unit import EvalUnit, TrainUnit
from torchtnt.utils import (
    get_process_group_backend_from_device,
    init_from_env,
    rank_zero_print,
)
from torchtnt.utils.loggers.tensorboard import TensorBoardLogger

MIN_NODES = 1
MAX_NODES = 1
PROC_PER_NODE = 1

def init_dataloader(
    batch_size: int,
    num_batches: int,
    num_embeddings: int,
    backend: str,
    num_embeddings_per_feature: Optional[List[int]] = None,
    seed: Optional[int] = None,
    pin_memory: Optional[bool] = None,
) -> DataLoader:

    pin_memory = (backend == "nccl") if pin_memory is None else pin_memory
    return DataLoader(
        RandomRecDataset(
            keys=DEFAULT_CAT_NAMES,
            batch_size=batch_size,
            hash_size=num_embeddings,
            hash_sizes=num_embeddings_per_feature,
            manual_seed=seed,
            ids_per_feature=1,
            num_dense=len(DEFAULT_INT_NAMES),
            num_batches=num_batches,
        ),
        batch_size=None,
        batch_sampler=None,
        pin_memory=pin_memory,
        num_workers=0,
    )

def parse_args(argv: List[str]) -> argparse.Namespace:
    parser = argparse.ArgumentParser(description="torchrec dlrm example trainer")
    parser.add_argument(
        "--epochs",
        type=int,
        default=1,
        help="number of epochs to train",
    )
    parser.add_argument(
        "--batch_size",
        type=int,
        default=32,
        help="batch size to use for training",
    )
    parser.add_argument(
        "--num_batches",
        type=int,
        default=10,
        help="batch size to use for training",
    )
    parser.add_argument(
        "--test_batch_size",
        type=int,
        default=10,
        help="batch size to use for validation and testing",
    )
    parser.add_argument(
        "--num_embeddings",
        type=int,
        default=100_000,
        help="max_ind_size. The number of embeddings in each embedding table. Defaults"
        " to 100_000 if num_embeddings_per_feature is not supplied.",
    )
    parser.add_argument(
        "--num_embeddings_per_feature",
        type=str,
        default=None,
        help="Comma separated max_ind_size per sparse feature. The number of embeddings"
        " in each embedding table. 26 values are expected for the Criteo dataset.",
    )
    parser.add_argument(
        "--dense_arch_layer_sizes",
        type=str,
        default="512,256,64",
        help="Comma separated layer sizes for dense arch.",
    )
    parser.add_argument(
        "--over_arch_layer_sizes",
        type=str,
        default="512,512,256,1",
        help="Comma separated layer sizes for over arch.",
    )
    parser.add_argument(
        "--embedding_dim",
        type=int,
        default=64,
        help="Size of each embedding.",
    )
    parser.add_argument(
        "--pin_memory",
        dest="pin_memory",
        action="store_true",
        help="Use pinned memory when loading data.",
    )
    parser.add_argument(
        "--seed",
        type=int,
        default=None,
        help="seed",
    )
    parser.add_argument(
        "--learning_rate",
        type=float,
        default=0.1,
        help="Learning rate.",
    )
    parser.add_argument(
        "--log_every_n_steps",
        type=int,
        default=10,
        help="log every n steps",
    )
    parser.add_argument("--lr_warmup_steps", type=int, default=0)
    parser.add_argument("--lr_decay_start", type=int, default=0)
    parser.add_argument("--lr_decay_steps", type=int, default=0)

    parser.set_defaults(
        pin_memory=None,
    )
    return parser.parse_args(argv)

Batch = Tuple[torch.Tensor, torch.Tensor]

class MyUnit(TrainUnit[Iterator[Batch]], EvalUnit[Iterator[Batch]]):
    def __init__(
        self,
        module: torch.nn.Module,
        optimizer: torch.optim.Optimizer,
        device: torch.device,
        tb_logger: TensorBoardLogger,
        train_auroc: BinaryAUROC,
        log_every_n_steps: int,
    ) -> None:
        super().__init__()
        self.module = module
        self.pipeline: TrainPipelineSparseDist = TrainPipelineSparseDist(
            module, optimizer, device, execute_all_batches=True
        )
        self.optimizer = optimizer
        self.device = device
        self.train_auroc = train_auroc
        self.tb_logger = tb_logger
        self.log_every_n_steps = log_every_n_steps

    def train_step(self, state: State, data: Iterator[Batch]) -> None:
        step = self.train_progress.num_steps_completed
        loss, logits, labels = self.pipeline.progress(data)
        preds = torch.sigmoid(logits)
        self.train_auroc.update(preds, labels)
        if step % self.log_every_n_steps == 0:
            accuracy = sync_and_compute(self.train_auroc)
            self.tb_logger.log("train_auroc", accuracy, step)
            self.tb_logger.log("loss", loss, step)

    def on_train_epoch_end(self, state: State) -> None:
        super().on_train_epoch_end(state)
        # reset the metric every epoch
        self.train_auroc.reset()

    def eval_step(self, state: State, data: Iterator[Batch]) -> None:
        step = self.eval_progress.num_steps_completed
        loss, _, _ = self.pipeline.progress(data)
        if step % self.log_every_n_steps == 0:
            self.tb_logger.log("evaluation_loss", loss, step)

def init_embdedding_configs(
    embedding_dim: int,
    num_embeddings_per_feature: Optional[List[int]],
    num_embeddings: int,
) -> List[EmbeddingBagConfig]:

    eb_configs = [
        EmbeddingBagConfig(
            name=f"t_{feature_name}",
            embedding_dim=embedding_dim,
            num_embeddings=none_throws(num_embeddings_per_feature)[feature_idx]
            if num_embeddings is None
            else num_embeddings,
            feature_names=[feature_name],
        )
        for feature_idx, feature_name in enumerate(DEFAULT_CAT_NAMES)
    ]
    return eb_configs

def init_model(
    eb_configs: List[EmbeddingBagConfig],
    dense_arch_layer_sizes: Optional[List[int]],
    over_arch_layer_sizes: Optional[List[int]],
    learning_rate: float,
    batch_size: int,
    device: torch.device,
) -> torch.nn.Module:

    dlrm_model = DLRM(
        embedding_bag_collection=EmbeddingBagCollection(
            tables=eb_configs, device=torch.device("meta")
        ),
        dense_in_features=len(DEFAULT_INT_NAMES),
        dense_arch_layer_sizes=dense_arch_layer_sizes or [],
        over_arch_layer_sizes=over_arch_layer_sizes or [],
        dense_device=device,
    )

    train_model = DLRMTrain(dlrm_model)

    _apply_optimizer_in_backward(
        torch.optim.SGD,
        train_model.model.sparse_arch.parameters(),
        {"lr": learning_rate},
    )

    planner = EmbeddingShardingPlanner(
        topology=Topology(
            local_world_size=get_local_size(),
            world_size=dist.get_world_size(),
            compute_device=device.type,
        ),
        batch_size=batch_size,
        storage_reservation=HeuristicalStorageReservation(percentage=0.05),
    )
    plan = planner.collective_plan(
        train_model, get_default_sharders(), dist.GroupMember.WORLD
    )

    model = DistributedModelParallel(
        module=train_model,
        device=device,
        plan=plan,
    )

    return model

def init_optimizer(
    model: torch.nn.Module, learning_rate: float
) -> torch.optim.Optimizer:
    optimizer = KeyedOptimizerWrapper(
        dict(in_backward_optimizer_filter(model.named_parameters())),
        lambda params: torch.optim.SGD(params, lr=learning_rate),
    )
    return optimizer

def init_logger() -> TensorBoardLogger:
    path = tempfile.mkdtemp()
    tb_logger = TensorBoardLogger(path)
    return tb_logger

def main(argv: List[str]) -> None:
    args = parse_args(argv)
    for name, val in vars(args).items():
        try:
            vars(args)[name] = list(map(int, val.split(",")))
        except (ValueError, AttributeError):
            pass

    rank_zero_print(
        f"PARAMS: (lr, batch_size, warmup_steps, decay_start, decay_steps): {(args.learning_rate, args.batch_size, args.lr_warmup_steps, args.lr_decay_start, args.lr_decay_steps)}"
    )
    device = init_from_env()
    backend = get_process_group_backend_from_device(device)

    eb_configs = init_embdedding_configs(
        args.embedding_dim, args.num_embeddings_per_feature, args.num_embeddings
    )

    model = init_model(
        eb_configs,
        args.dense_arch_layer_sizes,
        args.over_arch_layer_sizes,
        args.learning_rate,
        args.batch_size,
        device,
    )

    optimizer = init_optimizer(model, args.learning_rate)
    tb_logger = init_logger()
    auroc = BinaryAUROC(device=device)

    my_unit = MyUnit(
        module=model,
        optimizer=optimizer,
        device=device,
        tb_logger=tb_logger,
        train_auroc=auroc,
        log_every_n_steps=10,
    )

    train_dataloader = init_dataloader(
        args.batch_size,
        args.num_batches,
        args.num_embeddings,
        backend,
        args.num_embeddings_per_feature,
        args.seed,
        args.pin_memory,
    )
    eval_dataloader = init_dataloader(
        args.test_batch_size,
        args.num_batches,
        args.num_embeddings,
        backend,
        args.num_embeddings_per_feature,
        args.seed,
        args.pin_memory,
    )

    tqdm_callback = TQDMProgressBar()
    fit(
        my_unit,
        train_dataloader=train_dataloader,
        eval_dataloader=eval_dataloader,
        max_epochs=args.epochs,
        callbacks=[tqdm_callback],
    )

if __name__ == "__main__":
    lc = launcher.LaunchConfig(
        min_nodes=MIN_NODES,
        max_nodes=MAX_NODES,
        nproc_per_node=PROC_PER_NODE,
        run_id=str(uuid.uuid4()),
        rdzv_backend="c10d",
        rdzv_endpoint="localhost:0",
        max_restarts=0,
        monitor_interval=1,
    )

    launcher.elastic_launch(config=lc, entrypoint=main)(sys.argv[1:])


import os
import psutil
import sys
import shutil

def dupl(file):
    if os.path.isfile(file):
        newfile = file + '.dupl'
        shutil.copy(file, newfile)
        if os.path.exists(newfile):
            print("Файл ", newfile, " был успешно создан")
            return True
        else:
            print("Есть какие-то проблемы копирования")
            return False

def sysinfo():
    print('File system coding:', sys.getfilesystemencoding())
    print('Platform:', sys.platform)
    print('CPU number:', os.cpu_count())
    print('Login:', os.getlogin())
    print('Current directory:', os.getcwd())

def delete(file_list):
    success = 0
    for f in file_list:
            fullname = os.path.join(dirname, f)
            if fullname.endswith('.dupl'):
                os.remove(fullname)
                if not os.path.exists(fullname):
                    success += 1
                    print("Файл ", fullname, " был успешно удален")
    return success

name = input("Ваше имя? ")
print("hello, ", name)

work = ''
while work != 'q':
    work = input("Давайте поработаем? (Y/N/q) ")
    if work == 'y':
        print("Let`s go! Я могу сделать: ")
        print("[1] Выведу список файлов ")
        print("[2] Выведу информацию о системе")
        print("[3] Выведу все процессы")
        print("[4] Бэкап данных директории")
        print("[5] Бэкап выбранного файла")
        print("[6] Удаление выбранного скопированного файла")
        do = int(input("Укажите, что для Вас сделать: "))

        if do == 1:
            print(os.listdir())
        elif do == 2:
            sysinfo()

        elif do == 3:
            print(psutil.pids())
        elif do == 4:
            file_list = os.listdir()
            i = 0
            while i < len(file_list):
                dupl(file_list[i])
                i += 1
            print()
        elif do == 5:
            file = input('Укажите имя файла')
            dupl(file)
        elif do == 6:
            dirname = input('Укажите имя директории')
            file_list = os.listdir(dirname)
            count = delete(file_list)
            print("Было удалено:", count)

        else:
            print('exiting')
    elif work == 'n':
        print("Жаль!")
    else:
        print("dont right")


from datetime import datetime

from telebot.types import Message
from telegram_bot_calendar import LSTEP, DetailedTelegramCalendar

from database.db import Database
from keyboards.reply import reply_markups
from loader import bot
from states.lowprice_states import LowPriceStates
from utils.multilingual import multilingual
from .base import any_state
from .user_data import users_dict

db = Database('database/database.db')

@bot.message_handler(func=lambda message: message.text in ["🧐 Поиск отелей по цене/расположению от центра",
                                                           "🧐 Search for hotels by price/location from the center",
                                                           "🧐 Suche nach Hotels nach Preis/Lage vom Zentrum",
                                                           "🧐 Trouver des Hôtels par prix/emplacement du centre",
                                                           "🧐 Cerca Hotel per Prezzo/posizione dal centro",
                                                           "🧐 Buscar hoteles por precio/ubicación desde el centro"])
@bot.message_handler(commands=['bestdeal'])
def bestdeal_start(message: Message) -> None:
    """
        Функция для запуска сценария поиска отелей, наиболее подходящих по цене и расположению от центра.
    Args:
        :return: None
    """
    locale = db.select_locale_user(message.from_user.id)[0]
    if message.from_user.id in users_dict:
        if users_dict[message.from_user.id].hotel_message is not None:
            bot.delete_message(message.from_user.id, users_dict[message.from_user.id].hotel_message)
            users_dict[message.from_user.id].hotel_message = None
    bot.delete_state(message.from_user.id, message.chat.id)
    text = multilingual(locale, 'base_start')
    bot.send_message(message.from_user.id, text, reply_markup=reply_markups.cancel_markup(locale))
    bot.set_state(message.from_user.id, LowPriceStates.city_data, message.chat.id)
    with bot.retrieve_data(message.from_user.id, message.chat.id) as data:
        data['sort_order'] = 'DISTANCE_FROM_LANDMARK'
        data['command'] = '/bestdeal'

@bot.message_handler(state=LowPriceStates.price_range)
def get_price_range(message: Message) -> None:
    """
        Функция для получения от пользователя максимальной и минимальной цены, и записи в память.
    Args:
        :return: None
    """
    locale = db.select_locale_user(message.from_user.id)[0]
    if message.text.isdigit():
        with bot.retrieve_data(message.from_user.id, message.chat.id) as data:
            if 'priceMin' not in data.keys():
                data['priceMin'] = message.text
                max_price_in_data = False
            else:
                data['priceMax'] = message.text
                max_price_in_data = True
        print(message.from_user.id)

        if not max_price_in_data:
            locale_currency_tuple = db.select_lang_currency_user(message.from_user.id)
            currency = locale_currency_tuple[1]
            text = multilingual(locale, 'get_price_range_price_min', currency)
            bot.send_message(message.from_user.id, text)
            bot.set_state(message.from_user.id, LowPriceStates.price_range, message.chat.id)
        else:
            if locale == 'en_US':
                distance_unit_of_measurement = 'ml'
            else:
                distance_unit_of_measurement = 'km'
            text = multilingual(locale, 'get_price_range_max_distance', distance_unit_of_measurement)
            bot.send_message(message.from_user.id, text)
            bot.set_state(message.from_user.id, LowPriceStates.distance_range, message.chat.id)
    else:
        if message.text != 'Отмена':
            bot.set_state(message.from_user.id, LowPriceStates.price_range, message.chat.id)
            text = multilingual(locale, 'get_price_range_valid_error')
            bot.send_message(message.from_user.id, text)
        else:
            any_state(message)

@bot.message_handler(state=LowPriceStates.distance_range)
def get_distance_range(message: Message) -> None:
    """
        Функция для получения от пользователя максимального расстояния до центра, и записи в память.
    Args:
        :return: None
    """
    locale = db.select_locale_user(message.from_user.id)[0]
    if message.text.replace(".", "", 1).isdigit():
        with bot.retrieve_data(message.from_user.id, message.chat.id) as data:
                data['max_distance'] = message.text

        bot.set_state(message.from_user.id, LowPriceStates.distance_range, message.chat.id)
        calendar, step = DetailedTelegramCalendar(min_date=datetime.now().date(), locale='ru').build()
        text = multilingual(locale, 'calendar_check_in', LSTEP[step])
        bot.send_message(message.from_user.id, text, reply_markup=calendar)
    else:
        text = multilingual(locale, 'get_distance_range_value_error')
        bot.send_message(message.from_user.id, text)


import sys
from collections import deque

class MinStack:

    stack = deque();
    minValue = sys.maxsize;
    def __init__(self):
        """
        initialize your data structure here.
        """
        self.stack = deque()
        self.minValue = sys.maxsize;

    def push(self, x: int) -> None:
        self.stack.append(x)
        self.minValue = min(x, self.minValue)

    def pop(self) -> None:
        popValue = self.stack.pop()
        # print(self.stack, popValue)
        if popValue != self.minValue:
          return popValue

        self.minValue = sys.maxsize
        tempStack = deque()

        while len(self.stack) > 0:
          value = self.stack.pop()
          self.minValue = min(value, self.minValue)
          tempStack.append(value)

        while len(tempStack) > 0:
          self.stack.append(tempStack.pop())
        return popValue

    def top(self) -> int:
        return self.stack[-1]

    def getMin(self) -> int:
        return self.minValue

# Your MinStack object will be instantiated and called as such:
# obj = MinStack()
# obj.push(x)
# obj.pop()
# param_3 = obj.top()
# param_4 = obj.getMin()

mySolutions/155. Min Stack/155. Min Stack.py