import os

from django.core.exceptions import ObjectDoesNotExist

from mezzanine.pages.page_processors import processor_for

from hs_core.models import BaseResource, ResourceManager, ResourceFile, resource_processor

from hs_file_types.models import GenericLogicalFile, GeoFeatureLogicalFile, GeoRasterLogicalFile, \
    NetCDFLogicalFile, TimeSeriesLogicalFile, FileSetLogicalFile
from hs_file_types.utils import update_target_temporal_coverage, update_target_spatial_coverage

class CompositeResource(BaseResource):
    objects = ResourceManager("CompositeResource")

    discovery_content_type = 'Composite'  # used during discovery

    class Meta:
        verbose_name = 'Composite Resource'
        proxy = True

    @property
    def can_be_public_or_discoverable(self):
        # resource level metadata check
        if not super(CompositeResource, self).can_be_public_or_discoverable:
            return False

        # filetype level metadata check
        for lf in self.logical_files:
            if not lf.metadata.has_all_required_elements():
                return False

        return True

    @property
    def logical_files(self):
        """Returns a list of all logical file type objects associated with this resource """

        lf_list = []
        lf_list.extend(self.filesetlogicalfile_set.all())
        lf_list.extend(self.genericlogicalfile_set.all())
        lf_list.extend(self.geofeaturelogicalfile_set.all())
        lf_list.extend(self.netcdflogicalfile_set.all())
        lf_list.extend(self.georasterlogicalfile_set.all())
        lf_list.extend(self.reftimeserieslogicalfile_set.all())
        lf_list.extend(self.timeserieslogicalfile_set.all())

        return lf_list

    @property
    def can_be_published(self):
        # resource level metadata check
        if not super(CompositeResource, self).can_be_published:
            return False

        # filetype level metadata check
        for lf in self.logical_files:
            if not lf.metadata.has_all_required_elements():
                return False
            # url file cannot be published
            if 'url' in lf.extra_data:
                return False
        return True

    def set_default_logical_file(self):
        """sets an instance of GenericLogicalFile to any resource file objects of this instance
        of the resource that is not already associated with a logical file. """

        for res_file in self.files.all():
            if not res_file.has_logical_file:
                logical_file = GenericLogicalFile.create()
                res_file.logical_file_content_object = logical_file
                res_file.save()

    def get_folder_aggregation_object(self, dir_path):
        """Returns an aggregation (file type) object if the specified folder *dir_path* represents a
         file type aggregation (logical file), otherwise None.

         :param dir_path: Resource file directory path (full folder path starting with resource id)
         for which the aggregation object to be retrieved
        """

        aggregation_path = dir_path[len(self.file_path) + 1:]
        try:
            return self.get_aggregation_by_name(aggregation_path)
        except ObjectDoesNotExist:
            return None

    def get_file_aggregation_object(self, file_path):
        """Returns an aggregation (file type) object if the specified file *file_path* represents a
         file type aggregation (logical file), otherwise None.

         :param file_path: Resource file path (full file path starting with resource id)
         for which the aggregation object to be retrieved
        """
        for res_file in self.files.all():
            if res_file.full_path == file_path:
                if res_file.has_logical_file:
                    return res_file.logical_file
        return None

    def get_folder_aggregation_type_to_set(self, dir_path):
        """Returns an aggregation (file type) type that the specified folder *dir_path* can
        possibly be set to.

        :param dir_path: Resource file directory path (full folder path starting with resource id)
        for which the possible aggregation type that can be set needs to be determined

        :return If the specified folder is already represents an aggregation or does
        not contain suitable file(s) then returns "" (empty string). If the specified folder
        contains only the files that meet the requirements of a supported aggregation, and
        does not contain other folders or does not have a parent folder then return the
        class name of that matching aggregation type.
        """

        if self.get_folder_aggregation_object(dir_path) is not None:
            # target folder is already an aggregation
            return None

        istorage = self.get_irods_storage()
        irods_path = dir_path
        if self.is_federated:
            irods_path = os.path.join(self.resource_federation_path, irods_path)
        store = istorage.listdir(irods_path)
        files_in_folder = ResourceFile.list_folder(self, folder=irods_path, sub_folders=False)

        if not files_in_folder:
            # folder is empty
            # check sub folders for files - if file exist we can set FileSet aggregation
            files_in_sub_folders = ResourceFile.list_folder(self, folder=irods_path,
                                                            sub_folders=True)
            if files_in_sub_folders:
                return FileSetLogicalFile.__name__

            return None
        if store[0]:
            # there are folders under dir_path as well as files - only FileSet can bet set
            return FileSetLogicalFile.__name__

        if len(files_in_folder) > 1:
            # check for geo feature
            aggregation_type_to_set = GeoFeatureLogicalFile.check_files_for_aggregation_type(
                files_in_folder)
            if aggregation_type_to_set:
                return aggregation_type_to_set

            # check for raster
            aggregation_type_to_set = GeoRasterLogicalFile.check_files_for_aggregation_type(
                files_in_folder)
            if aggregation_type_to_set:
                return aggregation_type_to_set
            return FileSetLogicalFile.__name__
        else:
            # check for raster
            aggregation_type_to_set = GeoRasterLogicalFile.check_files_for_aggregation_type(
                files_in_folder)
            if aggregation_type_to_set:
                return aggregation_type_to_set
            # check for NetCDF aggregation type
            aggregation_type_to_set = NetCDFLogicalFile.check_files_for_aggregation_type(
                files_in_folder)
            if aggregation_type_to_set:
                return aggregation_type_to_set
            # check for TimeSeries aggregation type
            aggregation_type_to_set = TimeSeriesLogicalFile.check_files_for_aggregation_type(
                files_in_folder)
            if aggregation_type_to_set:
                return aggregation_type_to_set
            return FileSetLogicalFile.__name__

    @property
    def supports_folders(self):
        """ allow folders for CompositeResources """
        return True

    @property
    def supports_logical_file(self):
        """ if this resource allows associating resource file objects with logical file"""
        return True

    def get_metadata_xml(self, pretty_print=True, include_format_elements=True):
        from lxml import etree

        # get resource level core metadata as xml string
        # for composite resource we don't want the format elements at the resource level
        # as they are included at the aggregation map xml document
        xml_string = super(CompositeResource, self).get_metadata_xml(pretty_print=False,
                                                                     include_format_elements=False)

        # create an etree xml object
        RDF_ROOT = etree.fromstring(xml_string)

        return etree.tostring(RDF_ROOT, pretty_print=pretty_print)

    def create_aggregation_xml_documents(self, aggregation_name=None):
        """Creates aggregation map and metadata xml files for each of the contained aggregations

        :param  aggregation_name: (optional) name of the the specific aggregation for which xml
        documents need to be created
        """

        if aggregation_name is None:
            for aggregation in self.logical_files:
                if aggregation.metadata.is_dirty:
                    aggregation.create_aggregation_xml_documents()
        else:
            try:
                aggregation = self.get_aggregation_by_name(aggregation_name)
                if aggregation.metadata.is_dirty:
                    aggregation.create_aggregation_xml_documents()
            except ObjectDoesNotExist:
                # aggregation_name must be a folder path that doesn't represent an aggregation
                # there may be single file aggregation in that folder for which xml documents
                # need to be created
                self._recreate_xml_docs_for_folder(aggregation_name, check_metadata_dirty=True)

    def _recreate_xml_docs_for_folder(self, folder, check_metadata_dirty=False):
        """Re-creates xml metadata and map documents associated with the specified folder.
        If the *folder* represents an aggregation then map and metadata xml documents are
        recreated only for that aggregation. Otherwise, xml documents are created for any
        aggregation that may exist in the specified folder and its sub-folders.

        :param  folder: folder for which xml documents need to be re-created
        :param  check_metadata_dirty: if true, then xml files will be created only if the
        aggregation metadata is dirty
        """

        # first check if the the folder represents an aggregation
        try:
            aggregation = self.get_aggregation_by_name(folder)
            if check_metadata_dirty:
                if aggregation.metadata.is_dirty:
                    aggregation.create_aggregation_xml_documents()
            else:
                aggregation.create_aggregation_xml_documents()
                # if we found an aggregation by the folder name that means this folder doesn't
                # have any sub folders as multi-file aggregation folder can't have sub folders
        except ObjectDoesNotExist:
            # create xml map and metadata xml documents for all aggregations that exist
            # in *folder* and its sub-folders
            if not folder.startswith(self.file_path):
                folder = os.path.join(self.file_path, folder)

            res_file_objects = ResourceFile.list_folder(self, folder)
            aggregations = []
            for res_file in res_file_objects:
                if res_file.has_logical_file and res_file.logical_file not in aggregations:
                    aggregations.append(res_file.logical_file)

            if check_metadata_dirty:
                aggregations = [aggr for aggr in aggregations if aggr.metadata.is_dirty]
            for aggregation in aggregations:
                aggregation.create_aggregation_xml_documents()

    def get_aggregation_by_name(self, name):
        """Get an aggregation that matches the aggregation name specified by *name*
        :param  name: name (aggregation path) of the aggregation to find
        :return an aggregation object if found
        :raises ObjectDoesNotExist if no matching aggregation is found
        """
        for aggregation in self.logical_files:
            # remove the last slash in aggregation_name if any
            if aggregation.aggregation_name.rstrip('/') == name:
                return aggregation

        raise ObjectDoesNotExist("No matching aggregation was found for name:{}".format(name))

    def get_fileset_aggregation_in_path(self, path):
        """Get the first fileset aggregation in the path moving up (towards the root)in the path
        :param  path: directory path in which to search for a fileset aggregation
        :return a fileset aggregation object if found, otherwise None
        """

        def get_fileset(path):
            try:
                aggregation = self.get_aggregation_by_name(path)
                if aggregation.is_fileset:
                    return aggregation
            except ObjectDoesNotExist:
                return None

        while '/' in path:
            fileset = get_fileset(path)
            if fileset is not None:
                return fileset
            path = os.path.dirname(path)
        else:
            return get_fileset(path)

    def recreate_aggregation_xml_docs(self, orig_aggr_name, new_aggr_name):
        """
        When a folder or file representing an aggregation is renamed or moved,
        the associated map and metadata xml documents are deleted
        and then regenerated
        :param  orig_aggr_name: original aggregation name - used for deleting existing
        xml documents
        :param  new_aggr_name: new aggregation name - used for finding a matching
        aggregation so that new xml documents can be recreated
        """

        def delete_old_xml_files(folder=''):
            istorage = self.get_irods_storage()
            meta_xml_file_name = orig_aggr_name + "_meta.xml"
            map_xml_file_name = orig_aggr_name + "_resmap.xml"
            if not folder:
                # case if file rename/move for single file aggregation
                meta_xml_file_full_path = os.path.join(self.file_path, meta_xml_file_name)
                map_xml_file_full_path = os.path.join(self.file_path, map_xml_file_name)
            else:
                # case of folder rename - multi-file aggregation
                _, meta_xml_file_name = os.path.split(meta_xml_file_name)
                _, map_xml_file_name = os.path.split(map_xml_file_name)
                meta_xml_file_full_path = os.path.join(self.file_path, folder, meta_xml_file_name)
                map_xml_file_full_path = os.path.join(self.file_path, folder, map_xml_file_name)

            if istorage.exists(meta_xml_file_full_path):
                istorage.delete(meta_xml_file_full_path)

            if istorage.exists(map_xml_file_full_path):
                istorage.delete(map_xml_file_full_path)

        # first check if the new_aggr_name is a folder path or file path
        name, ext = os.path.splitext(new_aggr_name)
        is_new_aggr_a_folder = ext == ''

        if is_new_aggr_a_folder:
            delete_old_xml_files(folder=new_aggr_name)
            try:
                # in case of fileset aggregation need to update aggregation folder attribute to the
                # new folder name
                aggregation = self.get_aggregation_by_name(orig_aggr_name)
                if aggregation.is_fileset:
                    # update folder attribute of this fileset aggregation and all nested
                    # fileset aggregations of this aggregation
                    aggregation.update_folder(new_folder=new_aggr_name)
            except ObjectDoesNotExist:
                # not renaming a fileset aggregation folder
                pass
            self._recreate_xml_docs_for_folder(new_aggr_name)
        else:
            # check if there is a matching single file aggregation
            try:
                aggregation = self.get_aggregation_by_name(new_aggr_name)
                delete_old_xml_files()
                aggregation.create_aggregation_xml_documents()
            except ObjectDoesNotExist:
                # the file path *new_aggr_name* is not a single file aggregation - no more
                # action is needed
                pass

    def is_aggregation_xml_file(self, file_path):
        """ determine whether a given file in the file hierarchy is metadata.

        This is true if it is listed as metadata in any logical file.
        """
        if not (file_path.endswith('_meta.xml') or file_path.endswith('_resmap.xml')):
            return False
        for logical_file in self.logical_files:
            if logical_file.metadata_file_path == file_path or \
                    logical_file.map_file_path == file_path:
                return True
        return False

    def supports_folder_creation(self, folder_full_path):
        """this checks if it is allowed to create a folder at the specified path
        :param  folder_full_path: the target path where the new folder needs to be created

        :return True or False
        """

        if __debug__:
            assert(folder_full_path.startswith(self.file_path))

        # determine containing folder
        if "/" in folder_full_path:
            path_to_check, _ = os.path.split(folder_full_path)
        else:
            path_to_check = folder_full_path

        # find if the path represents a multi-file aggregation
        if path_to_check.startswith(self.file_path):
            aggregation_path = path_to_check[len(self.file_path) + 1:]
        else:
            aggregation_path = path_to_check
        try:
            aggregation = self.get_aggregation_by_name(aggregation_path)
            return aggregation.can_contain_folders
        except ObjectDoesNotExist:
            # target path doesn't represent an aggregation - so it is OK to create a folder
            pass
        return True

    def supports_rename_path(self, src_full_path, tgt_full_path):
        """checks if file/folder rename/move is allowed
        :param  src_full_path: name of the file/folder path to be renamed
        :param  tgt_full_path: new name for file/folder path
        :return True or False
        """

        if __debug__:
            assert(src_full_path.startswith(self.file_path))
            assert(tgt_full_path.startswith(self.file_path))

        istorage = self.get_irods_storage()

        # need to find out which of the following actions the user is trying to do:
        # renaming a file
        # renaming a folder
        # moving a file
        # moving a folder
        is_renaming_file = False
        is_moving_file = False
        is_moving_folder = False

        tgt_folder, tgt_file_name = os.path.split(tgt_full_path)
        _, tgt_ext = os.path.splitext(tgt_file_name)
        if tgt_ext:
            tgt_file_dir = os.path.dirname(tgt_full_path)
        else:
            tgt_file_dir = tgt_full_path

        src_folder, src_file_name = os.path.split(src_full_path)
        _, src_ext = os.path.splitext(src_file_name)
        if src_ext:
            src_file_dir = os.path.dirname(src_full_path)
        else:
            src_file_dir = src_full_path

        if src_ext and tgt_ext:
            is_renaming_file = True
        elif src_ext:
            is_moving_file = True
        elif not istorage.exists(tgt_file_dir):
            # renaming folder - no restriction
            return True
        else:
            is_moving_folder = True

        def check_file_rename_or_move():
            # see if the folder containing the file represents an aggregation
            if src_file_dir != self.file_path:
                aggregation_path = src_file_dir[len(self.file_path) + 1:]
                try:
                    aggregation = self.get_aggregation_by_name(aggregation_path)
                    return aggregation.supports_resource_file_rename
                except ObjectDoesNotExist:
                    # check if the source file represents an aggregation
                    aggregation_path = os.path.join(aggregation_path, src_file_name)
                    aggregation = self.get_aggregation_by_name(aggregation_path)
                    if is_renaming_file:
                        return aggregation.supports_resource_file_rename
                    else:
                        return aggregation.supports_resource_file_move
            else:
                # check if the source file represents an aggregation
                aggregation_path = src_file_name
                aggregation = self.get_aggregation_by_name(aggregation_path)
                if is_renaming_file:
                    return aggregation.supports_resource_file_rename
                else:
                    return aggregation.supports_resource_file_move

        if is_renaming_file:
            # see if the folder containing the file represents an aggregation
            try:
                can_rename = check_file_rename_or_move()
                return can_rename
            except ObjectDoesNotExist:
                return True

        elif is_moving_file:
            # check source - see if the folder containing the file represents an aggregation
            try:
                can_move = check_file_rename_or_move()
                if not can_move:
                    return can_move
            except ObjectDoesNotExist:
                pass

            # check target folder only if it is not the root
            if tgt_file_dir != self.file_path:
                aggregation_path = tgt_file_dir[len(self.file_path) + 1:]
                try:
                    aggregation = self.get_aggregation_by_name(aggregation_path)
                    return aggregation.supports_resource_file_add
                except ObjectDoesNotExist:
                    # target folder is not an aggregation - no restriction
                    return True
            return True
        elif is_moving_folder:
            # no check on source is needed in this case
            # check target - only if it is not the root
            if tgt_file_dir != self.file_path:
                aggregation_path = tgt_file_dir[len(self.file_path) + 1:]
                try:
                    aggregation = self.get_aggregation_by_name(aggregation_path)
                    return aggregation.can_contain_folders
                except ObjectDoesNotExist:
                    # target folder doesn't represent an aggregation - no restriction
                    return True
            return True

    def can_add_files(self, target_full_path):
        """
        checks if file(s) can be uploaded to the specified *target_full_path*
        :param target_full_path: full folder path name where file needs to be uploaded to
        :return: True or False
        """
        istorage = self.get_irods_storage()
        if istorage.exists(target_full_path):
            path_to_check = target_full_path
        else:
            return False

        if not path_to_check.endswith("data/contents"):
            # it is not the base directory - it must be a directory under base dir
            if path_to_check.startswith(self.file_path):
                aggregation_path = path_to_check[len(self.file_path) + 1:]
            else:
                aggregation_path = path_to_check
            try:
                aggregation = self.get_aggregation_by_name(aggregation_path)
                return aggregation.supports_resource_file_add
            except ObjectDoesNotExist:
                # target path doesn't represent an aggregation - so it is OK to add a file
                pass
        return True

    def supports_zip(self, folder_to_zip):
        """check if the given folder can be zipped or not"""

        # find all the resource files in the folder to be zipped
        # this is being passed both qualified and unqualified paths!

        full_path = folder_to_zip
        if not full_path.startswith(self.file_path):
            full_path = os.path.join(self.file_path, full_path)
        # get all resource files at full_path and its sub-folders
        res_file_objects = ResourceFile.list_folder(self, full_path)

        # check any logical file associated with the resource file supports zip functionality
        for res_file in res_file_objects:
            if res_file.has_logical_file:
                if not res_file.logical_file.supports_zip:
                    return False
        return True

    def supports_delete_folder_on_zip(self, original_folder):
        """check if the specified folder can be deleted at the end of zipping that folder"""

        # find all the resource files in the folder to be deleted
        # this is being passed both qualified and unqualified paths!
        full_path = original_folder
        if not full_path.startswith(self.file_path):
            full_path = os.path.join(self.file_path, full_path)

        # get all resource files at full_path and its sub-folders
        res_file_objects = ResourceFile.list_folder(self, full_path)

        # check any logical file associated with the resource file supports deleting the folder
        # after its zipped
        for res_file in res_file_objects:
            if res_file.has_logical_file:
                if not res_file.logical_file.supports_delete_folder_on_zip:
                    return False
        return True

    def get_missing_file_type_metadata_info(self):
        # this is used in page pre-processor to build the context
        # so that the landing page can show what metadata items are missing for each
        # logical file/aggregation
        metadata_missing_info = []
        for lfo in self.logical_files:
            if not lfo.metadata.has_all_required_elements():
                missing_elements = lfo.metadata.get_required_missing_elements()
                metadata_missing_info.append({'file_path': lfo.aggregation_name,
                                              'missing_elements': missing_elements})
        return metadata_missing_info

    def delete_coverage(self, coverage_type):
        """Deletes coverage data for the resource
        :param coverage_type: A value of either 'spatial' or 'temporal
        :return:
        """
        if coverage_type.lower() == 'spatial' and self.metadata.spatial_coverage:
            self.metadata.spatial_coverage.delete()
            self.metadata.is_dirty = True
            self.metadata.save()
        elif coverage_type.lower() == 'temporal' and self.metadata.temporal_coverage:
            self.metadata.temporal_coverage.delete()
            self.metadata.is_dirty = True
            self.metadata.save()

    def update_coverage(self):
        """Update resource spatial and temporal coverage based on the corresponding coverages
        from all the contained aggregations (logical file) only if the resource coverage is not
        already set"""

        # update resource spatial coverage only if there is no spatial coverage already
        if self.metadata.spatial_coverage is None:
            self.update_spatial_coverage()

        # update resource temporal coverage only if there is no temporal coverage already
        if self.metadata.temporal_coverage is None:
            self.update_temporal_coverage()

    def update_spatial_coverage(self):
        """Updates resource spatial coverage based on the contained spatial coverages of
        aggregations (file type). Note: This action will overwrite any existing resource spatial
        coverage data.
        """

        update_target_spatial_coverage(self)

    def update_temporal_coverage(self):
        """Updates resource temporal coverage based on the contained temporal coverages of
        aggregations (file type). Note: This action will overwrite any existing resource temporal
        coverage data.
        """

        update_target_temporal_coverage(self)

# this would allow us to pick up additional form elements for the template before the template
# is displayed
processor_for(CompositeResource)(resource_processor)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
import cherrypy
from cherrypy.lib.static import serve_file

import json

import datetime
import time

from path import path
from urlparse import urlparse
from hashlib import md5

from config import *

class Zoidberg(object):

    @cherrypy.expose
    def index(self):
        '''Index page. Just shows a form and shit'''
        return self.get_html('base.tpl',filecontent='index.tpl')

    @cherrypy.expose
    def wait(self,token):
        ''''''
        return self.get_html('base.tpl',filecontent='wait.tpl',content=token)

    def get_html(self,template,filecontent=None,content=None):
        if not filecontent is None:
            if not content is None :
                fill = open(filecontent).read().format(content)
            else:
                fill = open(filecontent).read()
        elif not content is None:
            fill = content
        else:
            fill = ''

        return open(template).read().format(fill)

    @cherrypy.expose
    def add(self,url):
        '''Adds url to the convert queue if it is a valid url.
        Otherwise returns as 401'''

        o = urlparse(url)
        if not o.netloc in ALLOWED_DOMAINS:
            raise cherrypy.HTTPError(401,'host not supported')

        token = md5(url).hexdigest()
        requestfile = PROCESSING_DIR/(token+'.request')
        requestfile.write_text(url)
        raise cherrypy.HTTPRedirect("/wait?token={}".format(token), 301)

    def token_status(self,token):
        '''Checks the status of the given video url associated with token'''

        statuspath = PROCESSING_DIR/(token+'.status')
        if not statuspath.exists():
            requestpath = PROCESSING_DIR/(token+'.request')
            if not requestpath.exists():
                return 'Unknown'
            else :
                return 'Queued'
        elif statuspath.text().strip() in ('Wip','Error'):
            return statuspath.text().strip()
        else:
            return 'Done'

    def isToken(self,token):
        if not len(token) == 32:
            return False
        if '.' in token or '/' in token:
            return False
        return True

    @cherrypy.expose
    def status(self,token):
        if self.isToken(token):
            status = self.token_status(token)
            if status == 'Unknown':
                raise cherrypy.HTTPError(404,'No corresponding token')
            else:
                return status
        else:
            raise cherrypy.HTTPError(401,'Invalid token')

    @cherrypy.expose
    def download(self,token):
        if not self.isToken(token):
            raise cherrypy.HTTPError(401,'Invalid token')

        filepath = PROCESSING_DIR/(token+'.mp3')
        if not filepath.exists():
            raise cherrypy.HTTPError(404,'Hey ! I dont know this token !')

        titlefile = PROCESSING_DIR/(token+'.status')
        title = titlefile.text().replace('/','_')

        serve = serve_file(filepath, "application/x-download", "attachment",title+'.mp3')
        return serve

cherrypy.quickstart(root=Zoidberg(),config="/Users/nikita/Code/bouncycastle/cherry.conf")


#!/usr/bin/env python
# Copyright 2016 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import argparse
import component_manifest
import json
import os
import paths
import re
import sys
import urlparse

class Filesystem:
    def __init__(self):
        self.files = []
        self.paths = {}

    def add_file(self, file):
        bootfs_path = file["bootfs_path"]
        if self.paths.has_key(bootfs_path):
            old_entry = self.paths[bootfs_path]
            if file["default"] and not old_entry["default"]:
                return  # we don't override a non-default with a default value
            if not old_entry["default"]:
                raise Exception('Duplicate bootfs path %s' % bootfs_path)
        self.files.append(file)
        self.paths[bootfs_path] = file

class Amalgamation:

    def __init__(self):
        self.packages = []
        self.deps = []
        self.labels = []
        self.config_paths = []
        self.component_urls = []
        self.build_root = ""
        self.omit_files = []
        self.boot = Filesystem() # Files that will live in /boot
        self.system = Filesystem() # Files that will live in /system
        self.resources = []
        self.gopaths = []

    def add_config(self, config, config_path):
        self.config_paths.append(config_path)
        packages = config.get("packages", {})
        for package in packages:
            if package in self.packages:
                raise Exception("Duplicate package name: %s" % package)
            self.packages.append(package)
            self.deps.append(packages[package])

        for c in config.get("components", []):
            # See https://fuchsia.googlesource.com/modular/src/component_manager/ for what a component is.
            manifest = component_manifest.ComponentManifest(os.path.join(paths.FUCHSIA_ROOT, c))
            self.component_urls.append(manifest.url)
            for component_file in manifest.files().values():
                self.system.add_file({
                    'file': os.path.join(self.build_root, 'components', component_file.url_as_path),
                    'bootfs_path': os.path.join('components', component_file.url_as_path),
                    'default': False
                })
        # TODO(jamesr): Everything below here is deprecated and no longer needed
        # when all package configurations are written in GN template. Migrate
        # and remove.
        for label in config.get("labels", []):
            self.deps.append(label)
        binaries_and_drivers = config.get("binaries", []) + config.get("drivers", [])
        for b in binaries_and_drivers:
            if b["binary"] in self.omit_files:
                continue
            file = {}
            file["file"] = os.path.join(self.build_root, b["binary"])
            file["bootfs_path"] = b["bootfs_path"]
            file["default"] = b.has_key("default")
            self.system.add_file(file)
        for d in config.get("early_boot", []):
            if d["binary"] in self.omit_files:
                continue
            file = {}
            file["file"] = os.path.join(self.build_root, d["binary"])
            file["bootfs_path"] = d["bootfs_path"]
            file["default"] = d.has_key("default")
            self.boot.add_file(file)
        for r in config.get("resources", []):
            if r["file"] in self.omit_files:
                continue
            file = {}
            source_path = os.path.join(paths.FUCHSIA_ROOT, r["file"])
            file["file"] = source_path
            self.resources.append(source_path)
            file["bootfs_path"] = r["bootfs_path"]
            file["default"] = r.has_key("default")
            self.system.add_file(file)
        if config.get("gopaths"):
            self.gopaths.extend(config.get("gopaths"))

def detect_duplicate_keys(pairs):
    keys = set()
    result = {}
    for k, v in pairs:
        if k in keys:
            raise Exception("Duplicate key %s" % k)
        keys.add(k)
        result[k] = v
    return result

def resolve_imports(import_queue, omit_files, build_root):
    # Hack: Add cpp manifest until we derive runtime information from the
    # package configs themselves.
    import_queue.append('cpp')
    imported = set(import_queue)
    amalgamation = Amalgamation()
    amalgamation.omit_files = omit_files
    amalgamation.build_root = build_root
    while import_queue:
        config_name = import_queue.pop()
        config_path = os.path.join(paths.SCRIPT_DIR, config_name)
        with open(config_path) as f:
            try:
                config = json.load(f, object_pairs_hook=detect_duplicate_keys)
                amalgamation.add_config(config, config_path)
                for i in config.get("imports", []):
                    if i not in imported:
                        import_queue.append(i)
                        imported.add(i)
            except Exception as e:
                import traceback
                traceback.print_exc()
                sys.stderr.write("Failed to parse config %s, error %s\n" % (config_path, str(e)))
                return None
    return amalgamation

def update_gopath(maps, build_root):
    for gopath in maps:
        for src in gopath:
            target = os.path.join(build_root, "src", gopath[src])
            src = os.path.join(paths.FUCHSIA_ROOT, src)
            if not os.path.exists(os.path.dirname(target)):
                os.makedirs(os.path.dirname(target))
            if os.path.lexists(target):
                os.remove(target)
            os.symlink(src, target)

def write_manifest(manifest, files, autorun=None):
    manifest_dir = os.path.dirname(manifest)
    if not os.path.exists(manifest_dir):
        os.makedirs(manifest_dir)
    with open(manifest, "w") as manifest_file:
        for f in files:
            manifest_file.write("%s=%s\n" % (f["bootfs_path"], f["file"]))
        if autorun:
            manifest_file.write("autorun=%s\n" % autorun)

def main():
    parser = argparse.ArgumentParser(description="Generate bootfs manifest and "
                                     + "list of GN targets for a list of Fuchsia modules")
    parser.add_argument("--packages", help="path to packages file to generate")
    parser.add_argument("--boot-manifest", help="path to manifest file to generate for /boot")
    parser.add_argument("--system-manifest", help="path to manifest file to generate for /system")
    parser.add_argument("--modules", help="list of modules", default="default")
    parser.add_argument("--omit-files", help="list of files omitted from user.bootfs", default="")
    parser.add_argument("--autorun", help="path to autorun script", default="")
    parser.add_argument("--build-root", help="path to root of build directory")
    parser.add_argument("--depfile", help="path to depfile to generate")
    parser.add_argument("--component-index", help="path to component index to generate")
    parser.add_argument("--arch", help="architecture being targetted")
    args = parser.parse_args()

    amalgamation = resolve_imports(args.modules.split(","), args.omit_files.split(","), args.build_root)
    if not amalgamation:
        return 1

    if len(amalgamation.boot.files) != 0:
        write_manifest(args.boot_manifest, amalgamation.boot.files)
    else:
        if os.path.exists(args.boot_manifest):
            os.remove(args.boot_manifest)
    write_manifest(args.system_manifest, amalgamation.system.files, autorun=args.autorun)

    update_gopath(amalgamation.gopaths, amalgamation.build_root)

    if args.depfile != "":
        with open(args.depfile, "w") as f:
            f.write("user.bootfs: ")
            f.write(args.boot_manifest + " " + args.system_manifest)
            for path in amalgamation.config_paths:
                f.write(" " + path)
            for resource in amalgamation.resources:
                f.write(" " + resource)

    if args.component_index != "":
        with open(args.component_index, "w") as f:
            json.dump(amalgamation.component_urls, f)

    with open(args.packages, "w") as f:
        f.write("\n".join(amalgamation.packages))
    sys.stdout.write("\n".join(amalgamation.deps))
    sys.stdout.write("\n")
    return 0

if __name__ == "__main__":
    sys.exit(main())


from click.testing import CliRunner
from tilestorm.cli import cli

def test_server_command(mocker):
    Flask = mocker.patch('tilestorm.app.app.Flask')

    runner = CliRunner()
    result = runner.invoke(cli, ['server'])
    assert result.exit_code == 0
    Flask.return_value.run.assert_called_with(host='127.0.0.1', port=5000, debug=True)


from setuptools import setup, find_packages
from freenet_installer.common import VERSION

install_requires = [
    "python-crontab", "psutil","python-gnupg", "requests" , "GitPython", "speedtest-cli"
]
with open("../README.md", "r") as fh:
    long_description = fh.read()

setup(name='freenet_installer',
      version=VERSION,
      description='Freenet python script to install freenet',
      author='Dator Rukitis',
      author_email='dator.rukitis@protonmail.com',
      packages=find_packages(),
      long_description=long_description,
      long_description_content_type="text/markdown",
      install_requires=install_requires,
      entry_points={'console_scripts':'freenet_installer=freenet_installer.freenet_installer:main'},
      package_data={
      'freenet_installer': ['*.gpg',"offline/*", "offline/*/*"],
      },
      data_files=[('gpg_keyring',["freenet_installer/keyring.gpg"])],
      include_package_data=True

      )


## To-do
#
#	Run all the web requests through tor using STEM
#
##
from bs4 import BeautifulSoup
import itertools
import requests
import logging
import time
import os

def query(url):
	"""
	Uses pycurl to fetch a site using the proxy on the SOCKS_PORT.
	"""
	output = io.BytesIO()
	SOCKS_PORT = 9050

	query = pycurl.Curl()
	query.setopt(pycurl.URL, url)
	query.setopt(pycurl.PROXY, 'localhost')
	query.setopt(pycurl.PROXYPORT, SOCKS_PORT)
	query.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS5_HOSTNAME)
	query.setopt(pycurl.WRITEFUNCTION, output.write)

	try:
		query.perform()
		return output.getvalue()
	except pycurl.error as exc:
		return "Unable to reach %s (%s)" % (url, exc)

def make_lesson_url(lesson_format,code_combination):
	'''
	Combine the code combination and format together to get the
	language combinations page
	'''
	return lesson_format.format(*code_combination)

def get_language_codes(main_page):
	'''
	Get all the language options available from the front page
	based on locale code
	'''
	codes = []

	soup = BeautifulSoup(requests.get(main_page).content,'lxml')
	selection_bar = soup.find(id='wordLanguage')
	selection_bar_options = selection_bar.find_all('option')

	for option in selection_bar_options:
		codes.append(option['value'])

	return codes

def get_lessons_urls(page_url):
	'''
	Get the urls of every lesson for a specific language combination
	'''
	lessons_urls = set([])

	page_soup = BeautifulSoup(requests.get(page_url).content,'lxml')
	anchor_elements = page_soup.find_all('a')
	for anchor in anchor_elements:
		if anchor.get('href','').startswith('lesson-'):
			lessons_urls.add(anchor['href'])
	return lessons_urls

def lesson_rip(base_url,lesson_fragment):
	'''
	Get the data for a specific lesson
	'''
	lesson_data = {'title':'','data':[]}

	page = '{}{}'.format(base_url,lesson_fragment)
	page_soup = BeautifulSoup(requests.get(page).content,'lxml')

	lesson_data['title'] = page_soup.title.text

	table = page_soup.find('table', {'class':'table table-bordered table-striped table-condensed'})
	trs = table.find_all('tr')[1:] # Get all rows, dropping header row
	for tr in trs:
		appended = 0
		line = ''
		tds = tr.find_all('td')
		if len(tds) != 2: # Skip if row not of correct type
			continue
		for td in tds:
			text = td.find_all('span',{'class':'big-text'})
			if text:
				line += text[0].text
				appended += 1
				if appended == 1:
					line += '\t\t\t' # Delimiter between languages
				else:
					line += '\n'
		lesson_data['data'].append(line)
	return lesson_data

def save_lesson_data(data):
	'''
	Save the data for a specific lesson
	'''
	file_handle = open(data['title'],'w')
	file_handle.writelines(data['data'])
	file_handle.close()

if __name__=='__main__':

	logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.DEBUG)

	main_page = 'http://www.internetpolyglot.com/'
	lesson_format = 'http://www.internetpolyglot.com/lessons-{}-{}'

	if not os.path.isdir('./data'): # Ensure storage location in place
		logging.info('Creating ./data')
		os.mkdir('./data')
	os.chdir('./data')

	codes = get_language_codes(main_page)

	for code_combo in itertools.product(codes,codes):
		if code_combo[0] == code_combo[1]: # Ignore same combinations
			continue

		else: # Do rip
			language_dir = './{}-{}'.format(code_combo[0],code_combo[1])

			if not os.path.isdir(language_dir):
				logging.info('Creating {}'.format(language_dir))
				os.mkdir(language_dir)
			os.chdir(language_dir)

			lessons_url = make_lesson_url(lesson_format,code_combo)
			lessons_urls = get_lessons_urls(lessons_url)

			for lesson in lessons_urls:
				logging.info('\tDealing with {}'.format(lesson))
				data = lesson_rip(main_page,lesson)
				save_lesson_data(data)

				time.sleep(10)

		os.chdir('..') # Return to data dir for next combination


# -*- coding: utf-8 -*-
'''
Views for book app
'''
from __future__ import unicode_literals

from django.shortcuts import render
from django.http import JsonResponse
from django.contrib.auth.decorators import login_required

from category.models import Category
from .models import Book
from .forms import BookForm

# Create your views here.

@login_required(login_url='/signin')
def book(request):
    '''
    Render principal page from book
    '''
    return render(request, 'book.html')

@login_required(login_url='/signin')
def get_books(request):
    '''
    View return all books
    '''
    if request.method == 'GET':
        books_raw = list(Book.objects.raw(
            "SELECT * FROM book_book"))
        books = [{"id": r.id, "number_serie": r.number_serie,
                  "name": r.name, "category": Category.objects.get(pk=r.category_book_id).category,
                  "resume": r.resume} for r in books_raw]
        return JsonResponse({"data": books}, safe=False)
    return None

@login_required(login_url='/signin')
def save_book(request):
    '''
    View save a book
    '''
    if request.method == 'POST':
        form = BookForm(request.POST)
        if form.is_valid():
            form.save()
            return JsonResponse({"status": "1", "type": "success", "message": "Book category."})
        return JsonResponse({"status": "-1", "type": "error", "message": "Form not valid."})
    return None

@login_required(login_url='/signin')
def edit_book(request, book_id):
    '''
    View edit a book
    '''
    book_edit = Book.objects.get(pk=id)
    if request.method == 'POST':
        form = BookForm(request.POST, instance=book_edit)
        if form.is_valid():
            form.save()
            return JsonResponse({"status": "1", "type": "success", "message": "Book edited."})
        return JsonResponse({"status": "-1", "type": "error", "message": "Form not valid."})
    return None

@login_required(login_url='/signin')
def delete_book(request):
    '''
    View delete a book
    '''
    if request.method == 'POST':
        id_book = request.POST['delete_book_id']
        book_delete = Book.objects.get(pk=id_book)
        book_delete.delete()
        return JsonResponse({"status": "1", "type": "success", "message": "Book deleted."})
    return None


import asyncio
import json
import base64
from . import AbstractStorage, Session, SESSION_KEY

from Crypto.Cipher import AES
from Crypto import Random

class SimpleCookieStorage(AbstractStorage):
    """Simple JSON storage.

    Doesn't any encryption/validation, use it for tests only"""

    def __init__(self, identity="AIOHTTP_COOKIE_SESSION", *,
                 domain=None, max_age=None, path='/',
                 secure=None, httponly=True):
        super().__init__(identity, domain=domain, max_age=max_age,
                         path=path, secure=secure, httponly=httponly)

    @asyncio.coroutine
    def make_session(self, request):
        cookie = self.load_cookie(request)
        if cookie is None:
            session = Session(self.identity, new=True)
        else:
            data = json.loads(cookie)
            session = Session(self.identity, data=data, new=False)

        request[SESSION_KEY] = session

    @asyncio.coroutine
    def save_session(self, request, response):
        session = request[SESSION_KEY]
        if not session._changed:
            return

        cookie_data = json.dumps(session._mapping)
        self.store_cookie(response, cookie_data)

class EncryptedCookieStorage(AbstractStorage):
    """Encrypted JSON storage.
    """

    def __init__(self, secret_key, identity="AIOHTTP_COOKIE_SESSION", *,
                 domain=None, max_age=None, path='/',
                 secure=None, httponly=True):
        super().__init__(identity, domain=domain, max_age=max_age,
                         path=path, secure=secure, httponly=httponly)

        self._secret_key = secret_key
        if len(self._secret_key) % AES.block_size != 0:
            raise TypeError(
                'Secret key must be a multiple of {} in length'.format(
                    AES.block_size))

    @asyncio.coroutine
    def make_session(self, request):
        cookie = self.load_cookie(request)
        if cookie is None:
            session = Session(self.identity, new=True)
        else:
            cookie = base64.b64decode(cookie)
            iv = cookie[:AES.block_size]
            data = cookie[AES.block_size:]
            cipher = AES.new(self._secret_key, AES.MODE_CBC, iv)
            decrypted = cipher.decrypt(data)
            data = json.loads(decrypted.decode('utf-8'))
            session = Session(self.identity, data=data, new=False)

        request[SESSION_KEY] = session

    @asyncio.coroutine
    def save_session(self, request, response):
        session = request[SESSION_KEY]
        if not session._changed:
            return

        cookie_data = json.dumps(session._mapping).encode('utf-8')
        if len(cookie_data) % AES.block_size != 0:
            # padding with spaces to full blocks
            to_pad = AES.block_size - (len(cookie_data) % AES.block_size)
            cookie_data += b' ' * to_pad

        iv = Random.new().read(AES.block_size)
        cipher = AES.new(self._secret_key, AES.MODE_CBC, iv)
        encrypted = cipher.encrypt(cookie_data)
        encrypted = iv + encrypted
        b64coded = base64.b64encode(encrypted).decode('utf-8')
        self.store_cookie(response, b64coded)