r"""
Generating Series

This file makes a number of extensions to lazy power series by
endowing them with some semantic content for how they're to be
interpreted.

This code is based on the work of Ralf Hemmecke and Martin Rubey's
Aldor-Combinat, which can be found at
http://www.risc.uni-linz.ac.at/people/hemmecke/aldor/combinat/index.html.
In particular, the relevant section for this file can be found at
http://www.risc.uni-linz.ac.at/people/hemmecke/AldorCombinat/combinatse10.html.
One notable difference is that we use power-sum symmetric functions
as the coefficients of our cycle index series.

TESTS::

    sage: from sage.combinat.species.stream import Stream, _integers_from
    sage: from sage.combinat.species.generating_series import CycleIndexSeriesRing
    sage: p = SymmetricFunctions(QQ).power()
    sage: CIS = CycleIndexSeriesRing(QQ)

::

    sage: geo1 = CIS((p([1])^i  for i in _integers_from(0)))
    sage: geo2 = CIS((p([2])^i  for i in _integers_from(0)))
    sage: s = geo1 * geo2
    sage: s[0]
    p[]
    sage: s[1]
    p[1] + p[2]
    sage: s[2]
    p[1, 1] + p[2, 1] + p[2, 2]
    sage: s[3]
    p[1, 1, 1] + p[2, 1, 1] + p[2, 2, 1] + p[2, 2, 2]

Whereas the coefficients of the above test are homogeneous with
respect to total degree, the following test groups with respect to
weighted degree where each variable x_i has weight i.

::

    sage: def g():
    ....:     for i in _integers_from(0):
    ....:         yield p([2])^i
    ....:         yield p(0)
    sage: geo1 = CIS((p([1])^i  for i in _integers_from(0)))
    sage: geo2 = CIS(g())
    sage: s = geo1 * geo2
    sage: s[0]
    p[]
    sage: s[1]
    p[1]
    sage: s[2]
    p[1, 1] + p[2]
    sage: s[3]
    p[1, 1, 1] + p[2, 1]
    sage: s[4]
    p[1, 1, 1, 1] + p[2, 1, 1] + p[2, 2]

REFERENCES:

.. [BLL] \F. Bergeron, G. Labelle, and P. Leroux.
   "Combinatorial species and tree-like structures".
   Encyclopedia of Mathematics and its Applications, vol. 67, Cambridge Univ. Press. 1998.
.. [BLL-Intro] Francois Bergeron, Gilbert Labelle, and Pierre Leroux.
   "Introduction to the Theory of Species of Structures", March 14, 2008.
"""
from __future__ import absolute_import

#*****************************************************************************
#       Copyright (C) 2008 Mike Hansen <mhansen@gmail.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.
#                  http://www.gnu.org/licenses/
#*****************************************************************************

from .series import LazyPowerSeriesRing, LazyPowerSeries
from .stream import Stream, _integers_from
from sage.rings.all import Integer, RationalField
from sage.arith.all import moebius, gcd, lcm, divisors
from sage.combinat.partition import Partition, Partitions
from functools import partial
from sage.combinat.sf.sf import SymmetricFunctions
from sage.misc.cachefunc import cached_function

@cached_function
def OrdinaryGeneratingSeriesRing(R):
    """
    Return the ring of ordinary generating series over ``R``.

    Note that it is just a
    :class:`LazyPowerSeriesRing` whose elements have
    some extra methods.

    EXAMPLES::

        sage: from sage.combinat.species.generating_series import OrdinaryGeneratingSeriesRing
        sage: R = OrdinaryGeneratingSeriesRing(QQ); R
        Lazy Power Series Ring over Rational Field
        sage: R([1]).coefficients(4)
        [1, 1, 1, 1]
        sage: R([1]).counts(4)
        [1, 1, 1, 1]

    TESTS:

    We test to make sure that caching works.

    ::

        sage: R is OrdinaryGeneratingSeriesRing(QQ)
        True
    """
    return OrdinaryGeneratingSeriesRing_class(R)

class OrdinaryGeneratingSeriesRing_class(LazyPowerSeriesRing):
    def __init__(self, R):
        """
        EXAMPLES::

            sage: from sage.combinat.species.generating_series import OrdinaryGeneratingSeriesRing
            sage: R = OrdinaryGeneratingSeriesRing(QQ)
            sage: R == loads(dumps(R))
            True
        """
        LazyPowerSeriesRing.__init__(self, R, OrdinaryGeneratingSeries)

class OrdinaryGeneratingSeries(LazyPowerSeries):
    def count(self, n):
        """
        Return the number of structures on a set of size ``n``.

        EXAMPLES::

            sage: from sage.combinat.species.generating_series import OrdinaryGeneratingSeriesRing
            sage: R = OrdinaryGeneratingSeriesRing(QQ)
            sage: f = R(range(20))
            sage: f.count(10)
            10
        """
        return self.coefficient(n)

    def counts(self, n):
        """
        Return the number of structures on a set for size ``i`` for
        each ``i`` in ``range(n)``.

        EXAMPLES::

            sage: from sage.combinat.species.generating_series import OrdinaryGeneratingSeriesRing
            sage: R = OrdinaryGeneratingSeriesRing(QQ)
            sage: f = R(range(20))
            sage: f.counts(10)
            [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
        """
        return [self.count(i) for i in range(n)]

@cached_function
def ExponentialGeneratingSeriesRing(R):
    """
    Return the ring of exponential generating series over ``R``.

    Note that it is just a
    :class:`LazyPowerSeriesRing` whose elements have
    some extra methods.

    EXAMPLES::

        sage: from sage.combinat.species.generating_series import ExponentialGeneratingSeriesRing
        sage: R = ExponentialGeneratingSeriesRing(QQ); R
        Lazy Power Series Ring over Rational Field
        sage: R([1]).coefficients(4)
        [1, 1, 1, 1]
        sage: R([1]).counts(4)
        [1, 1, 2, 6]

    TESTS:

    We test to make sure that caching works.

    ::

        sage: R is ExponentialGeneratingSeriesRing(QQ)
        True
    """
    return ExponentialGeneratingSeriesRing_class(R)

class ExponentialGeneratingSeriesRing_class(LazyPowerSeriesRing):
    def __init__(self, R):
        """
        EXAMPLES::

            sage: from sage.combinat.species.generating_series import ExponentialGeneratingSeriesRing
            sage: R = ExponentialGeneratingSeriesRing(QQ)
            sage: R == loads(dumps(R))
            True
        """
        LazyPowerSeriesRing.__init__(self, R, ExponentialGeneratingSeries)

class ExponentialGeneratingSeries(LazyPowerSeries):
    def count(self, n):
        """
        Return the number of structures of size ``n``.

        EXAMPLES::

            sage: from sage.combinat.species.generating_series import ExponentialGeneratingSeriesRing
            sage: R = ExponentialGeneratingSeriesRing(QQ)
            sage: f = R([1])
            sage: [f.count(i) for i in range(7)]
            [1, 1, 2, 6, 24, 120, 720]
        """
        return factorial_stream[n] * self.coefficient(n)

    def counts(self, n):
        """
        Return the number of structures on a set for size ``i`` for
        each ``i`` in ``range(n)``.

        EXAMPLES::

            sage: from sage.combinat.species.generating_series import ExponentialGeneratingSeriesRing
            sage: R = ExponentialGeneratingSeriesRing(QQ)
            sage: f = R(range(20))
            sage: f.counts(5)
            [0, 1, 4, 18, 96]
        """
        return [self.count(i) for i in range(n)]

    def functorial_composition(self, y):
        r"""
        Return the exponential generating series which is the functorial
        composition of ``self`` with ``y``.

        If `f = \sum_{n=0}^{\infty} f_n \frac{x^n}{n!}` and
        `g = \sum_{n=0}^{\infty} g_n \frac{x^n}{n!}`, then
        functorial composition `f \Box g` is defined as

        .. MATH::

             f \Box g = \sum_{n=0}^{\infty} f_{g_n} \frac{x^n}{n!}

        REFERENCES:

        - Section 2.2 of [BLL]_.

        EXAMPLES::

            sage: G = species.SimpleGraphSpecies()
            sage: g = G.generating_series()
            sage: g.coefficients(10)
            [1, 1, 1, 4/3, 8/3, 128/15, 2048/45, 131072/315, 2097152/315, 536870912/2835]
        """
        return self._new(partial(self._functorial_compose_gen, y), lambda a,b: 0, self, y)

    def _functorial_compose_gen(self, y, ao):
        """
        Returns a generator for the coefficients of the functorial
        composition of self with y.

        EXAMPLES::

            sage: E = species.SetSpecies()
            sage: E2 = E.restricted(min=2, max=3)
            sage: WP = species.SubsetSpecies()
            sage: P2 = E2*E
            sage: g1 = WP.generating_series()
            sage: g2 = P2.generating_series()
            sage: g = g1._functorial_compose_gen(g2, 0)
            sage: [next(g) for i in range(10)]
            [1, 1, 1, 4/3, 8/3, 128/15, 2048/45, 131072/315, 2097152/315, 536870912/2835]
        """
        n = 0
        while True:
            yield self.count(y.count(n))/factorial_stream[n]
            n += 1

def factorial_gen():
    """
    A generator for the factorials starting at 0.

    EXAMPLES::

        sage: from sage.combinat.species.generating_series import factorial_gen
        sage: g = factorial_gen()
        sage: [next(g) for i in range(5)]
        [1, 1, 2, 6, 24]
    """
    z = Integer(1)
    yield z
    yield z
    n = Integer(2)
    while True:
        z *= n
        yield z
        n += 1

factorial_stream = Stream(factorial_gen())

@cached_function
def CycleIndexSeriesRing(R):
    r"""
    Return the ring of cycle index series over ``R``.

    This is the ring of formal power series `\Lambda[x]`, where
    `\Lambda` is the ring of symmetric functions over ``R`` in the
    `p`-basis. Its purpose is to house the cycle index series of
    species (in a somewhat nonstandard notation tailored to Sage):
    If `F` is a species, then the *cycle index series* of `F` is
    defined to be the formal power series

    .. MATH::

        \sum_{n \geq 0} \frac{1}{n!} (\sum_{\sigma \in S_n}
        \operatorname{fix} F[\sigma]
        \prod_{z \text{ is a cycle of } \sigma}
        p_{\text{length of } z}) x^n
        \in \Lambda_\QQ [x],

    where `\operatorname{fix} F[\sigma]` denotes the number of
    fixed points of the permutation `F[\sigma]` of `F[n]`. We
    notice that this power series is "equigraded" (meaning that
    its `x^n`-coefficient is homogeneous of degree `n`). A more
    standard convention in combinatorics would be to use
    `x_i` instead of `p_i`, and drop the `x` (that is, evaluate
    the above power series at `x = 1`); but this would be more
    difficult to implement in Sage, as it would be an element
    of a power series ring in infinitely many variables.

    Note that it is just a :class:`LazyPowerSeriesRing` (whose base
    ring is `\Lambda`) whose elements have some extra methods.

    EXAMPLES::

        sage: from sage.combinat.species.generating_series import CycleIndexSeriesRing
        sage: R = CycleIndexSeriesRing(QQ); R
        Cycle Index Series Ring over Symmetric Functions over Rational Field in the powersum basis
        sage: R([1]).coefficients(4) # This is not combinatorially
        ....:                        # meaningful.
        [1, 1, 1, 1]

    TESTS:

    We test to make sure that caching works.

    ::

        sage: R is CycleIndexSeriesRing(QQ)
        True
    """
    return CycleIndexSeriesRing_class(R)

class CycleIndexSeriesRing_class(LazyPowerSeriesRing):
    def __init__(self, R):
        """
        EXAMPLES::

            sage: from sage.combinat.species.generating_series import CycleIndexSeriesRing
            sage: R = CycleIndexSeriesRing(QQ); R
            Cycle Index Series Ring over Symmetric Functions over Rational Field in the powersum basis
            sage: R == loads(dumps(R))
            True
        """
        R = SymmetricFunctions(R).power()
        LazyPowerSeriesRing.__init__(self, R, CycleIndexSeries)

    def __repr__(self):
        """
        EXAMPLES::

            sage: from sage.combinat.species.generating_series import CycleIndexSeriesRing
            sage: CycleIndexSeriesRing(QQ)
            Cycle Index Series Ring over Symmetric Functions over Rational Field in the powersum basis
        """
        return "Cycle Index Series Ring over %s"%self.base_ring()

class CycleIndexSeries(LazyPowerSeries):
    def count(self, t):
        """
        Return the number of structures corresponding to a certain cycle
        type ``t``.

        EXAMPLES::

            sage: from sage.combinat.species.generating_series import CycleIndexSeriesRing
            sage: p = SymmetricFunctions(QQ).power()
            sage: CIS = CycleIndexSeriesRing(QQ)
            sage: f = CIS([0, p([1]), 2*p([1,1]), 3*p([2,1])])
            sage: f.count([1])
            1
            sage: f.count([1,1])
            4
            sage: f.count([2,1])
            6
        """
        t = Partition(t)
        return t.aut() * self.coefficient_cycle_type(t)

    def coefficient_cycle_type(self, t):
        """
        Returns the coefficient of a cycle type ``t`` in ``self``.

        EXAMPLES::

            sage: from sage.combinat.species.generating_series import CycleIndexSeriesRing
            sage: p = SymmetricFunctions(QQ).power()
            sage: CIS = CycleIndexSeriesRing(QQ)
            sage: f = CIS([0, p([1]), 2*p([1,1]),3*p([2,1])])
            sage: f.coefficient_cycle_type([1])
            1
            sage: f.coefficient_cycle_type([1,1])
            2
            sage: f.coefficient_cycle_type([2,1])
            3
        """
        t = Partition(t)
        p = self.coefficient(t.size())
        return p.coefficient(t)

    def stretch(self, k):
        r"""
        Return the stretch of the cycle index series ``self`` by a positive
        integer `k`.

        If

        .. MATH::

           f = \sum_{n=0}^{\infty} f_n(p_1, p_2, p_3, \ldots ),

        then the stretch `g` of `f` by `k` is

        .. MATH::

           g = \sum_{n=0}^{\infty} f_n(p_k, p_{2k}, p_{3k}, \ldots ).

        EXAMPLES::

            sage: from sage.combinat.species.generating_series import CycleIndexSeriesRing
            sage: p = SymmetricFunctions(QQ).power()
            sage: CIS = CycleIndexSeriesRing(QQ)
            sage: f = CIS([p([]), p([1]), p([2]), p.zero()])
            sage: f.stretch(3).coefficients(10)
            [p[], 0, 0, p[3], 0, 0, p[6], 0, 0, 0]
        """
        return self._new(partial(self._stretch_gen, k), lambda ao: k*ao, self)

    def _stretch_gen(self, k, ao):
        """
        EXAMPLES::

            sage: from sage.combinat.species.generating_series import CycleIndexSeriesRing
            sage: p = SymmetricFunctions(QQ).power()
            sage: CIS = CycleIndexSeriesRing(QQ)
            sage: f = CIS([p([1])]) # This is the power series whose all coefficients
            ....:                   # are p[1]. Not combinatorially meaningful!
            sage: g = f._stretch_gen(2,0)
            sage: [next(g) for i in range(10)]
            [p[2], 0, p[2], 0, p[2], 0, p[2], 0, p[2], 0]
        """
        from sage.combinat.partition import Partition
        BR = self.base_ring()
        zero = BR.zero()

        stretch_k = lambda p: Partition([k*i for i in p])

        yield self.coefficient(0).map_support(stretch_k)

        n = 1
        while True:
            for i in range(k-1):
                yield zero
            yield self.coefficient(n).map_support(stretch_k)
            n += 1

    def isotype_generating_series(self):
        """
        EXAMPLES::

            sage: P = species.PermutationSpecies()
            sage: cis = P.cycle_index_series()
            sage: f = cis.isotype_generating_series()
            sage: f.coefficients(10)
            [1, 1, 2, 3, 5, 7, 11, 15, 22, 30]
        """
        R = self.base_ring().base_ring()
        OGS = OrdinaryGeneratingSeriesRing(R)()
        return OGS._new(self._ogs_gen, lambda ao: ao, self)

    def expand_as_sf(self, n, alphabet='x'):
        """
        Returns the expansion of a cycle index series as a symmetric function in
        ``n`` variables.

        Specifically, this returns a :class:`~sage.combinat.species.series.LazyPowerSeries` whose
        ith term is obtained by calling :meth:`~sage.combinat.sf.sfa.SymmetricFunctionAlgebra_generic_Element.expand`
        on the ith term of ``self``.

        This relies on the (standard) interpretation of a cycle index series as a symmetric function
        in the power sum basis.

        INPUT:

        - ``self`` -- a cycle index series

        - ``n`` -- a positive integer

        - ``alphabet`` -- a variable for the expansion (default: `x`)

        EXAMPLES::

            sage: from sage.combinat.species.set_species import SetSpecies
            sage: SetSpecies().cycle_index_series().expand_as_sf(2).coefficients(4)
            [1, x0 + x1, x0^2 + x0*x1 + x1^2, x0^3 + x0^2*x1 + x0*x1^2 + x1^3]

        """
        expanded_poly_ring = self.coefficient(0).expand(n, alphabet).parent()
        LPSR = LazyPowerSeriesRing(expanded_poly_ring)

        expander_gen = (LPSR.term(self.coefficient(i).expand(n, alphabet), i) for i in _integers_from(0))

        return LPSR.sum_generator(expander_gen)

    def _ogs_gen(self, ao):
        """
        Returns a generator for the coefficients of the ordinary generating
        series obtained from a cycle index series.

        EXAMPLES::

            sage: P = species.PermutationSpecies()
            sage: cis = P.cycle_index_series()
            sage: g = cis._ogs_gen(0)
            sage: [next(g) for i in range(10)]
            [1, 1, 2, 3, 5, 7, 11, 15, 22, 30]
        """
        for i in range(ao):
            yield 0
        for i in _integers_from(ao):
            yield sum( self.coefficient(i).coefficients() )

    def generating_series(self):
        """
        EXAMPLES::

            sage: P = species.PartitionSpecies()
            sage: cis = P.cycle_index_series()
            sage: f = cis.generating_series()
            sage: f.coefficients(5)
            [1, 1, 1, 5/6, 5/8]
        """
        R = self.base_ring().base_ring()
        EGS = ExponentialGeneratingSeriesRing(R)()
        return EGS._new(self._egs_gen, lambda ao: ao, self)

    def _egs_gen(self, ao):
        """
        Returns a generator for the coefficients of the exponential
        generating series obtained from a cycle index series.

        EXAMPLES::

            sage: P = species.PermutationSpecies()
            sage: cis = P.cycle_index_series()
            sage: g = cis._egs_gen(0)
            sage: [next(g) for i in range(10)]
            [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
        """
        for i in range(ao):
            yield 0
        for i in _integers_from(ao):
            yield self.coefficient(i).coefficient([1]*i)

    def __invert__(self):
        r"""
        Return the multiplicative inverse of ``self``.

        This algorithm is derived from [BLL]_.

        EXAMPLES::

            sage: E = species.SetSpecies().cycle_index_series()
            sage: E.__invert__().coefficients(4)
            [p[], -p[1], 1/2*p[1, 1] - 1/2*p[2], -1/6*p[1, 1, 1] + 1/2*p[2, 1] - 1/3*p[3]]

        The defining characteristic of the multiplicative inverse `F^{-1}` of a cycle index series `F`
        is that `F \cdot F^{-1} = F^{-1} \cdot F = 1` (that is, both products with `F` yield the multiplicative identity `1`)::

            sage: E = species.SetSpecies().cycle_index_series()
            sage: (E * ~E).coefficients(6)
            [p[], 0, 0, 0, 0, 0]

        REFERENCES:

        [BLL]_

        [BLL-Intro]_

        http://bergeron.math.uqam.ca/Site/bergeron_anglais_files/livre_combinatoire.pdf

        AUTHORS:

        - Andrew Gainer-Dewar
        """
        if self.coefficient(0) == 0:
            raise ValueError("Constant term must be non-zero")

        def multinv_builder(i):
            return self.coefficient(0)**(-i-1) * (self.coefficient(0) + (-1)*self)**i

        return self.parent().sum_generator(multinv_builder(i) for i in _integers_from(0))

    def _div_(self, y):
        """
        TESTS::

            sage: E = species.SetSpecies().cycle_index_series()
            sage: (E / E).coefficients(6)
            [p[], 0, 0, 0, 0, 0]
        """
        return self*(~y)

    def functorial_composition(self, g):
        r"""
        Returns the functorial composition of ``self`` and ``g``.

        If `F` and `G` are species, their functorial composition is the species
        `F \Box G` obtained by setting `(F \Box G) [A] = F[ G[A] ]`.
        In other words, an `(F \Box G)`-structure on a set `A` of labels is an
        `F`-structure whose labels are the set of all `G`-structures on `A`.

        It can be shown (as in section 2.2 of [BLL]_) that there is a corresponding operation on cycle indices:

        .. MATH::

            Z_{F} \Box Z_{G} = \sum_{n \geq 0} \frac{1}{n!} \sum_{\sigma \in \mathfrak{S}_{n}} \operatorname{fix} F[ (G[\sigma])_{1}, (G[\sigma])_{2}, \dots ] \, p_{1}^{\sigma_{1}} p_{2}^{\sigma_{2}} \dots.

        This method implements that operation on cycle index series.

        EXAMPLES:

        The species `G` of simple graphs can be expressed in terms of a functorial
        composition: `G = \mathfrak{p} \Box \mathfrak{p}_{2}`, where
        `\mathfrak{p}` is the :class:`~sage.combinat.species.subset_species.SubsetSpecies`.
        This is how it is implemented in :meth:`~sage.combinat.species.library.SimpleGraphSpecies`::

            sage: S = species.SimpleGraphSpecies()
            sage: S.cycle_index_series().coefficients(5)
            [p[],
             p[1],
             p[1, 1] + p[2],
             4/3*p[1, 1, 1] + 2*p[2, 1] + 2/3*p[3],
             8/3*p[1, 1, 1, 1] + 4*p[2, 1, 1] + 2*p[2, 2] + 4/3*p[3, 1] + p[4]]
        """
        return self._new(partial(self._functorial_compose_gen, g), lambda a,b: 0, self, g)

    def _functorial_compose_gen(self, g, ao):
        """
        Return a generator for the coefficients of the functorial
        composition of ``self`` with ``g``.

        EXAMPLES::

            sage: E = species.SetSpecies()
            sage: E2 = species.SetSpecies(size=2)
            sage: WP = species.SubsetSpecies()
            sage: P2 = E2*E
            sage: P2_cis = P2.cycle_index_series()
            sage: WP_cis = WP.cycle_index_series()
            sage: g = WP_cis._functorial_compose_gen(P2_cis,0)
            sage: [next(g) for i in range(5)]
            [p[],
             p[1],
             p[1, 1] + p[2],
             4/3*p[1, 1, 1] + 2*p[2, 1] + 2/3*p[3],
             8/3*p[1, 1, 1, 1] + 4*p[2, 1, 1] + 2*p[2, 2] + 4/3*p[3, 1] + p[4]]
        """
        p = self.parent().base_ring()
        n = 0
        while True:
            res = p(0)
            for s in Partitions(n):
                t = g._cycle_type(s)
                q = self.count(t) / s.aut()
                res += q*p(s)
            yield res
            n += 1

    def arithmetic_product(self, g, check_input = True):
        r"""
        Return the arithmetic product of ``self`` with ``g``.

        For species `M` and `N` such that `M[\\varnothing] = N[\\varnothing] = \\varnothing`,
        their arithmetic product is the species `M \\boxdot N` of "`M`-assemblies of cloned `N`-structures".
        This operation is defined and several examples are given in [MM]_.

        The cycle index series for `M \\boxdot N` can be computed in terms of the component series `Z_M` and `Z_N`,
        as implemented in this method.

        INPUT:

        - ``g`` -- a cycle index series having the same parent as ``self``.

        - ``check_input`` -- (default: ``True``) a Boolean which, when set
          to ``False``, will cause input checks to be skipped.

        OUTPUT:

        The arithmetic product of ``self`` with ``g``. This is a cycle
        index series defined in terms of ``self`` and ``g`` such that
        if ``self`` and ``g`` are the cycle index series of two species
        `M` and `N`, their arithmetic product is the cycle index series
        of the species `M \\boxdot N`.

        EXAMPLES:

        For `C` the species of (oriented) cycles and `L_{+}` the species of nonempty linear orders, `C \\boxdot L_{+}` corresponds
        to the species of "regular octopuses"; a `(C \\boxdot L_{+})`-structure is a cycle of some length, each of whose elements
        is an ordered list of a length which is consistent for all the lists in the structure. ::

            sage: C = species.CycleSpecies().cycle_index_series()
            sage: Lplus = species.LinearOrderSpecies(min=1).cycle_index_series()
            sage: RegularOctopuses = C.arithmetic_product(Lplus)
            sage: RegOctSpeciesSeq = RegularOctopuses.generating_series().counts(8)
            sage: RegOctSpeciesSeq
            [0, 1, 3, 8, 42, 144, 1440, 5760]

        It is shown in [MM]_ that the exponential generating function for regular octopuses satisfies
        `(C \\boxdot L_{+}) (x) = \\sum_{n \geq 1} \\sigma (n) (n - 1)! \\frac{x^{n}}{n!}` (where `\\sigma (n)` is
        the sum of the divisors of `n`). ::

            sage: RegOctDirectSeq = [0] + [sum(divisors(i))*factorial(i-1) for i in range(1,8)]
            sage: RegOctDirectSeq == RegOctSpeciesSeq
            True

        AUTHORS:

        - Andrew Gainer-Dewar (2013)

        REFERENCES:

        .. [MM] \M. Maia and M. Mendez. "On the arithmetic product of combinatorial species".
           Discrete Mathematics, vol. 308, issue 23, 2008, pp. 5407-5427.
           :arXiv:`math/0503436v2`.

        """
        from itertools import product, repeat, chain

        p = self.base_ring()

        if check_input:
            assert self.coefficient(0) == p.zero()
            assert g.coefficient(0) == p.zero()

        # We first define an operation `\\boxtimes` on partitions as in Lemma 2.1 of [MM]_.
        def arith_prod_of_partitions(l1, l2):
            # Given two partitions `l_1` and `l_2`, we construct a new partition `l_1 \\boxtimes l_2` by
            # the following procedure: each pair of parts `a \\in l_1` and `b \\in l_2` contributes
            # `\\gcd (a, b)`` parts of size `\\lcm (a, b)` to `l_1 \\boxtimes l_2`. If `l_1` and `l_2`
            # are partitions of integers `n` and `m`, respectively, then `l_1 \\boxtimes l_2` is a
            # partition of `nm`.
            term_iterable = chain.from_iterable(repeat(lcm(pair), gcd(pair))
                                                for pair in product(l1, l2))
            return Partition(sorted(term_iterable, reverse=True))

        # We then extend this to an operation on symmetric functions as per eq. (52) of [MM]_.
        # (Maia and Mendez, in [MM]_, are talking about polynomials instead of symmetric
        # functions, but this boils down to the same: Their x_i corresponds to the i-th power
        # sum symmetric function.)
        def arith_prod_sf(x, y):
            ap_sf_wrapper = lambda l1, l2: p(arith_prod_of_partitions(l1, l2))
            return p._apply_multi_module_morphism(x, y, ap_sf_wrapper)

        # Sage stores cycle index series by degree.
        # Thus, to compute the arithmetic product `Z_M \\boxdot Z_N` it is useful
        # to compute all terms of a given degree `n` at once.
        def arith_prod_coeff(n):
            if n == 0:
                res = p.zero()
            else:
                index_set = ((d, n // d) for d in divisors(n))
                res = sum(arith_prod_sf(self.coefficient(i), g.coefficient(j)) for i,j in index_set)

            # Build a list which has res in the `n`th slot and 0's before and after
            # to feed to sum_generator
            res_in_seq = [p.zero()]*n + [res, p.zero()]

            return self.parent(res_in_seq)

        # Finally, we use the sum_generator method to assemble these results into a single
        # LazyPowerSeries object.
        return self.parent().sum_generator(arith_prod_coeff(n) for n in _integers_from(0))

    def _cycle_type(self, s):
        """
        EXAMPLES::

            sage: cis = species.PartitionSpecies().cycle_index_series()
            sage: [cis._cycle_type(p) for p in Partitions(3)]
            [[3, 1, 1], [2, 1, 1, 1], [1, 1, 1, 1, 1]]
            sage: cis = species.PermutationSpecies().cycle_index_series()
            sage: [cis._cycle_type(p) for p in Partitions(3)]
            [[3, 1, 1, 1], [2, 2, 1, 1], [1, 1, 1, 1, 1, 1]]
            sage: cis = species.SetSpecies().cycle_index_series()
            sage: [cis._cycle_type(p) for p in Partitions(3)]
            [[1], [1], [1]]
        """
        if s == []:
            return self._card(0)
        res = []
        for k in range(1, self._upper_bound_for_longest_cycle(s)+1):
            e = 0
            for d in divisors(k):
                m = moebius(d)
                if m == 0:
                    continue
                u = s.power(k/d)
                e += m*self.count(u)
            res.extend([k]*int(e/k))
        res.reverse()
        return Partition(res)

    def _upper_bound_for_longest_cycle(self, s):
        """
        EXAMPLES::

            sage: cis = species.PartitionSpecies().cycle_index_series()
            sage: cis._upper_bound_for_longest_cycle([4])
            4
            sage: cis._upper_bound_for_longest_cycle([3,1])
            3
            sage: cis._upper_bound_for_longest_cycle([2,2])
            2
            sage: cis._upper_bound_for_longest_cycle([2,1,1])
            2
            sage: cis._upper_bound_for_longest_cycle([1,1,1,1])
            1
        """
        if s == []:
            return 1
        return min(self._card(sum(s)), lcm(list(s)))

    def _card(self, n):
        """
        Returns the number of structures on an underlying set of size n for
        the species associated with self. This is just n! times the
        coefficient of p[1]n in self.

        EXAMPLES::

            sage: cis = species.PartitionSpecies().cycle_index_series()
            sage: cis._card(4)
            15
        """
        p = self.coefficient(n)
        return factorial_stream[n]*p.coefficient([1]*n)

    def _compose_gen(self, y, ao):
        """
        Return a generator for the coefficients of the composition of this
        cycle index series and the cycle index series ``y``. This overrides
        the method defined in ``LazyPowerSeries``.

        The notion "composition" means plethystic substitution here, as
        defined in Section 2.2 of [BLL-Intro]_.

        EXAMPLES::

            sage: E = species.SetSpecies(); C = species.CycleSpecies()
            sage: E_cis = E.cycle_index_series()
            sage: g = E_cis._compose_gen(C.cycle_index_series(),0)
            sage: [next(g) for i in range(4)]
            [p[], p[1], p[1, 1] + p[2], p[1, 1, 1] + p[2, 1] + p[3]]
        """
        assert y.coefficient(0) == 0
        y_powers = Stream(y._power_gen())

        parent = self.parent()
        res =  parent.sum_generator(self._compose_term(self.coefficient(i), y_powers)
                                    for i in _integers_from(0))

        for i in _integers_from(0):
            yield res.coefficient(i)

    def _compose_term(self, p, y_powers):
        """
        Returns the composition of one term in self with y.

        INPUT:

        -  ``p`` - a term in self

        -  ``y_powers`` - a stream for the powers of y
           starting with y

        EXAMPLES::

            sage: from sage.combinat.species.stream import Stream
            sage: E = species.SetSpecies(); C = species.CycleSpecies()
            sage: E_cis = E.cycle_index_series()
            sage: C_cis = C.cycle_index_series()
            sage: c_powers = Stream(C_cis._power_gen())
            sage: p2 = E_cis.coefficient(2); p2
            1/2*p[1, 1] + 1/2*p[2]
            sage: E_cis._compose_term(p2, c_powers).coefficients(4)
            [0, 0, 1/2*p[1, 1] + 1/2*p[2], 1/2*p[1, 1, 1] + 1/2*p[2, 1]]
        """
        parent = self.parent()
        if p == 0:
            return parent(0)

        res = []
        #Go through all the partition, coefficient pairs in the term p
        for m, c in p:
            res_t = parent.term(c, 0)

            for e,v in enumerate(m.to_exp()):
                if v == 0:
                    continue
                res_t = res_t * y_powers[v-1].stretch(e+1)
            res.append(res_t)

        return parent.sum(res)

    def weighted_composition(self, y_species):
        """
        Returns the composition of this cycle index series with the cycle
        index series of y_species where y_species is a weighted species.

        Note that this is basically the same algorithm as composition
        except we can not use the optimization that the powering of cycle
        index series commutes with 'stretching'.

        EXAMPLES::

            sage: E = species.SetSpecies(); C = species.CycleSpecies()
            sage: E_cis = E.cycle_index_series()
            sage: E_cis.weighted_composition(C).coefficients(4)
            [p[], p[1], p[1, 1] + p[2], p[1, 1, 1] + p[2, 1] + p[3]]
            sage: E(C).cycle_index_series().coefficients(4)
            [p[], p[1], p[1, 1] + p[2], p[1, 1, 1] + p[2, 1] + p[3]]
        """
        base_ring = self.base_ring()
        y = y_species.cycle_index_series(base_ring)
        assert y.coefficient(0) == 0
        return self._new(partial(self._weighted_compose_gen, y_species), lambda a,b:a*b, self, y)

    def _weighted_compose_gen(self, y_species, ao):
        """
        Returns an iterator for the composition of this cycle index series
        and the cycle index series of the weighted species y_species.

        EXAMPLES::

            sage: E = species.SetSpecies(); C = species.CycleSpecies()
            sage: E_cis = E.cycle_index_series()
            sage: g = E_cis._weighted_compose_gen(C,0)
            sage: [next(g) for i in range(4)]
            [p[], p[1], p[1, 1] + p[2], p[1, 1, 1] + p[2, 1] + p[3]]
        """
        parent = self.parent()
        res =  parent.sum_generator(self._weighted_compose_term(self.coefficient(i), y_species)
                                    for i in _integers_from(0))

        for i in _integers_from(0):
            yield res.coefficient(i)

    def _weighted_compose_term(self, p, y_species):
        """
        Returns the weighted composition of one term in self with y.

        INPUT:

        -  ``p`` - a term in self

        -  ``y_species`` - a species

        EXAMPLES::

            sage: E = species.SetSpecies(); C = species.CycleSpecies()
            sage: E_cis = E.cycle_index_series()
            sage: p2 = E_cis.coefficient(2); p2
            1/2*p[1, 1] + 1/2*p[2]
            sage: E_cis._weighted_compose_term(p2, C).coefficients(4)
            [0, 0, 1/2*p[1, 1] + 1/2*p[2], 1/2*p[1, 1, 1] + 1/2*p[2, 1]]
        """
        parent = self.parent()
        if p == 0:
            return parent(0)

        base_ring = self.base_ring().base_ring()

        res = []
        #Go through all the partition, coefficient pairs in the term p
        for m, c in p:
            res_t = parent.term(c, 0)

            for e,v in enumerate(m.to_exp()):
                if v == 0:
                    continue
                res_t = res_t * (y_species.weighted(y_species._weight**(e+1)).cycle_index_series(base_ring)**v).stretch(e+1)
            res.append(res_t)

        return parent.sum(res)

    def compositional_inverse(self):
        r"""
        Return the compositional inverse of ``self`` if possible.

        (Specifically, if ``self`` is of the form `0 + p_{1} + \dots`.)

        The compositional inverse is the inverse with respect to
        plethystic substitution. This is the operation on cycle index
        series which corresponds to substitution, a.k.a. partitional
        composition, on the level of species. See Section 2.2 of
        [BLL]_ for a definition of this operation.

        EXAMPLES::

            sage: Eplus = species.SetSpecies(min=1).cycle_index_series()
            sage: Eplus(Eplus.compositional_inverse()).coefficients(8)
            [0, p[1], 0, 0, 0, 0, 0, 0]

        TESTS::

            sage: Eplus = species.SetSpecies(min=2).cycle_index_series()
            sage: Eplus.compositional_inverse()
            Traceback (most recent call last):
            ...
            ValueError: not an invertible series

        ALGORITHM:

        Let `F` be a species satisfying `F = 0 + X + F_2 + F_3 + \dots` for `X` the species of singletons.
        (Equivalently, `\lvert F[\varnothing] \rvert = 0` and `\lvert F[\{1\}] \rvert = 1`.)
        Then there exists a (virtual) species `G` satisfying `F \circ G = G \circ F = X`.

        It follows that `(F - X) \circ G = F \circ G - X \circ G = X - G`.
        Rearranging, we obtain the recursive equation `G = X - (F - X) \circ G`, which can be
        solved using iterative methods.

        .. WARNING::

            This algorithm is functional but can be very slow.
            Use with caution!

        .. SEEALSO::

            The compositional inverse `\Omega` of the species `E_{+}`
            of nonempty sets can be handled much more efficiently
            using specialized methods. These are implemented in
            :class:`~sage.combinat.species.combinatorial_logarithm.CombinatorialLogarithmSeries`.

        AUTHORS:

        - Andrew Gainer-Dewar
        """
        cisr = self.parent()
        sfa = cisr._base

        X = cisr([0, sfa([1]), 0])

        if self.coefficients(2) != X.coefficients(2):
            raise ValueError('not an invertible series')

        res = cisr()
        res.define(X - (self - X).compose(res))

        return res

    def derivative(self, order=1):
        r"""
        Return the species-theoretic nth derivative of ``self``, where n is ``order``.

        For a cycle index series `F (p_{1}, p_{2}, p_{3}, \dots)`, its derivative is the cycle index series
        `F' = D_{p_{1}} F` (that is, the formal derivative of `F` with respect to the variable `p_{1}`).

        If `F` is the cycle index series of a species `S` then `F'` is the cycle index series of an associated
        species `S'` of `S`-structures with a "hole".

        EXAMPLES:

        The species `E` of sets satisfies the relationship `E' = E`::

            sage: E = species.SetSpecies().cycle_index_series()
            sage: E.coefficients(8) == E.derivative().coefficients(8)
            True

        The species `C` of cyclic orderings and the species `L` of linear orderings satisfy the relationship `C' = L`::

            sage: C = species.CycleSpecies().cycle_index_series()
            sage: L = species.LinearOrderSpecies().cycle_index_series()
            sage: L.coefficients(8) == C.derivative().coefficients(8)
            True

        """

        # Make sure that order is integral
        order = Integer(order)

        if order < 0:
            raise ValueError("Order must be a non-negative integer")

        elif order == 0:
            return self

        elif order == 1:
            parent = self.parent()
            derivative_term = lambda n: parent.term(self.coefficient(n+1).derivative_with_respect_to_p1(), n)
            return parent.sum_generator(derivative_term(i) for i in _integers_from(0))

        else:
            return self.derivative(order-1)

    def pointing(self):
        r"""
        Return the species-theoretic pointing of ``self``.

        For a cycle index `F`, its pointing is the cycle index series `F^{\bullet} = p_{1} \cdot F'`.

        If `F` is the cycle index series of a species `S` then `F^{\bullet}` is the cycle index series of an associated
        species `S^{\bullet}` of `S`-structures with a marked "root".

        EXAMPLES:

        The species `E^{\bullet}` of "pointed sets" satisfies `E^{\bullet} = X \cdot E`::

            sage: E = species.SetSpecies().cycle_index_series()
            sage: X = species.SingletonSpecies().cycle_index_series()
            sage: E.pointing().coefficients(8) == (X*E).coefficients(8)
            True

        """
        p1 = self.base_ring()([1])
        X = self.parent()([0, p1, 0])

        return X*self.derivative()

    def integral(self, *args):
        """
        Given a cycle index `G`, it is not in general possible to recover a single cycle index `F`
        such that `F' = G` (even up to addition of a constant term).

        More broadly, it may be the case that there are many non-isomorphic species `S` such that
        `S' = T` for a given species `T`.
        For example, the species `3 C_{3}` of 3-cycles from three distinct classes
        and the species `X^{3}` of 3-sets are not isomorphic, but `(3 C_{3})' = (X^{3})' = 3 X^{2}`.

        EXAMPLES::

            sage: C3 = species.CycleSpecies(size=3).cycle_index_series()
            sage: X = species.SingletonSpecies().cycle_index_series()
            sage: (3*C3).derivative().coefficients(8) == (3*X^2).coefficients(8)
            True
            sage: (X^3).derivative().coefficients(8) == (3*X^2).coefficients(8)
            True

        .. WARNING::

            This method has no implementation and exists only to prevent you from doing something
            strange. Calling it raises a ``NotImplementedError``!

        """

        raise NotImplementedError

    def exponential(self):
        r"""
        Return the species-theoretic exponential of ``self``.

        For a cycle index `Z_{F}` of a species `F`, its exponential is the cycle index series
        `Z_{E} \\circ Z_{F}`, where `Z_{E}` is the :meth:`~sage.combinat.species.generating_series.ExponentialCycleIndexSeries`.

        The exponential `Z_{E} \circ Z_{F}` is then the cycle index series of the species `E \\circ F` of
        "sets of `F`-structures".

        EXAMPLES:

        Let `BT` be the species of binary trees, `BF` the species of binary forests, and
        `E` the species of sets. Then we have `BF = E \circ BT`::

            sage: BT = species.BinaryTreeSpecies().cycle_index_series()
            sage: BF = species.BinaryForestSpecies().cycle_index_series()
            sage: BT.exponential().isotype_generating_series().coefficients(8) == BF.isotype_generating_series().coefficients(8)
            True

        """
        base_ring = self.parent().base_ring().base_ring()
        E = ExponentialCycleIndexSeries(base_ring)
        return E.compose(self)

    def logarithm(self):
        r"""
        Return the combinatorial logarithm of ``self``.

        For a cycle index `Z_{F}` of a species `F`, its logarithm is the cycle index series
        `Z_{\Omega} \circ Z_{F}`, where `Z_{\Omega}` is the
        :meth:`~sage.combinat.species.generating_series.LogarithmCycleIndexSeries`.

        The logarithm `Z_{\Omega} \circ Z_{F}` is then the cycle index series of the (virtual) species
        `\Omega \circ F` of "connected `F`-structures".
        In particular, if `F = E^{+} \circ G` for `E^{+}` the species of nonempty sets and `G`
        some other species, then `\Omega \circ F = G`.

        EXAMPLES:

        Let `G` be the species of nonempty graphs and  `CG` be the species of nonempty connected
        graphs. Then `G = E^{+} \circ CG`, so `CG = \Omega \circ G`::

            sage: G = species.SimpleGraphSpecies().cycle_index_series() - 1
            sage: from sage.combinat.species.generating_series import LogarithmCycleIndexSeries
            sage: CG = LogarithmCycleIndexSeries().compose(G)
            sage: CG.isotype_generating_series().coefficients(8)
            [0, 1, 1, 2, 6, 21, 112, 853]
        """

        base_ring = self.parent().base_ring().base_ring()
        Omega = LogarithmCycleIndexSeries(base_ring)
        return Omega.compose(self)

@cached_function
def _exp_term(n, R = RationalField()):
    """
    Compute the order-n term of the cycle index series of the species `E` of sets.

    EXAMPLES::

        sage: from sage.combinat.species.generating_series import _exp_term
        sage: [_exp_term(i) for i in range(4)]
        [p[], p[1], 1/2*p[1, 1] + 1/2*p[2], 1/6*p[1, 1, 1] + 1/2*p[2, 1] + 1/3*p[3]]
    """
    p = SymmetricFunctions(R).power()
    return sum(p(part) / part.aut() for part in Partitions(n))

def _exp_gen(R = RationalField()):
    """
    Produce a generator which yields the terms of the cycle index series of the species `E` of sets.

    EXAMPLES::

        sage: from sage.combinat.species.generating_series import _exp_gen
        sage: g = _exp_gen()
        sage: [next(g) for i in range(4)]
        [p[], p[1], 1/2*p[1, 1] + 1/2*p[2], 1/6*p[1, 1, 1] + 1/2*p[2, 1] + 1/3*p[3]]
    """
    return (_exp_term(i, R) for i in _integers_from(0))

@cached_function
def ExponentialCycleIndexSeries(R = RationalField()):
    """
    Return the cycle index series of the species `E` of sets.

    This cycle index satisfies

    .. MATH::

        Z_{E} = \\sum_{n \\geq 0} \\sum_{\\lambda \\vdash n} \\frac{p_{\\lambda}}{z_{\\lambda}}.

    EXAMPLES::

        sage: from sage.combinat.species.generating_series import ExponentialCycleIndexSeries
        sage: ExponentialCycleIndexSeries().coefficients(5)
        [p[], p[1], 1/2*p[1, 1] + 1/2*p[2], 1/6*p[1, 1, 1] + 1/2*p[2, 1] + 1/3*p[3], 1/24*p[1, 1, 1, 1] + 1/4*p[2, 1, 1] + 1/8*p[2, 2] + 1/3*p[3, 1] + 1/4*p[4]]
    """
    CIS = CycleIndexSeriesRing(R)
    return CIS(_exp_gen(R))

@cached_function
def _cl_term(n, R = RationalField()):
    r"""
    Compute the order-n term of the cycle index series of the virtual species `\Omega`,
    the compositional inverse of the species `E^{+}` of nonempty sets.

    EXAMPLES::

        sage: from sage.combinat.species.generating_series import _cl_term
        sage: [_cl_term(i) for i in range(4)]
        [0, p[1], -1/2*p[1, 1] - 1/2*p[2], 1/3*p[1, 1, 1] - 1/3*p[3]]
    """
    n = Integer(n)  # check that n is an integer

    p = SymmetricFunctions(R).power()

    res = p.zero()
    if n == 1:
        res = p([1])
    elif n > 1:
        res = 1/n * ((-1)**(n-1) * p([1])**n - sum(d * p([n // d]).plethysm(_cl_term(d, R)) for d in divisors(n)[:-1]))

    return res

def _cl_gen (R = RationalField()):
    r"""
    Produce a generator which yields the terms of the cycle index series of the virtual species
    `\Omega`, the compositional inverse of the species `E^{+}` of nonempty sets.

    EXAMPLES::

        sage: from sage.combinat.species.generating_series import _cl_gen
        sage: g = _cl_gen()
        sage: [next(g) for i in range(4)]
        [0, p[1], -1/2*p[1, 1] - 1/2*p[2], 1/3*p[1, 1, 1] - 1/3*p[3]]
    """
    return (_cl_term(i, R) for i in _integers_from(0))

@cached_function
def LogarithmCycleIndexSeries(R = RationalField()):
    r"""
    Return the cycle index series of the virtual species `\Omega`, the compositional inverse
    of the species `E^{+}` of nonempty sets.

    The notion of virtual species is treated thoroughly in [BLL]_. The specific algorithm used
    here to compute the cycle index of `\Omega` is found in [Labelle]_.

    EXAMPLES:

    The virtual species `\Omega` is 'properly virtual', in the sense that its cycle index
    has negative coefficients::

        sage: from sage.combinat.species.generating_series import LogarithmCycleIndexSeries
        sage: LogarithmCycleIndexSeries().coefficients(4)
        [0, p[1], -1/2*p[1, 1] - 1/2*p[2], 1/3*p[1, 1, 1] - 1/3*p[3]]

    Its defining property is that `\Omega \circ E^{+} = E^{+} \circ \Omega = X` (that is, that
    composition with `E^{+}` in both directions yields the multiplicative identity `X`)::

        sage: Eplus = sage.combinat.species.set_species.SetSpecies(min=1).cycle_index_series()
        sage: LogarithmCycleIndexSeries().compose(Eplus).coefficients(4)
        [0, p[1], 0, 0]

    REFERENCES:

    .. [Labelle] \G. Labelle. "New combinatorial computational methods arising from pseudo-singletons." DMTCS Proceedings 1, 2008.
    """
    CIS = CycleIndexSeriesRing(R)
    return CIS(_cl_gen(R))


# -*- coding: utf-8 -*-
import os, unittest
from io import BytesIO
from chappie.client.aws import S3Client

class TestS3Client(unittest.TestCase):
    def setUp(self):
        """ """
        self.client = S3Client()
        self.client.bucket = "bucket-test-w37ryifgckcbjqiu4yrf78qy3874yr8cwfc"

        self.params_dict = {
            "Service": 's3'
        }
        file_obj = BytesIO(b'test')
        self.params_dict = self.client.upload_file(file_obj, 'chappie-test/tmp/', 'test.txt', self.params_dict)

    def test_file_upload(self):
        """Test upload of file data"""

        self.assertIsInstance(self.params_dict, dict)
        self.assertEqual(self.params_dict.get('Bucket'), self.client.bucket)
        self.assertEqual(self.params_dict.get('Key'), 'chappie-test/tmp/test.txt')

    def test_file_download(self):
        """Test download of file data"""

        filedata = self.client.download_file(self.params_dict)

        self.assertIsInstance(filedata, bytes)
        self.assertEqual(filedata.decode(), 'test')

    def test_get_url(self):
        """Test get file access url"""

        url = self.client.get_url(self.params_dict)
        self.assertGreater(len(url), 0)

    def test_delete_file(self):
        """Test file delete"""

        status = self.client.delete_file(self.params_dict)


#!/usr/bin/env python
"""
    This code reduces the dimensionality of a dataset using t-SNE.
    It has several options:
        * --exp_name: Name of the experiment. Useful if you want to do several
        experiments with different target dimensions.
        * --representations_file: Pth file containing the data to reduce. This file
        is one of the representation_files generated  by the code "feature_extraction.py"
        that are usually stored in "../models/MNIST_EXP_ID/CompressedRepresentations/"
        * --target_dim: Dimension wanted for the dimensionality reduction

    It generates various folders in ../models/MNIST_EXP_ID/Projections/ named
    EmbeddedRepresentations_perpVal_lrVal_earlyExVal_dimVal. These folders contains
    five files:
        - labels_ID.pth: List of labels of the different 2D/3D points generated
        by t-SNE.
        - originalImages_ID.pth: List of images representing the different 2D/3D
        points generated by TSNE.
        - representations_ID.pth: List of the 2D/3D points generated by t-SNE.
        - tsne_params.json: Parameters used to obtain the projected points.
        - dataSplits_ID.pth: List indicating if samples i is from the training
        set or the testing set.
"""
import os
import argparse
import numpy as np
from sklearn.manifold import TSNE
import pickle
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as plt
import pickle
from matplotlib import image
import json

def rearangeData(data):
    """
        Function that puts the data of the list of points under the right format:

        Arguments:
        ----------
        data: list
            List where each element is a dict that correspond to one epoch of the training
            process. Each dict contains two keys:
                - 'compressed_representation': List where each element is the
                output of a batch for the given epoch.
                - 'label': List where each element are the labels of the output of a
                batch for a given epoch

        Returns:
        --------
        newData: list
            List containing the data points to plot. The elements of the list
            are dictionaries with two keys:
                - 'compressed_representation': List where each element is one
                output for that epoch
                - 'label': List where each element is the label of the corresponding
                output point in the output list
    """
    newData = []
    for epochData in data:
        newOutput, newOriginalImage, newLabel, newDataSplits = [], [], [], []
        # Labels
        for labelLists in epochData['label']:
            nbSamplesBatch = len(labelLists)# It's the same as len(labelLists[1]), ..,. len(labelLists[9])
            for i in range(nbSamplesBatch):
                newLabel.append(int(labelLists[i]))
        # Auto-encoder representations
        for outputLists in epochData['compressed_representation']:
            for output in outputLists:
                newOutput.append(output.cpu().detach().numpy())
        # Original images
        for original_image_list in epochData['original_image']:
            for originalImage in original_image_list:
                if (type(originalImage) == str):
                    sample_data = np.asarray(Image.open(originalImage))/255.
                    if (len(sample_data.shape) == 3):
                        sample_data = np.moveaxis(sample_data, 2, 0) # Because in pytorch the channel has to be in the first position
                    else:
                        sample_data = sample_data.reshape((1, sample_data.shape[0], sample_data.shape[1]))
                    image = torch.tensor(sample_data)
                    originalImage = image.type("torch.FloatTensor") # To avoid problems with types
                newOriginalImage.append(originalImage.cpu().detach().numpy())
        # Split type of the sample (train or test splits)
        if ('data_split' in epochData):
            for dataSplitLists in epochData['data_split']:
                for dataSplit in dataSplitLists:
                    newDataSplits.append(dataSplit)
        newData.append({'compressed_representation': newOutput, 'original_image': newOriginalImage, 'label': newLabel, 'data_split':newDataSplits})
    return newData

def main():
    # =========================================================================#
    # Construct the argument parser
    ap = argparse.ArgumentParser()
    # Add the arguments to the parser
    ap.add_argument("--exp_name", default='tSNE_GridSearch', help="Name of the experiment", type=str)
    ap.add_argument("--representations_file", required=False, help="Pth file containing the data to reduce", type=str)
    ap.add_argument("--target_dim", required=False, default=2, help="Target dimension of the embedded representation (2 or 3)", type=int)
    ap.add_argument("--use_default_params_tsne", default=False, help="True if want to use the default parameters for t-SNE instead of doing a GridSearch", type=bool)
    args = vars(ap.parse_args())

    # Points file and target dim
    exp_name = args['exp_name']
    representations_file = args['representations_file']
    target_dim = args['target_dim']
    use_default_params_tsne = args['use_default_params_tsne']

    #==========================================================================#
    # Creating the folder with the projections
    # =========================================================================#
    model_folder = '/'.join(representations_file.split('/')[:-2])
    projections_folder = model_folder + '/Projections_{}_'.format(exp_name)
    inc = 0
    while (os.path.isdir(projections_folder + str(inc))):
        inc += 1
    projections_folder = projections_folder + str(inc) + '/'
    os.mkdir(projections_folder)

    #==========================================================================#
    # Parameters for the tgrid search experiment
    # =========================================================================#
    if (use_default_params_tsne):
        perplexities = [30.0]
        early_exaggerations = [12.0]
        learning_rates = [200]
        n_iter = 1000
    else:
        # perplexities = [5, 10, 15, 20, 25, 30, 35, 40, 45, 50]
        perplexities = [10, 30, 50]
        # early_exaggerations = [5, 10, 25, 50, 75, 100, 200, 500]
        early_exaggerations = [50, 250, 500]
        # learning_rates = [10, 50, 100, 500, 1000]
        learning_rates = [10, 100, 1000]
        n_iter = 3000

    #==========================================================================#
    #==========================================================================#
    #==========================================================================#
    for perplexity in perplexities:
        for early_exaggeration in early_exaggerations:
            for learning_rate in learning_rates:
                parameters_dict = {
                                    'perplexity': perplexity,
                                    'early_exaggeration': early_exaggeration,
                                    'learning_rate': learning_rate,
                                    'n_iter': n_iter,
                                    'n_jobs': 4
                                  }

                # Loading the representations
                with open(representations_file, "rb") as fp:   # Unpickling
                    data = pickle.load(fp)

                # Seing if there is any data to process
                noData = True
                for element in data:
                    if len(element['compressed_representation']) > 0:
                        noData = False
                        break
                if (noData):
                    print("The file {} does not contain any compressed representations to reduce".format(representations_file))
                else:
                    # Rearanging data
                    data = rearangeData(data)

                    final_representations = data[-1] # To only get the representations obtained
                    # at the last epoch

                    #==========================================================================#
                    # Applying t-SNE
                    representation_list = []
                    for representation in final_representations['compressed_representation']:
                        if (len(representation.shape) == 3): # This is needed becayse
                        # the input of t-SNE should be a vector
                            representation_list.append(representation.reshape(representation.shape[0]*representation.shape[1]*representation.shape[2]))
                        elif (len(representation.shape) == 1):
                            representation_list.append(representation)
                        else:
                            raise ValueError("Wrong dimensions for the representations: {}".format(representation.shape))
                    representation_list = np.array(representation_list)

                    print("=======> Starting TSNE")
                    if (target_dim == 3):
                        tsneInstance = TSNE(n_components=3, perplexity=parameters_dict['perplexity'], early_exaggeration=parameters_dict['early_exaggeration'], learning_rate=parameters_dict['learning_rate'], n_jobs=parameters_dict['n_jobs'])
                    elif (target_dim == 2):
                        tsneInstance = TSNE(n_components=2, perplexity=parameters_dict['perplexity'], early_exaggeration=parameters_dict['early_exaggeration'], learning_rate=parameters_dict['learning_rate'], n_jobs=parameters_dict['n_jobs'])
                    else:
                        raise ValueError("Dimension {} for the embedded representation is not valid".format(target_dim))
                    embedded_representations = tsneInstance.fit_transform(representation_list)
                    print("The method converged in {} iterations".format(tsneInstance.n_iter))
                    print("=======> Ending TSNE")

                    # Creating the folder where we are going to store the different files of the
                    # embedded representations
                    inc = 0
                    folderName = projections_folder +\
                                '/EmbeddedRepresentations_perp{}_lr{}_earlyEx{}_dim{}'.format(parameters_dict['perplexity'],\
                                                                                parameters_dict['learning_rate'],\
                                                                                parameters_dict['early_exaggeration'],\
                                                                                target_dim)
                    while (os.path.isdir(folderName + '_' + str(inc))):
                        inc += 1
                    folderName = folderName + '_' + str(inc)
                    os.mkdir(folderName)
                    print("Saving Embedded Representations at: {}".format(folderName))
                    # Saving the embedded points got with T-SNE
                    embReprFileName = folderName + "/representations"
                    inc = 0
                    while os.path.isfile(embReprFileName+'_'+str(inc)+'.pth'):
                        inc +=1
                    embReprFileName += '_'+str(inc)+'.pth'
                    with open(embReprFileName, "wb") as fp:   #Pickling
                        pickle.dump(embedded_representations, fp)
                    # Saving the original images
                    originalImagesFileName = folderName + "/images"
                    inc = 0
                    while os.path.isfile(originalImagesFileName+'_'+str(inc)+'.pth'):
                        inc +=1
                    originalImagesFileName += '_'+str(inc)+'.pth'
                    with open(originalImagesFileName, "wb") as fp:   #Pickling
                        pickle.dump(np.array(final_representations['original_image']), fp)
                    # Saving the labels
                    labelsFileName = folderName + "/labels"
                    inc = 0
                    while os.path.isfile(labelsFileName+'_'+str(inc)+'.pth'):
                        inc +=1
                    labelsFileName += '_'+str(inc)+'.pth'
                    with open(labelsFileName, "wb") as fp:   #Pickling
                        pickle.dump(final_representations['label'], fp)
                    # Saving the data splits of the samples (i.e. if they are train or test samples)
                    dataSplitsFileName = folderName + "/dataSplits"
                    inc = 0
                    while os.path.isfile(dataSplitsFileName+'_'+str(inc)+'.pth'):
                        inc +=1
                    dataSplitsFileName += '_'+str(inc)+'.pth'
                    with open(dataSplitsFileName, "wb") as fp:   #Pickling
                        pickle.dump(final_representations['data_split'], fp)
                    # Saving the parameters used to train TSNE
                    with open(folderName + '/tsne_params.json', 'w') as outfile:
                        json.dump(parameters_dict, outfile)

if __name__=="__main__":
    main()


from django.urls import path

from . import views

urlpatterns = [
    path('', views.person_list_view, name="list"),
    path('create/', views.create, name="create"),
    path("edit/<int:id>", views.edit, name="edit"),
    path("delete_person/<int:id>", views.delete_person, name="delete_person"),
]


"""

@author: Osama Shakeel

"""

import cv2

path = input("Enter your image Path: \n") # i.e: D:\\music\\ava.jpg
print("The path you entered is == ", path)

#Now Read Image

img = cv2.imread(path,0)  #Convert image to Gray Scale Image
img = cv2.resize(img, (760,540))
img = cv2.flip(img, 1) #it accept three parameter 1,0,-1

cv2.imshow("Flip image ", img)

cv2.waitKey(0)
cv2.destroyAllWindows()

"""
#If you want to save the image than
k = cv2.waitKey(0)
if k == ord("s"):
    cv2.imwrite("D://music//avagrayflip.jpg", img)
else:
    cv2.destroyAllWindows()

"""


from telegram.ext import Updater
import telegram.error

from cfg import config

class Bot:
    def __init__(self):
        self.updater = Updater(token=config.BOT_TOKEN,
                               request_kwargs=config.REQUEST_KWARGS,
                               use_context=True)
        self.channels = config.CHANNELS

    def send_msg(self, msg):
        found = False

        for ch_id, apps in self.channels.items():
            for app, levels in apps.items():
                if msg['service'] == app and msg['level'] in levels:
                    self._send(msg, ch_id)
                    found = True
                    break

        if not found:
            self._send(msg, config.GENERAL_CHANNEL_ID)

    def _send(self, msg, channel_id):
        try:
            self.updater.bot.send_message(channel_id, text=Bot._format_msg(msg), timeout=20)
        except telegram.error.TimedOut:
            print('Couldn\'t connect to telegram.')

    @staticmethod
    def _format_msg(msg):
        res = []
        for k, v in msg.items():
            s = '{0}:   {1} \n'.format(k, v)
            res.append(s)

        return '\n'.join(res)


from collections import deque

def solution(priorities, location):
    L = deque()
    for i in range(len(priorities)):
        L.append((priorities[i], i))
    cnt = 0
    while True:
        if len(L) == 1:
            return cnt+1
        priority, idx = L.popleft()
        for i in range(len(L)):
            if L[i][0] > priority:
                L.append((priority, idx))
                idx = -1
                break
        else:
            cnt += 1
        if idx == location:
            return cnt

print(solution([2, 1, 3, 2],2))
print(solution([1, 1, 9, 1, 1, 1],0))


from django.test import TestCase
from .models import *
from .forms import *

class TestModels(TestCase):

    def setUp(self):
        self.client = Client.objects.create(
            meter_number=1234567,
            first_name='John',
            middle_name='Doe',
            last_name='Smith',
            contact_number='+639123456789',
            address='123 Main St., Anytown, USA',
            status='Connected'
        )

    def test_client_str(self):
        self.assertEqual(str(self.client), 'Smith, John Doe')

    def test_water_bill_payable_with_penalty(self):
        metric = Metric.objects.create(consump_amount=1, penalty_amount=2)
        bill = WaterBill.objects.create(
            name=self.client,
            meter_consumption=100,
            status='Pending',
            duedate=datetime.date.today(),
            penaltydate=datetime.date.today()
        )
        self.assertEqual(bill.payable(), 102)

class TestForms(TestCase):

    def test_bill_form_valid(self):
        form_data = {
            'name': Client.objects.create(
                meter_number=1234567,
                first_name='John',
                middle_name='Doe',
                last_name='Smith',
                contact_number='+639123456789',
                address='123 Main St., Anytown, USA',
                status='Connected'
            ).id,
            'meter_consumption': 100,
            'status': 'Pending',
            'duedate': datetime.date.today(),
            'penaltydate': datetime.date.today()
        }
        form = BillForm(data=form_data)
        self.assertTrue(form.is_valid())

    def test_client_form_valid(self):
        form_data = {
            'meter_number':1234567,
            'first_name': 'John',
            'middle_name': 'Doe',
            'last_name': 'Smith',
            'contact_number': '+639123456789',
            'address': '123 Main St., Anytown, USA',
            'status': 'Connected'
        }
        form = ClientForm(data=form_data)
        self.assertTrue(form.is_valid())

    def test_metrics_form_valid(self):
        form_data = {
            'consump_amount': 1,
            'penalty_amount': 2
        }
        form = MetricsForm(data=form_data)
        self.assertTrue(form.is_valid())


#Authors: Luca Bolognini & Matteo Antonio Inajetovic

from matplotlib.patches import Rectangle
import matplotlib.pyplot as plt
import os
import numpy as np
from numpy.core.fromnumeric import argmax
import pandas as pd
################# Parameters #########################
path = './Instances/'
data_path='./Instances_fin/'
output_smt_path_ext='./smt_out_extended/'
output_smt_path_base='./smt_out_base/'
elem = os.listdir(path)

###################### Methods ######################
def data_file_reader (read_path, file, sol = False):
    print('Reading Data files, wait...\n')

    with open(read_path + file, 'rt', ) as f:
        if not sol:
            line = f.readlines()
            lines= [line[i].split() for i in range(len(line))]
            widths = [lines[i][0] for i in range(2,len(lines)-2)]
            heights  = [lines[i][1] for i in range(2,len(lines)-2)]
            posx = None
            posy = None
        else:
            line = f.readlines()
            lines= [line[i].split() for i in range(len(line))]
            widths = [lines[i][0] for i in range(2,len(lines))]
            heights  = [lines[i][1] for i in range(2,len(lines))]
            posx = [lines[i][2] for i in range(2,len(lines))]
            posy = [lines[i][3] for i in range(2,len(lines))]

    w = lines[0][0]
    h = lines[0][1]
    n = lines[1][0]

    return w, h, n, widths, heights, posx, posy

def output_writer(read_path, write_path, sol_source, ext='.txt', rot =False):

    if rot:
        fl= pd.read_excel(io= sol_source, names=['X','Y','R'])
    else:
        fl= pd.read_excel(io= sol_source, names=['X','Y'])
    r = 0
    for file in os.listdir(read_path):
        w, h, n, widths, heights,_,_ = data_file_reader(read_path, file)
        x = fl['X'][r].split(',')
        y = fl['Y'][r].split(',')
        if rot:
            rotation = fl['R'][r].split('r')
            rotation.reverse()
        x.reverse()
        y.reverse()
        with open(write_path + w +'x'+ h +'-out'+ ext, 'w') as out:
            out.write( w +' '+ h +'\n'
                     + n + '\n')
            for i in range(0, int(n)):
                if not rot:
                    out.write(widths[i] +' '+ heights[i] +'  '+ x[i] +' '+ y[i]+'\n')
                else:
                    out.write(widths[i] +' '+ heights[i] +'  '+ x[i] +' '+ y[i]+ rotation[i] +'\n')
        r+=1

def solution_output (path, file_name = "every"):

    """Path = directory containing the solution
     File name = "LxL-out.txt" where l is the paper length, if default or "every" then all solution are sequentially inspected"""

    if file_name == "every":
        for file in os.listdir(path):
            output_visualizer(path, file)
    else:
            output_visualizer(path, file_name)

def output_visualizer (out_path, file_name = "8x8-out.txt", border_reduction = 0.15):

    """
    visualizes the solutions given the path containing the output instances with the positions solutions
    """
    dim, _, _, width, height,posx,posy = data_file_reader(out_path, file_name, sol = True)

    colors = ['blue','green','yellow','red','orange',
              'purple','black','pink', 'cyan','gold',
              'silver','indigo', 'aquamarine', 'beige',
              'magenta','teal','aqua', 'lavender',
              'crimson', 'orchid']

    rotations = np.zeros(len(posx))

    fig = plt.figure()
    ax = fig.add_subplot(111)

    for i in range(len(posx)):
        if rotations[i]:
            rect = Rectangle((int(posx[i]), int(posy[i])), float(height[i])-border_reduction, float(width[i])-border_reduction, color = colors[i % len(colors)], ec ='black', label = i+1)
        else:
            rect = Rectangle((int(posx[i]), int(posy[i])), float(width[i])-border_reduction, float(height[i])-border_reduction, color = colors[i % len(colors)], ec ='black', label = i+1)

        ax.add_patch(rect)

    plt.title(dim + 'x' + dim)
    plt.xlim(-0.15,float(dim) +1.25)
    plt.ylim(-0.15,float(dim) +1.25)
    #plt.legend()
    plt.show()

def solution_visualizer (posx, posy, width, height, dim,rotations):
    """
    visualizes a solution given the positions obtained with the solver
    """
    colors = [  'blue','green','yellow','red','orange',
                'purple','black','pink', 'cyan','gold',
                'aqua','indigo', 'aquamarine', 'beige',
                'magenta','teal','silver', 'lavender',
                'crimson', 'orchid']

    fig = plt.figure()
    ax= fig.add_subplot(111)

    for i in range(len(posx)):
       if rotations[i]==True:
           print('Present {} with w = {} and h = {} has been rotated'.format(i,width[i], height[i]))
           rect = Rectangle((posx[i], posy[i]),height[i] , width[i], color = colors[i % len(colors)],label=i, ec ='white')
           ax.add_patch(rect)
       else:
           rect = Rectangle((posx[i], posy[i]), width[i], height[i], color = colors[i % len(colors)], label=i,ec ='white')
           ax.add_patch(rect)

    plt.xlim(0,dim)
    plt.ylim(0,dim)
    plt.legend()
    plt.show()

def reordering(sizes,widths,heights,resx,resy,n):
    """
    function implemented to reorder the positions results when using the SB constraint
    to output the results sorted like the input instances.
    """
    new_widths = [i for i,_ in sizes]
    new_heights = [i for _,i in sizes]

    posx=[]
    posy=[]
    for a,b in zip(widths,heights):
        for i in range(n):
            if a == new_widths[i] and b == new_heights[i]:
                x=resx[i]
                y=resy[i]
        posx.append(x)
        posy.append(y)

    return posx,posy

###########CALL UTILS#############
solution_output(output_smt_path_ext, "every")

#print('DONE')


from contextlib import contextmanager
from logging import Logger

@contextmanager
def logging_exception(exc_msg: str, logger: Logger) -> None:
    """Wrap any code block with this context manager to catch exceptions occurring in it and log them"""
    try:
        yield None
    except Exception as e:
        logger.exception(exc_msg)


#!/bin/env python3
"""Create beautiful, heavily customizable progress bars... probarly"""
import time, os
from typing import Any, Dict, List, TypeVar, Union

os.system("")  # enable color support

AnimationType = TypeVar("AnimationType")
TaskType = TypeVar("TaskType")

COLOURS = {
    "RED": "\033[0m\033[0;31m",
    "YELLOW": "\033[0m\033[0;33m",
    "GREEN": "\033[0m\033[0;32m",
    "BLUE": "\033[0m\033[0;34m",
    "CYAN": "\033[0m\033[0;36m",
    "PURPLE": "\033[0m\033[0;35m",
}

class Task:
    """Define a task."""

    def __init__(self, text: str, id_: int):
        """Store the id and the text of a task."""
        self.id_ = id_
        self.text = text

    def __hash__(self) -> int:
        return hash(self.id_)

    def __eq__(self, other: Any) -> bool:
        if not isinstance(other, self.__class__):
            return False
        return self.id_ == other.id_

    def __ne__(self, other: Any) -> bool:
        return not self.__eq(other)

class Animation:
    """Define an animation. (Don't use to make a custom bar animation)"""

    def __init__(self, frames: List[str], slow: int = 5):
        """Store the speed and frames of the animation."""
        self.frames = frames
        self.slow = slow

class BarAnimations:
    """Contains animations for the progress bar."""

    classic_fill = [" ", "▏", "▎", "▍", "▌", "▋", "▊", "▉", "█"]
    hash_ = [" ", "#"]

class OtherAnimations:
    """Contains animations for after the progress bar itself."""

    up_down = ["▁", "▂", "▃", "▄", "▅", "▆", "▇", "█"]
    spinner_clockwise = Animation(["/", "-", "\\", "|"], 10)
    spinner_counterclockwise = Animation(spinner_clockwise.frames[::-1], 10)
    morph = Animation(BarAnimations.classic_fill + up_down[::-1])
    up_down = Animation(up_down + up_down[::-1])

class Progbar:
    """The progress bar."""

    def __init__(
        self,
        length: int = 10,
        filler: str = " ",
        start: str = "|",
        end: str = "|",
        frames: List[str] = BarAnimations.classic_fill,
        animations: Dict[Union[int, str], AnimationType] = {
            "*": OtherAnimations.up_down
        },
        colours: Dict[Union[int, str], str] = {"*": ""},
    ):
        """
        How to use:
          - length: specifies length of the progress bar (default 10)
          - filler: what to fill the rest of the progress bar with (default ' ')
          - start: what the start of the progress bar should look like (default '|')
          - end: what the end of the progress bar should look like (default '|')
          - frames: frames for the animation of the progress bar (default BarAnimations.classic_fill)
          - animations: how to use:
             - you can use any task id to specify the animation to use when that task is active
             - an id of -1 is reserved for tasks with no text
             - you can use a '*' to denote all tasks that are not specified in the dict
          - colours:
             - you can use any task id to specify the colour to use when that task is active
             - an id of -1 is reserved for tasks with no text
             - you can use a '*' to denote all tasks that are not specified in the dict
        """
        self.__animation = Animation([""])
        self.__animation_frame = 0
        self.__colour = ""
        self.__frame_index = 0
        self.__full = 0
        self.animations = animations
        self.bar = [frames[0]]
        self.colours = colours
        self.end = end
        self.filler = filler
        self.frames = frames
        self.length = length
        self.start = start
        self.task = Task("", -1)

    def show(self) -> None:
        """Show the current state of the bar."""
        print(
            f"\x1b[1K\r{self.task.text}{self.start}{self.__colour}{''.join(self.bar)}{self.filler * (self.length - self.__full - 1)}\033[0m{self.end} {int(round((self.__full/self.length) * 100))}% {self.__animation.frames[self.__animation_frame]}",
            end="",
        )

    def adv(
        self,
        to: int,
        time_: float = 0.005,
        task: TaskType = Task("", -1),
        show_newline=True,
        show=True,
    ) -> None:
        """
        How to use:
          - to: a percentage; the bar will go forward that amount
          - time_: the time between switching frames of the progress bar (default 0.005)
          - task: specify a task to show what you are currently doing (default '')
          - show_newline: show newline after bar reaches 100% (default True)
          - show: actually show the bar (default True)
        """
        self.task = task
        if task.id_ in self.colours:
            self.__colour = self.colours[task.id_]
        elif "*" in self.colours:
            self.__colour = self.colours["*"]
        else:
            self.__colour = ""
        for _ in range(int(round((to / 100) * self.length)) * len(self.frames)):
            self.wait(time_)
            if self.bar[-1] in self.frames[:-1]:
                self.bar[-1] = self.frames[self.frames.index(self.bar[-1]) + 1]
            elif self.bar[-1] == self.frames[-1]:
                if (self.length - self.__full - 1) > 0:
                    self.bar.append(self.frames[0])
                else:
                    if self.length == self.__full + 1:
                        self.__full += 1
                    self.show()
                    if show_newline:
                        print()
                    break
                self.__full += 1
            self.show()

    def wait(self, time_: float) -> None:
        """
        How to use:
          - time_: time to wait for
        """
        if self.task.id_ in self.animations:
            self.__animation = self.animations[self.task.id_]
        elif "*" in self.animations:
            self.__animation = self.animations["*"]
        else:
            self.__animation = Animation([""])
        slow = self.__animation.slow
        while time_ >= 0:
            time.sleep(0.005)
            time_ -= 0.005
            self.__frame_index += 1
            if not self.__frame_index % slow:
                self.__animation_frame = (self.__animation_frame + 1) % len(
                    self.__animation.frames
                )
            self.show()

    def reset(self, show_: bool = False) -> None:
        """
        How to use:
          - show_: change to True if you want to show the bar's state after the reset
        """
        self.__animation_frame = 0
        self.__frame_index = 0
        self.__full = 0
        self.bar = [self.frames[0]]
        self.task = ""
        if show_:
            self.show()


import sys, os, stat, string, urllib
import time

from www.web_types import HTMLResource, FreevoResource
import util
import config
import mimetypes
import kaa.metadata as metadata
from twisted.web import static

class Html5WinResource(FreevoResource):

    def __init__(self):
        #print 'Html5WinResource.__init__(self)'
        self.allowed_dirs = []
        self.allowed_dirs.extend(config.VIDEO_ITEMS)
        self.allowed_dirs.extend(config.AUDIO_ITEMS)
        self.allowed_dirs.extend( [ ('Recorded TV', config.TV_RECORD_DIR) ])
        self.allowed_dirs.extend(config.IMAGE_ITEMS)

    def _render(self, request):
        #print '_render(self, %s)' % (request)
        fv = HTMLResource()
        form = request.args
        file = fv.formValue(form, 'dir')

        if file:
            file_link = self.convert_dir(file)
            #print 'file_link=%s' % (file_link)
            (mimetype, encoding) = mimetypes.guess_type(file_link)

            fv.res += (
               u"\n" \
	       u"<video controls='controls' poster='http://sandbox.thewikies.com/vfe-generator/images/big-buck-bunny_poster.jpg' width='640' height='480'>\n" \
               u"<source src='/%s' type='%s' />\n" ) % (urllib.quote(file_link), mimetype)

            fv.res += (
                u"</video>\n" \
                u"\n" \
            )

        return String(fv.res)

    def convert_dir(self, dir_str):
        #print 'convert_dir(self, dir_str=%r)' % (dir_str)
        for i in range(len(self.allowed_dirs)):
            val = self.allowed_dirs[i][1]
            if dir_str.startswith(val):
                child_res = val.replace("/", "_")
                location = dir_str[len(val):]
                if not location[0] == "/":
                    child_res += "/"
                return child_res + location
        return dir_str

resource = Html5WinResource()


### Hello ###

### Name of AI is under development ###

### AI itself is under development ###

# There are resources for voice recognition and speech #
import pyttsx3

engine = pyttsx3.init('sapi5')
voices = engine.getProperty('voices')
engine.setProperty('voice', voices[0].id)

def speak():
    engine.say("abc")
    engine.runAndWait()

speak()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Created on Thu Feb 05 11:00:28 2015

@author: jhchen
"""
import gdal,ogr    ### osheo can't be imported with pyhdf
import osgeo
import gdalinfo
import numpy as np
import matplotlib.pyplot as plt
### this is a fuction to obtain the lon and lat in gdal
def pixel2coord(x, y,geoform):
    """Returns global coordinates from pixel x, y coords"""
    xoff=geoform[0]
    a=geoform[1]
    b=geoform[2]
    yoff=geoform[3]
    d=geoform[4]
    e=geoform[5]
    xp = a * x + b * y + xoff
    yp = d * x + e * y + yoff
    return(xp, yp)
# xoff, a, b, yoff, d, e = ds.GetGeoTransform()
dirin="D:/MyPaper/PhD04/Cases/ETP/20100604_0704/MODIS/ETP20100604-32d/"
fnm="MYD08_D3.A2010154.006.2014264200556.hdf"
fpath=dirin+fnm
modis=gdal.Open(fpath)
modisinfo=gdalinfo(fpath)
listvar=modis.GetSubDatasets()
lc_data=gdal.Open(listvar[253][0])
lc=lc_data.ReadAsArray()
nx=lc_data.RasterXSize
ny=lc_data.RasterYSize
geoform=lc_data.GetGeoTransform()
lon=np.ndarray(shape=(nx,ny), dtype=float)
lat=np.ndarray(shape=(nx,ny), dtype=float)
for i in range(0,nx):
    for j in range(0,ny):
        lon[i,j],lat[i,j] =pixel2coord(i,j,geoform)
minlat=int(lat.min())
maxlat=int(lat.max())
minlon=int(lon.min())
maxlon=int(lon.max())
lonlabs=[]
latlabs=[]
for i in range(0,nx):
    itm="%d"%int(lon[i,0])
    lonlabs.append(itm)
for i in range(0,ny):
    itm="%d"%int(lat[0,i])
    latlabs.append(itm)
fig,ax = plt.subplots(nrows=1,ncols=1)
#ax.set_ylim(35,80)
#ax.set_xlim(250,350)
plt.imshow(lc[:,:])
ax.set_xticks(range(0,nx,30))
xticklabels = [lonlabs[nn] for nn in range(0,nx,30)]
ax.set_xticklabels(xticklabels, size=14)
ax.set_yticks(range(0,ny,15))
yticklabels = [latlabs[nn] for nn in range(0,ny,15)]
ax.set_yticklabels(yticklabels, size=14)
plt.colorbar()
plt.show()
"""
fpath=dirin+"MYD08_D3.txt"
i=0
f=open(fpath,'w')
for nm in listvar:
    iband="%d "%i
    f.write(iband)
    itme="%s "%nm[1]
    f.write(itme)
    f.write('\n')
    i+=1
f.close()
"""


import configparser
import psycopg2
from sql_queries_etl_stage import copy_table_queries
from sql_queries_etl_star import insert_table_queries

def load_staging_tables(cur, conn):
    """Extract and Transform S3 files, then load into Redshift Staging Tables."""
    for query in copy_table_queries:
        cur.execute(query)
        conn.commit()

def insert_tables(cur, conn):
    """Extract and Transform Redshift Staging Tables, then load into Redshift STAR-schema Tables."""
    for query in insert_table_queries:
        cur.execute(query)
        conn.commit()

def main():
    """Build staging and STAR-schema tables in one go."""

    config = configparser.ConfigParser()
    config.read('dwh_035_access.cfg')

    conn = psycopg2.connect("host={} dbname={} user={} password={} port={}".format(*config['CLUSTER'].values()))
    cur = conn.cursor()

    load_staging_tables(cur, conn)
    insert_tables(cur, conn)

    conn.close()

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
import dumpling.fields

class Migration(migrations.Migration):

    dependencies = [
        ('contenttypes', '0002_remove_content_type_name'),
        ('dumpling', '0001_initial'),
    ]

    operations = [
        migrations.CreateModel(
            name='PageWidget',
            fields=[
                ('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')),
                ('template', models.FilePathField(recursive=True, path='/Users/curtis/env/holster/site_templates/')),
                ('object_id', models.PositiveIntegerField()),
                ('content_type', models.ForeignKey(to='contenttypes.ContentType')),
            ],
        ),
        migrations.AlterField(
            model_name='page',
            name='template',
            field=dumpling.fields.RelativePathField(recursive=True, path='/Users/curtis/env/holster/site_templates/'),
        ),
        migrations.AddField(
            model_name='pagewidget',
            name='page',
            field=models.ForeignKey(to='dumpling.Page'),
        ),
        migrations.AlterUniqueTogether(
            name='pagewidget',
            unique_together=set([('page', 'template')]),
        ),
    ]


# this file contains all fuctions that have to do with training

# third party imports:

# first party imports:
import sql
import time
import color
import format
from GAUD import get, update, add
import functions

# functions:
# Get
def habits(showOperation, sqloperation):
    result = get.getIntern("habits", ["habits_name", "habits_count", "habits_done"],
                           "habits_count < 100000 ORDER BY habits_done DESC", showOperation)
    color.printBlue("habits:")
    print("")
    entryColors = []
    for row in result:
        if row[2] == "true":
            entryColors.append("green")
        else:
            entryColors.append("red")
    functions.printTable(result, ["name", "count", "done?"],
                         "blue", entryColors, "blue", showIndexes=True)
    print("")
    color.printBlue("add habbit: a | habit done: d")
    next = input("next: ")
    while next == "a" or next == "d":
        if next == "a":
            name = input("habits_name: ")
            description = input("description: ")
            add.addIntern("habits", ["all"], [
                          name, description, "0", "false"], showOperation)
            next = input("next: ")
        elif next == "d":
            number = input("habit number: ")
            sql.execute(
                f"UPDATE habits SET habits_count = habits_count + 1, habits_done = 'true' WHERE habits_name = '{str(result[int(number) - 1][0])}'", showOperation, "post")
            habits(showOperation, sqloperation)

# Add

# Update

# Delete


import os
import json
import logging

import vlc
import pika

from somaradio.soma_station import SomaStation

class SomaPlayer:

    def __init__(self):
        self._logger = logging.getLogger(__name__)
        self._logger.setLevel(logging.DEBUG)

        credentials = pika.PlainCredentials('soma', 'soma')
        parameters = pika.ConnectionParameters(os.environ.get('AMQP_HOST'),
                                       os.environ.get('AMQP_PORT'),
                                       '/',
                                       credentials)
        self._logger.info('connecting with AMQP parameter: %s', parameters)
        self._connection = pika.BlockingConnection(parameters)
        channel = self._connection.channel()
        channel.queue_declare(queue='soma_player')
        channel.basic_consume(queue='soma_player',
                      auto_ack=True,
                      on_message_callback=self.digest_amqp)

        self.vlc_instance = vlc.Instance('--verbose 2'.split())

        # self.vlc_instance = vlc.Instance()

        self.player = self.vlc_instance.media_player_new()
        self.station = None
        self._logger.info("Instantiation successful")

        channel.start_consuming()

    def __del__(self):
        if self.player is not None:
            self.player.stop()
        self._connection.close()

    def digest_amqp(self, ch, method, properties, body: bytes):
        self._logger.info("ch: %s:%s | method: %s:%s | properties: %s:%s | body: %s:%s",
                          ch, type(ch), method, type(method), properties, type(properties), body, type(body))

        payload = json.loads(body.decode('utf-8'))
        self._logger.info("Message payload: %s", payload)
        if 'cmd' in payload:
            if payload['cmd'] == 'stop':
                self.stop()
            elif payload['cmd'] == 'play':
                self.play()
            elif payload['cmd'] == 'set_station':
                try:
                    station = SomaStation(title=payload['station']['title'],
                                        desc=payload['station']['desc'],
                                        playlist_url=payload['station']['playlist_url']
                                        )
                    self.set_station(station)
                except KeyError as exc:
                    self._logger.error(exc)
                self.play()

    def set_station(self, station:SomaStation) -> None:
        self.station = station

    def stop(self):
        self.player.stop()

    def play(self):
        if self.station is not None:
            media = self.vlc_instance.media_new(self.station.tracks[0])
            self.player.set_media(media)
            self.player.play()
        else:
            self._logger.warn("Cannot play without station. Use call play_station first.")


import pyttsx3
import datetime
import speech_recognition as sr
import wikipedia
import smtplib
import webbrowser as wb
import os
import subprocess
import pyautogui

engine = pyttsx3.init()

def speak(audio):
    engine.say(audio)
    engine.runAndWait()

def time():
    Time = datetime.datetime.now().strftime("%I:%M:%S")
    speak("The current time is %s" %Time)

def date():
    Year = int(datetime.datetime.now().year)
    Month = int(datetime.datetime.now().month)
    Day = int(datetime.datetime.now().day)
    speak("Today's date is %d \n %d \n %d" % (Day, Month, Year))

def wishme():
    hour = datetime.datetime.now().hour
    if hour >= 5 and hour < 12:
        speak("Hello!, Good Morning!")
    elif hour >= 12 and hour <= 16:
        speak("Hello!, Good Afternoon")
    else:
        speak("Hello!, Good Evening")
    speak("I am Jarvis, AI assistant of Mr. Sarath")
    #time()
    #date()
    speak("Jarvis at your service \n How can I help you?")

def sendemail(to, content):
    server = smtplib.SMTP('smtp.gmail.com', 587)
    server = ehlo()
    server.starttls()
    server.login('sarathsgvr@gmail.com', '######')
    server.sendmail('sarathsgvr@gmail.com', to, content)
    server.close()

def listenvoice():
    r = sr.Recognizer()
    with sr.Microphone() as source:
        print("Jarvis is listening....")
        r.pause_threshold = 1
        audio = r.listen(source)

    try:
        print("Recognizing...")
        query = r.recognize_google(audio, language='en-in')
        print(query)

    except Exception as e:
        print(e)
        return "None"
    return query

if __name__ == "__main__":
    wishme()
    while True:
        query = listenvoice().lower()

        if 'time' in query:
            time()
        elif 'date' in query:
            date()
        elif 'wikipedia' in query:
            speak('searching')
            query = query.replace('wikipedia', "")
            result = wikipedia.summary(query, sentences=2)
            print(result)
            speak(result)
        elif ('send email' or 'sent email') in query:
            try:
                speak("What is the content?")
                content = listenvoice()
                speak("Please tell me the to address")
                to = listenvoice()
                sendemail(to, content)
                speak("Email successfully sent")

            except Exception as e:
                print(e)
                speak("Sorry, unable to send the email")

        elif 'search in chrome' in query:
            try:
                speak('What should I search')
                chromepath = 'C:\Program Files (x86)\Google\Chrome\Application\chrome.exe %s'
                search = listenvoice()
                wb.get(chromepath).open_new_tab(search)
            except Exception as e:
                print(e)
                speak("Sorry I couldn't understand")

        #elif 'logout' in query:
         #   speak("Signing out")
          #  os.system("shutdown -l")

        elif 'lock' in query:
            speak('Locking the system')
            cmd='rundll32.exe user32.dll, LockWorkStation'
            subprocess.call(cmd)
            quit()

        #elif 'shutdown' or 'poweroff' in query:
         #   speak("shutting down system")
          #  os.system("shutdown /s /t 1")

        #elif 'restart' in query:
         #   speak("restarting system")
          #  os.system("shutdown /r /t 1")

        elif 'remember' in query:
            speak('Tell me, what should I remeber?')
            data = listenvoice()
            speak("You said to remember that" +data)
            remember = open('data.txt', 'w')
            remember.write(data)
            remember.close()

        elif ('who' or 'why' or 'how' or 'when' or 'what') in query:
            #os.startfile('C:\Program Files (x86)\Google\Chrome\Application\chrome.exe')
            search = listenvoice()
            #speak('searching...')
            #speak("Showing you some results")
            wb.open_new(search)

        elif 'thank you' in query:
            speak("Good bye, Have a nice day")
            quit()


import requests
from bs4 import BeautifulSoup
from selenium import webdriver
import time

ch_driver = webdriver.Chrome('C:\\Python37\\chromedriver.exe')
ch_driver.get('http://www.g2b.go.kr:8100/index.jsp')

search_product = ch_driver.find_element_by_xpath("//*[@id='top-sub-menu2']/li[1]/a")
search_product.click()
#//*[@id="top-sub-menu2"]/li[1]/a

category_input = ch_driver.find_element_by_id("CATE_ID")
category = '30131502'
category_input.send_keys(category)

search_btn = ch_driver.find_element_by_id('Search')
search_btn.click()

#time.sleep(90)

html = ch_driver.page_source
soup = BeautifulSoup(html,'lxml')
trs = soup.find('table',attrs={'class':'tb02'}).tbody.find_all('tr')
image_url_prefix = 'http://www.g2b.go.kr:8097/'

product_dict = dict()

for tr,i in zip(trs,range(len(tds))):
    product_image = image_url_prefix + tr.find_all('td')[0].a['href']
    category_subno = tr.find_all('td')[1].text
    product_no = tr.find_all('td')[2].text
    category_name = tr.find_all('td')[3].text
    product_name = tr.find_all('td')[4].text
    product_type = tr.find_all('td')[5].text
    product_dict[product_no]=dict([(name,eval(name)) for name in
                                   ["product_image","category_subno","category_name","product_name","product_type"]])

print(product_dict)


# coding: utf-8

# In[1]:

import pandas
import numpy as np
import statsmodels.formula.api as sm
import sklearn.preprocessing
import matplotlib.pyplot as plt

# In[2]:

def standardize(feature):
    return (feature-feature.mean(axis=0))/feature.std(axis=0)

# In[3]:

Volatility = pandas.read_csv('NYSE Volatility.csv',index_col='Date')
PE = pandas.read_csv('NYSE PE.csv',index_col='Date')

Volatility = standardize(Volatility)
PE = standardize(PE)

# In[4]:

vol = Volatility.drop(Volatility.index[0]).values.flatten()
pe = PE.drop(PE.index[33]).values.flatten()

# In[5]:

# drop nan
vol = list(vol)
pe = list(pe)

i = 0
while i < len(vol):
    if (np.isnan(vol[i])) or (np.isnan(pe[i])):
        vol.pop(i)
        pe.pop(i)
    else:
        i = i + 1

# In[10]:

# drop outlier
P_vol = np.percentile(vol, [2.5, 97.5])
P_pe = np.percentile(pe, [2.5, 97.5])
i = 0
while i < len(vol):
    if (vol[i]>P_vol[1]) or (vol[i]<P_vol[0]) or (pe[i]>P_pe[1]) or (pe[i]<P_pe[0]):
        vol.pop(i)
        pe.pop(i)
    else:
        i = i + 1

# In[11]:

# training set
vol_train = vol[:int(np.size(vol)*0.9)]
pe_train = pe[:int(np.size(pe)*0.9)]

# test set
vol_test = vol[int(np.size(vol)*0.1):]
pe_test = pe[int(np.size(pe)*0.1):]

# In[12]:

len(pe_test)

# In[13]:

reg = pandas.DataFrame()
reg['Vol_train']=vol_train
reg['PE_train']=pe_train

# In[14]:

model = sm.ols(formula="Vol_train ~ PE_train", data=reg).fit()
print(model.summary())

# In[15]:

w = np.array(model.params)

# In[17]:

vol_pre2 = []
for i in range(len(vol_test)):
    vol_pre2.append(w[0] + w[1]*pe_test[i])

# In[42]:

plt.subplots_adjust(top=3, bottom=1, left=0, right=4, hspace=0.3,wspace=0.25)

plt.subplot(1,2,1)

p=plt.scatter(vol_pre2,vol_test,s=20,marker='+',c=np.abs(np.array(vol_pre2)-np.array(vol_test)))
cbar=plt.colorbar(p)
cbar.set_label('L1 Distance')
plt.plot(np.linspace(-0.4,0,1000),1.2*np.linspace(-0.4,0,1000)-0.1,c=(0.6,0.1,0.5),label="Prediction = Reality")
#plt.plot(np.linspace(-1.1,0.5,1000),np.linspace(-1.5,0.5,1000),'r',label="Prediction = Reality")
plt.ylabel('Volatility Prediction',fontsize=15)
plt.xlabel('Volatility Reality',fontsize=15)
plt.title('One Feature (PE) Prediction vs Reality',fontsize=18)
plt.legend(loc=2)

plt.subplot(1,2,2)

p=plt.scatter(yh,y,marker='+',c=np.abs(np.array(yh)-np.array(y)))
cbar=plt.colorbar(p)
cbar.set_label('L1 Distance')
plt.plot(np.linspace(-1.5,0.5,1000),np.linspace(-1.5,0.5,1000),'r',label="Prediction = Reality")
plt.xlabel('Volatility Prediction',fontsize=15)
plt.ylabel('Volatility Reality',fontsize=15)
plt.title('Prediction vs Reality',fontsize=18)
plt.legend(loc=2)

# In[ ]: