# -*- coding: utf-8 -*-
"""
Created on Wed Sep 30 10:35:25 2020

@author: derek.bickhart-adm
"""

import argparse
import sys
from collections import defaultdict
import os
from os.path import basename
import subprocess as sp
import upsetplot
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import pandas as pd

def arg_parse():
    parser = argparse.ArgumentParser(
            description = "A wrapper script for my modified version of meryl"
            )
    parser.add_argument('-m', '--meryl',
                        help="The full path to the meryl executable.",
                        default="meryl", type=str
                        )
    parser.add_argument('-o', '--output',
                        help="Output file basename. Produces a text file (.raw.tab) and a plot (.pdf)",
                        required=True, type=str,
                        )
    parser.add_argument('-d', '--db',
                        help="Meryl database folder name. At least two are required",
                        action="append", default=[],
                        )
    return parser.parse_args(), parser

def main(args, parser):
    if len(args.db) < 2:
        print("Must provide two or more meryl databases!")
        parser.print_usage()
        sys.exit(-1)

    worker = merylWrapper(args.meryl, args.db)

    worker.run(args.output)

    print(f'Finished! Output is in {args.output}')

class merylWrapper:

    def __init__(self, meryl, dbs):
        self.meryl = meryl
        self.dbs = dbs
        self.numdbs = len(dbs)

        # Non final attributes
        self.counter = defaultdict(int)

    def run(self, output):
        dcount = 0
        dbstr = " ".join(self.dbs)

        if os.path.exists(output + ".raw.tab"):
            print("Starting from previous task")
            with open(output + ".raw.tab", 'r') as input:
                for l in input:
                    s = l.rstrip().split()
                    self.counter[s[0]] = int(s[1])
        else:
            with sp.Popen(f'{self.meryl} print venn {dbstr}', shell=True, stdout=sp.PIPE, bufsize=1, universal_newlines=True) as sf:
                for h in sf.stdout:
                    s = h.split()
                    self.counter[s[1]] += 1
                    dcount += 1
                    if dcount % 10000000 == 0:
                        print(f'Progress: {dcount}')

            # print out raw data
            with open(output + ".raw.tab", 'w') as out:
                for w in sorted(self.counter, key=self.counter.get, reverse=True):
                    out.write(f'{w}\t{self.counter[w]}\n')

            print("Created raw output file")

        # Prepare membership df
        array = list()
        data = list()
        for k, v in self.counter.items():
            tlist = list()
            for i, e in enumerate(self.dbs):
                if (int(k) & (1 << int(i))):
                    # The bit is set, add the file name
                    tlist.append(basename(e).split('.')[0])
            array.append(tlist)
            data.append(v)

        # Plot things out
        dataset = upsetplot.from_memberships(array, data=data)
        print(dataset)

        upset = upsetplot.UpSet(dataset, sort_by='cardinality', show_percentages=True)
        upset.plot()

        plt.savefig(output + ".pdf")

if __name__ == "__main__":
    args, parser = arg_parse()
    main(args, parser)


# Probabilistic graphical models ------------------

"""

Probabilistic graphical models give a compact representation
of a joint probability distribution [X_1, X_2,.., X_N] binary RV's
there are 2^N assignments

The factorization of the probabilities happens along
conditional indepences among rv's.
X is conditionally independent of Y given Z (X _|_ Y | Z)
P(X=x, Y=y|Z=z) = P(X=x|Z=z) P(Y=y|Z=z) for all x in X, y in Y, z in Z

The graph can be directed which in this case these are Bayesian networks
or undirected which in this case you have Markov networks.
Graphical models are pretty much generative.

In a markov random field, cycles are allowed in the graph and
switch from local normalization (conditional prob dist at each node) to
global normalization of probabilities (i.e. partition function)

The factorization is given as a sum
P(X_1,.., X_n) = 1/Z exp(-sum_k E[C_k]) where C_k are cliques of the graph
and E[.] is the energy defined over the cliquies
"""

# Define a markov field of binary fields
# This will be an Ising model over three nodes.
# There are three cliques of a single node (on site fields)
# and two cliques  of two nodes - the edges that connect the nodes

import matplotlib.pyplot as plt
import numpy as np
import dimod

n_spins = 3
h = {v: 1 for v in range(n_spins)}
J = {(0, 1):2,
     (1, 2): -1}

model = dimod.BinaryQuadraticModel(h, J, 0.0, dimod.SPIN)
print(model.variables)
print(len(model.variables))
print(model.linear)
print(model.quadratic)
sampler = dimod.SimulatedAnnealingSampler()

# The prob distribution of the config does not explicityl define the temp
# But it is implicitly there in the constants defining the hamiltonian
# We can scale it a temp T = 1

# Now find the Probability P(E) of each energy level E
# It can be expressed as a sum over all states with energy E
# P(E) = sum_E(x_1,..,x-n) P(X1,...,X_N) = sum 1/Z  exp(-E/T)

# The term in the sum is constant (it doesnt depend on the RV's anymore)
# Just need to count the number of states such that
# E(X_1, ..., X_N) = E
# This is called the degeneracy of the energy level E, g(E)
# P(E) = 1/Z * g(E) * exp(-E/T)

temperature = 1
response = sampler.sample(model, beta_range=[1/temperature, 1/temperature], num_reads=100)

# Dict that associate to each energy E the degeneracy g(E)
g = {}
for solution in response.aggregate().data():
    if solution.energy in g.keys():
        g[solution.energy] += 1
    else:
        g[solution.energy] = 1

print('Degeneracy', g)
probabilities = np.array([g[E] * np.exp(-E/temperature) for E in g.keys()])
Z = probabilities.sum()
probabilities /= Z
fig, ax = plt.subplots()
ax.plot([E for E in g.keys()], probabilities, linewidth=3)
ax.set_xlim(min(g.keys()), max(g.keys()))
ax.set_xticks([])
ax.set_yticks([])
ax.set_xlabel('Energy')
ax.set_ylabel('Probability')
#plt.show()

# In this case, the conditional probabilities are already encapsulated
# by the model
# Spins 0 and 2 do not interact
# In general it is hard to learn the strucutre of a probabilistic graph
# given a set of observed correlations in the sample S.
# We can only rely on heuristics
# The typical way is to define a socring function and do a global optimization

# Identify a graph G, learn the probabilities in the graph
# Rely on the sample and its correlations and use a max likelihood or
# maximum a posteriori estimate of the corresponding parameters
# t_G with likelihood P(S|t_G)

# Applying the learned model means probabilistic inference to answer
# queries of the following types
# Conditional prob: P(Y|E=e) = P(Y,e)/P(e)
# Max aposteriori: argmax_y P(y|e) = argmax_y sum_Z P(y, Z|e)

# In Deep learning, running a prediction is cheap after the network is traine
# Inference is computationally demanding even after training a model
# Instead of solving the inference problem directly, we use approximate inference
# with sampling, which is done with Monte carlo methods classically .

# Let's do a max a posteriori on the Ising Model
# clamp the first spin to -1 and run simulated annealing for the rest of them
# to find the optimal config

# simulated annealing routine is modified to account for clamping

from dimod.reference.samplers.simulated_annealing import greedy_coloring

clamped_spins = {0: -1}
num_sweeps = 1000
betas = [1.0 - 1.0*i / (num_sweeps - 1.) for i in range(num_sweeps)]

# Set up adjacency matrix
adj = {n: set() for n in h}
for n0, n1 in J:
    adj[n0].add(n1)
    adj[n1].add(n0)
# Use a vertex coloring for the graph and update the nodes by colour
__, colors = greedy_coloring(adj)

spins = {v: np.random.choice((-1, 1)) if v not in clamped_spins else clamped_spins[v]
         for v in h}

for beta in betas:
    energy_diff_h = {v: -2 * spins[v] * h[v] for v in h}
    # For each color, do updates
    for color in colors:
        nodes = colors[color]
        energy_diff_J = {}
        for v0 in nodes:
            ediff = 0
            for v1 in adj[v0]:
                if (v0, v1) in J:
                    ediff += spins[v0] * spins[v1] * J[(v0, v1)]
                if (v1, v0) in J:
                    ediff += spins[v0] * spins[v1] * J[(v1, v0)]

            energy_diff_J[v0] = -2. * ediff
        for v in filter(lambda x: x not in clamped_spins, nodes):
            logp = np.log(np.random.uniform(0, 1))
            if logp < -1. * beta * (energy_diff_h[v] + energy_diff_J[v]):
                spins[v] *= -1
print(spins)


from typing import List

class Solution:
    def maxAreaOfIsland(self, grid: List[List[int]]) -> int:
        def dfs(row: int, col: int) -> int:
            if not (0 <= row < row_num and 0 <= col < col_num):
                return 0
            if grid[row][col] != 1:
                return 0

            grid[row][col] = -1
            current_area = 1 + dfs(row+1, col) + dfs(row-1, col) + dfs(row, col+1) + dfs(row, col-1)
            return current_area

        max_area = 0
        row_num, col_num = len(grid), len(grid[0])
        for row in range(row_num):
            for col in range(col_num):
                if grid[row][col] == 1:
                    max_area = max(max_area, dfs(row, col))

        return max_area


from rest_framework import serializers
from core.models import Category, Contact, Course, Branch

class CategorySerializer(serializers.ModelSerializer):
    """Serializer for Category objects, used as a link to Category serializer"""

    class Meta:
        model = Category
        fields = (
            "name",
            "imgpath",
        )

class BranchSerializer(serializers.ModelSerializer):
    """Serializer for Branch objects, used as a link to Category serializer"""

    class Meta:
        model = Branch
        fields = (
            "latitude",
            "longitude",
            "address",
        )

class ContactSerializer(serializers.ModelSerializer):
    """Serializer for Contact objects, used as a link to Category serializer"""

    class Meta:
        model = Contact
        fields = (
            "contact_type",
            "value",
        )

class CourseSerializer(serializers.ModelSerializer):
    """Serializer for Course objects, main serializer in this project required to serialize multiple relations"""
    contacts = ContactSerializer(
        many=True,
    )
    branches = BranchSerializer(
        many=True,
    )

    class Meta:
        model = Course
        fields = (
            "name",
            "description",
            "category",
            "uuid",
            "logo",
            "contacts",
            "branches",
        )

        extra_kwargs = {'id': {'read_only': True}}

    def create(self, validated_data):
        branches_data = validated_data.pop("branches")
        contacts_data = validated_data.pop("contacts")
        course = Course.objects.create(**validated_data)

        for branch_data in branches_data:
            Branch.objects.create(course=course, **branch_data)

        for contact_data in contacts_data:
            Contact.objects.create(course=course, **contact_data)

        return course


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        ('homepage', '0009_auto_20170415_1728'),
    ]

    operations = [
        migrations.CreateModel(
            name='fcomplain',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('complain', models.CharField(max_length=250)),
                ('status_update', models.CharField(max_length=250)),
                ('f', models.ForeignKey(to='homepage.farmer')),
            ],
        ),
    ]


from plugin import InvenTreePlugin
from plugin.mixins import PanelMixin, SettingsMixin
from plugin.base.integration.UrlsMixin import UrlsMixin
from django.core.validators import  MinValueValidator
from stock.views import StockLocationDetail, StockIndex
from django.template.loader import render_to_string
from django.urls import path, reverse
from django.utils.translation import gettext as _

from plugins.StockTestPanel_plugin import views as STPviews
from part.models import PartTestTemplate
### ------------------------------------------- Plugin Class ------------------------------------------------ ###

class StockTestPanel(PanelMixin, UrlsMixin, InvenTreePlugin):

    NAME = "StockTestPanel"
    SLUG = "stocktest"
    TITLE = "StockTest panel"
    AUTHOR = "Bbillyben"
    DESCRIPTION = "A plugin to visualize last stock test item by location"
    VERSION = "0.1"

    # SETTINGS = {
    #     'MONTH_FOLLOW': {
    #         'name': 'Number of Month to follow',
    #         'description': 'number ofmonth from last entry to display in Stock Track Panel',
    #         'default': 1,
    #         'validator': MinValueValidator(1),
    #     },
    # }

    def get_panel_context(self, view, request, context):
        """Returns enriched context."""
        ctx = super().get_panel_context(view, request, context)

        return ctx

    def get_custom_panels(self, view, request):
        if isinstance(view, StockIndex) or isinstance(view, StockLocationDetail):

            if isinstance(view, StockLocationDetail):
                loc=view.get_object()
                urlTrac=reverse('plugin:stocktest:test-location', kwargs={'loc': loc.pk})
            else:
                urlTrac=reverse('plugin:stocktest:test-list')

            context={'dataUrl':urlTrac}

            # Test Names
            tNames = PartTestTemplate.objects.all().values('test_name').distinct()
            context['testName']=tNames

            tmpRend = render_to_string(template_name='stocktestpanel/STP_panel.html',context=context)

            panels = [
                {
                    # Simple panel without any actual content
                    'title': 'Stock Test',
                    'content': tmpRend,
                    'icon' : 'fa-vial',
                    'javascript': 'STP_initPanel',
                    'javascript_template': 'stocktestpanel/STP_panel.js'
                }
            ]
            return panels
        return []

    def setup_urls(self):
        # """Urls that are exposed by this plugin."""

        SMP_URL=[
            path('test/', STPviews.STPtestViewSet.get_track, name='test-list'),
            path('test/location/<loc>/', STPviews.STPtestViewSet.get_track, name='test-location'),
        ]
        return SMP_URL


import platform
import subprocess
import time
from datetime import datetime

pf = platform.system()

def jtalk(t, server_id=None):
    open_jtalk = ['open_jtalk']
    filename = '{}.wav'.format(time.time())
    if pf == 'Darwin':
        mech = ['-x', '/usr/local/Cellar/open-jtalk/1.11/dic']
        htsvoice = ['-m', '/usr/local/Cellar/open-jtalk/1.11/voice/htsvoice-tohoku-f01/tohoku-f01-neutral.htsvoice']
    else:
        mech = ['-x', '/usr/local/share/open_jtalk_dic']
        htsvoice = ['-m', '/usr/local/share/hts_voice/htsvoice-tohoku-f01/tohoku-f01-neutral.htsvoice']

    speed = ['-r', '1.0']
    outwav = ['-ow', '{}'.format(filename)]
    cmd = open_jtalk + mech + htsvoice + speed + outwav
    c = subprocess.Popen(cmd, stdin=subprocess.PIPE)
    c.communicate(t.encode())
    c.wait()
    return filename

def say_datetime():
    d = datetime.now()
    text = '%s月%s日、%s時%s分%s秒' % (d.month, d.day, d.hour, d.minute, d.second)
    jtalk(text)

if __name__ == '__main__':
    say_datetime()


from app.vendors import GraphQL, load_schema_from_path, make_executable_schema, MutationType, QueryType
from fastapi.middleware.cors import CORSMiddleware
from .db_main import resolve_custom_method, resolve_generic_query, resolve_generic_update, resolve_query_clients, resolve_update_client, resolve_update_user

type_defs = load_schema_from_path('app/db')
query = QueryType()
mutation = MutationType()

@mutation.field('customMethod')
async def custom_method(_, info, value=''):
    return (resolve_custom_method(info, value))

@query.field('genericQuery')
async def generic_query(_, info, value=''):
    return (resolve_generic_query(info, value))

@mutation.field('genericUpdate')
async def generic_update(_, info, value=''):
    return (resolve_generic_update(info, value))

@query.field('queryClients')
async def query_client(_, info, value=''):
    return (resolve_query_clients(info, value))

@mutation.field('updateClient')
async def update_client(_, info, value=''):
    return (resolve_update_client(info, value))

@mutation.field('updateUser')
async def update_user(_, info, value):
    return (resolve_update_user(info, value))

schema = make_executable_schema(type_defs, query, mutation)

GraphQLApp: GraphQL = CORSMiddleware(
    GraphQL(schema), allow_origins=['http://localhost:3000', 'http://127.0.0.1:3000'], allow_methods=['*'], allow_headers=['*'], allow_credentials=True
)


"""
# Sample code to perform I/O:

name = input()                  # Reading input from STDIN
print('Hi, %s.' % name)         # Writing output to STDOUT

# Warning: Printing unwanted or ill-formatted data to output will cause the test cases to fail
"""

# Write your code here
from datetime import datetime, timedelta

time = input().strip()
x = int(input())
hh, mm = map(int, time.split(':'))
dt = datetime.strptime(time, '%H:%M')
td = timedelta(minutes=1)
t_minutes = 0
while True:
    hours = dt.hour
    minutes = dt.minute
    total = sum(divmod(hours, 10)) + sum(divmod(minutes, 10))
    if total % x == 0:
        print(t_minutes)
        break
    dt += td
    if dt.hour == hh and dt.minute == mm:
        print(-1)
        break
    t_minutes += 1


from setuptools import setup

with open("multiprom/__init__.py", "r") as f:
    version_marker = "__version__ = "
    for line in f:
        if line.startswith(version_marker):
            _, version = line.split(version_marker)
            version = version.strip().strip('"')
            break
    else:
        raise RuntimeError("Version marker not found.")

def parse_dependencies(filename):
    with open(filename) as reqs:
        for line in reqs:
            line = line.strip()
            if line.startswith("#"):
                continue

            elif line.startswith("-r"):
                yield from parse_dependencies(line[len("-r "):])

            else:
                yield line

dependencies = list(parse_dependencies("requirements.txt"))

extras = ()
extra_dependencies = {}
for extra in extras:
    filename = "{}.txt".format(extra)
    extra_dependencies[extra] = list(parse_dependencies(filename))

setup(
    name="multiprom",
    version=version,
    packages=["multiprom"],
    install_requires=dependencies,
    extras_require=extra_dependencies,
)


import sqlalchemy as sa
from sqlalchemy.orm import sessionmaker
from sqlalchemy.ext.declarative import declarative_base

Base = declarative_base()

class Database:

    def __init__(self):
        self.base_path = "sqlite:///gifs.sqlite3"
        engine = sa.create_engine(self.base_path)
        Base.metadata.create_all(engine)
        session = sessionmaker(engine)
        self.session = session()

    class Gif(Base):
        __tablename__ = 'gif'
        id = sa.Column(sa.Integer, primary_key = True)
        gif_path = sa.Column(sa.Text)
        description = sa.Column(sa.Text)

    def add_gif_to_db(self, gif_path, description):
        new_gif = self.Gif(
            gif_path=gif_path,
            description=description
        )
        self.session.add(new_gif)
        self.session.commit()

    def update_gif(self, gif_path, description):
        matched_gif = self.session.query(self.Gif).filter(self.Gif.gif_path == gif_path).first()
        new_description = " ".join(set((f"{matched_gif.description} {description}").split(" ")))
        matched_gif.description = new_description
        self.session.commit()

    def get_all_gifs(self):
        return self.session.query(self.Gif)


from __future__ import print_function

import torch
from torch.autograd import Variable

import numpy as np
# -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=

# ======================================================================================================================
def train_vae(epoch, args, train_loader, model, optimizer):
    # set loss to 0
    train_loss = 0
    train_re = 0
    train_kl = 0
    # set model in training mode
    model.train()

    # start training
    if args.warmup == 0:
        beta = 1.
    else:
        beta = 1.* epoch / args.warmup
        if beta > 1.:
            beta = 1.
    print('beta: {}'.format(beta))

    for batch_idx, (data, target) in enumerate(train_loader):
        if args.cuda:
            data, target = data.cuda(), target.cuda()
        data, target = Variable(data), Variable(target)
        # dynamic binarization
        if args.dynamic_binarization:
            x = torch.bernoulli(data)
        else:
            x = data

        # reset gradients
        optimizer.zero_grad()
        # loss evaluation (forward pass)
        loss, RE, KL = model.calculate_loss(x, beta, average=True)
        # backward pass
        loss.backward()
        # optimization
        optimizer.step()

        train_loss += loss.data[0]
        train_re += -RE.data[0]
        train_kl += KL.data[0]

    # calculate final loss
    train_loss /= len(train_loader)  # loss function already averages over batch size
    train_re /= len(train_loader)  # re already averages over batch size
    train_kl /= len(train_loader)  # kl already averages over batch size

    return model, train_loss, train_re, train_kl


import numpy as np
from imblearn.over_sampling import SMOTE
from imblearn.pipeline import Pipeline as ImbPipeline
from imblearn.under_sampling import RandomUnderSampler
from sklearn.compose import make_column_transformer
from sklearn.ensemble import (
    AdaBoostClassifier,
    ExtraTreesClassifier,
    RandomForestClassifier,
)
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import (
    MinMaxScaler,
    Normalizer,
    OneHotEncoder,
    RobustScaler,
    StandardScaler,
)
from sklearn.tree import DecisionTreeClassifier

RANDOM_SEED = 2357

def load_random_forest():
    params = {
        "clf__n_estimators": [10, 50, 100, 150, 200],
        "clf__criterion": ["gini", "entropy"],
        "clf__max_depth": [3, 6, 12, 24, None],
        "clf__max_features": ["auto", "sqrt", "log2", None],
        "clf__min_samples_split": [2, 3, 4, 5, 10],
        "clf__bootstrap": [True, False],
    }
    return RandomForestClassifier(random_state=RANDOM_SEED), params

def load_extra_trees():
    params = {
        "clf__n_estimators": [10, 50, 100, 150, 200],
        "clf__criterion": ["gini", "entropy"],
        "clf__max_depth": [3, 6, 12, 24, None],
        "clf__max_features": ["auto", "sqrt", "log2", None],
        "clf__min_samples_split": [2, 3, 4, 5, 10],
        "clf__bootstrap": [True, False],
    }
    return ExtraTreesClassifier(random_state=RANDOM_SEED), params

def load_decision_tree():
    params = {
        "clf__criterion": ["gini", "entropy"],
        "clf__splitter": ["best", "random"],
        "clf__max_depth": [3, 6, 12, 24, None],
        "clf__max_features": ["auto", "sqrt", "log2", None],
        "clf__min_samples_split": [2, 3, 5, 10, 11],
    }
    return DecisionTreeClassifier(random_state=RANDOM_SEED), params

def load_ada_boost():
    params = {
        "clf__n_estimators": [10, 50, 100, 150, 200],
        "clf__learning_rate": [0.0001, 0.001, 0.01, 0.1, 1.0],
        "clf__algorithm": ["SAMME", "SAMME.R"],
    }
    return AdaBoostClassifier(random_state=RANDOM_SEED), params

def load_logreg():
    r = np.random.RandomState(seed=RANDOM_SEED)
    params = {
        "clf__max_iter": np.arange(250, 350, 10),
        "clf__C": r.uniform(0.01, 10, 10),
        "transformer__remainder": [
            RobustScaler(),
            StandardScaler(),
            MinMaxScaler(),
            Normalizer(),
        ],
    }
    return LogisticRegression(random_state=RANDOM_SEED), params

def create_pipeline(categ_cols, model_param, balancing=None):
    if model_param == "rf":
        model, params = load_random_forest()
    elif model_param == "dt":
        model, params = load_decision_tree()
    elif model_param == "et":
        model, params = load_extra_trees()
    elif model_param == "ab":
        model, params = load_ada_boost()
    elif model_param == "lr":
        model, params = load_logreg()
    else:
        raise Exception(f"Unkown parameter {model_param}")

    # Common pipeline steps
    pipeline_steps = []

    if len(categ_cols):
        column_transformer = make_column_transformer(
            (OneHotEncoder(), categ_cols), remainder="passthrough"
        )
        pipeline_steps.append(("transformer", column_transformer))

    pipeline_steps.append(("clf", model))

    # Actual pipeline instantiation
    if balancing is None:
        pipeline = Pipeline(steps=pipeline_steps)
    elif balancing in {"smote", "rus"}:
        sampler = RandomUnderSampler(random_state=RANDOM_SEED)
        if balancing == "smote":
            sampler = SMOTE(random_state=RANDOM_SEED)
        # sampler is always one step before the transformer
        pipeline_steps.insert(-1, ("sampler", sampler))
        pipeline = ImbPipeline(steps=pipeline_steps)
    else:
        raise Exception(f"Unknown balancing {balancing}")

    return pipeline, params


import requests
from bs4 import BeautifulSoup
import pandas as pd
from selenium import webdriver
import time

driver=webdriver.Chrome('/home/ai/Documents/chromedriver_linux64/chromedriver')
driver.get('https://store.unionlosangeles.com/collections/outerwear')
driver.maximize_window()

scroll_height=driver.execute_script('return document.body.scrollHeight')

while True:
    driver.execute_script('window.scrollTo(0,document.body.scrollHeight)')
    time.sleep(3)
    new_scrll=driver.execute_script('return document.body.scrollHeight')
    if new_scrll == scroll_height:
        break
    scroll_height = new_scrll

df=pd.DataFrame({'Name':[''],'Title':[''],'Price':[''],'Link':['']})

soup=BeautifulSoup(driver.page_source, 'lxml')
section=soup.find('div', {'id':'main','role':'main'})

dresses=section.find_all('li')
for dress in dresses:
    link=dress.find('a').get('href')
    name=dress.find(class_='cap-vendor').text
    title=dress.find(class_='cap-title').text
    price=dress.find(class_='cap-price').text
    df=df.append({'Name':name,'Title':title,'Price':price,'Link':link}, ignore_index=True)

df['Link'][4:]='https://store.unionlosangeles.com'+df['Link'][4:]


from llama_index import SimpleDirectoryReader, GPTListIndex, readers, GPTSimpleVectorIndex, LLMPredictor, PromptHelper, ServiceContext
from langchain import OpenAI
import sys
import os
import streamlit as st

os.environ["OPENAI_API_KEY"] = st.secrets["OPEN_AI_KEY"]

def construct_index(directory_path):
    # set maximum input size
    max_input_size = 100
    # set number of output tokens
    num_outputs = 100
    # set maximum chunk overlap
    max_chunk_overlap = 20
    # set chunk size limit
    chunk_size_limit = 600

    # define prompt helper
    prompt_helper = PromptHelper(max_input_size, num_outputs, max_chunk_overlap, chunk_size_limit=chunk_size_limit)

    # define LLM
    llm_predictor = LLMPredictor(llm=OpenAI(temperature=0.5, model_name="text-babbage-001", max_tokens=num_outputs))

    documents = SimpleDirectoryReader(directory_path).load_data()

    service_context = ServiceContext.from_defaults(llm_predictor=llm_predictor, prompt_helper=prompt_helper)
    index = GPTSimpleVectorIndex.from_documents(documents, service_context=service_context)

    index.save_to_disk('index.json')

    return index

construct_index("context_data/data")

import gradio as gr
def ask_ai(AskAnythingAboutShreyaAndIshan):
    index = GPTSimpleVectorIndex.load_from_disk('index.json')
    while True:
        response = index.query(AskAnythingAboutShreyaAndIshan)

        return response

demo = gr.Interface(fn=ask_ai, inputs="text", outputs="text")
demo.launch()


from collections import Counter
from unittest import TestCase
from helper.helper import Helper

class HelperTests(TestCase):

    def setUp(self):
        self.helper = Helper()
        self.board = self.helper.add_board('Board Name')

    def test_board(self):
        self.assertEqual(self.board.name, 'Board Name')
        self.assertEqual(self.board.closed, False)
        self.board.close()
        self.assertEqual(self.board.closed, True)
        self.board.open()
        self.assertEqual(self.board.closed, False)
        self.assertEqual(self.board.id, 'Board Name_id')
        self.assertEqual(self.board.open_lists(), [])
        self.assertEqual(self.board.open_cards(), [])
        self.assertEqual(self.board.get_labels(), [])

    def test_list(self):
        trello_list_1 = self.board.add_list('List 1')
        trello_list_2 = self.board.add_list('List 2')
        trello_list_3 = self.board.add_list('List 3')
        self.assertEqual(self.board.open_lists(), [trello_list_1, trello_list_2, trello_list_3])

    def test_card(self):
        trello_list_1 = self.board.add_list('List 1')
        trello_list_2 = self.board.add_list('List 2')
        trello_list_3 = self.board.add_list('List 3')
        card_1 = trello_list_1.add_card('Card 1')
        card_2 = trello_list_1.add_card('Card 2')
        card_3 = trello_list_1.add_card('Card 3')
        self.assertEqual(trello_list_1.list_cards(), [card_1, card_2, card_3])

    def test_create_test_data(self):
        boards = self.helper.create_test_data()
        board_names = [board.name for board in boards]
        expected_boards = ['imaging', 'vector', 'claims', 'infrastructure', 'mips', 'misc', 'apps']
        self.assertCountEqual(board_names, expected_boards)
        expected_lists = {'coding done': 7, 'qa': 7, 'in progress': 7, 'production': 7, 'final qa': 7, 'backlog': 7}
        list_counter = Counter()
        for board in boards:
            list_counter += Counter([trello_list.name for trello_list in board.open_lists()])
        self.assertEqual(dict(list_counter), expected_lists)
        self.assert_labels(boards)

    def assert_labels(self, boards):
        label_counter = Counter()
        no_labels = []
        for board in boards:
            cards = board.open_cards()
            for card in cards:
                if card.labels:
                    label_counter += Counter([label.name for label in card.list_labels])
                else:
                    no_labels.append(card)
        self.assertEqual(len(no_labels), 42)
        expected_labels = {'task': 126, 'production feedback': 84, 'staging feedback': 42}
        self.assertEqual(dict(label_counter), expected_labels)


import logging
import os
import numpy as np
import scipy.sparse as ss
import tensorflow as tf
import time

from tensorlog import comline
from tensorlog import config
from tensorlog import funs
from tensorlog import ops
from tensorlog import xcomp
from tensorlog import expt
from tensorlog import util
from tensorlog import learnxcomp
from tensorlog import dataset
from tensorlog import declare

class TensorFlowCrossCompiler(xcomp.AbstractCrossCompiler):

  def __init__(self,db,summaryFile=None):
    """If summaryFile is provided, save some extra information to pass on
    to tensorboard.
    """
    super(TensorFlowCrossCompiler,self).__init__(db)
    self.tfVarsToInitialize = []
    self.summaryFile = summaryFile
    self.session = None
    self.sessionInitialized = None
    logging.debug('TensorFlowCrossCompiler initialized %.3f Gb' % util.memusage())

  def close(self):
    if self.session is not None:
      self.session.close()

  #
  # tensorflow specific routines
  #

  # low-level training stuff

  def setSession(self,session=None):
    """ Insert a session for the
    """
    self.session = session

  def ensureSessionInitialized(self):
    """ Make sure the varables in the session have been initialized,
    initializing them if needed
    """
    if self.session is None:
      logging.debug('creating session %.3f Gb' % util.memusage())
      self.session = tf.Session()
      logging.debug('session created %.3f Gb' % util.memusage())
    if not self.sessionInitialized:
      logging.debug('initializing session %.3f Gb' % util.memusage())
      for var in self.tfVarsToInitialize:
        self.session.run(var.initializer)
      #self.session.run(tf.global_variables_initializer()) #-kmm

      self.sessionInitialized = True
      logging.debug('session initialized %.3f Gb' % util.memusage())

  def getInputName(self,mode,inputs=None):
    """ String key for the input placeholder
    """
    mode = self.ensureCompiled(mode,inputs=inputs)
    assert len(self._wsDict[mode].inferenceArgs)==1
    return self._wsDict[mode].inferenceArgs[0].name

  def getInputPlaceholder(self,mode,inputs=None):
    """ The input placeholder
    """
    mode = self.ensureCompiled(mode,inputs=inputs)
    assert len(self._wsDict[mode].inferenceArgs)==1
    return self._wsDict[mode].inferenceArgs[0]

  def getTargetOutputName(self,mode,inputs=None):
    """ String key for the target-output placeholder
    """
    mode = self.ensureCompiled(mode,inputs=inputs)
    assert len(self._wsDict[mode].dataLossArgs)==2
    return self._wsDict[mode].dataLossArgs[-1].name

  def getTargetOutputPlaceholder(self,mode,inputs=None):
    """  The target-output placeholder
    """
    mode = self.ensureCompiled(mode,inputs=inputs)
    assert len(self._wsDict[mode].dataLossArgs)==2
    return self._wsDict[mode].dataLossArgs[-1]

  def getFeedDict(self,mode,X,Y,wrapped=False,inputs=None):
    """ Create a feed dictionary for training based on X and Y
    """
    mode = self.ensureCompiled(mode,inputs=inputs)
    (X,Y) = self._ensureWrapped(X,Y,wrapped)
    return { self.getInputName(mode):X, self.getTargetOutputName(mode):Y }

  def optimizeDataLoss(self,mode,optimizer,X,Y,epochs=1,minibatchSize=0,wrapped=False):
    """ Train
    """
    self.ensureSessionInitialized()
    if self.summaryFile:
      self.summaryWriter = tf.summary.FileWriter(self.summaryFile, self.session.graph)
    def runAndSummarize(fd,i):
      if not self.summaryFile:
        self.session.run([trainStep],feed_dict=fd)
      else:
        (stepSummary, _) = self.session.run([self.summaryMergeAll,trainStep],feed_dict=fd)
        self.summaryWriter.add_summary(stepSummary,i)

    trainStep = optimizer.minimize(self._wsDict[mode].dataLossExpr, var_list=self.getParamVariables(mode))
    self.ensureSessionInitialized()
    if not minibatchSize:
      fd = self.getFeedDict(mode,X,Y,wrapped)
      for i in range(epochs):
        runAndSummarize(fd,i)
    else:
      X1,Y1 = self._ensureUnwrapped(X,Y,wrapped)
      dset = dataset.Dataset({targetMode:X1},{targetMode:Y1})
      for i in range(epochs):
        for mode,miniX,miniY in dset.minibatchIterator(batchsize=minibatchSize):
          fd = self.getFeedDict(mode,miniX,miniY,wrapped=False)
          runAndSummarize(fd,i)

  def accuracy(self,mode,X,Y,wrapped=False,inputs=None):
    """ Return accuracy of a model on a test set
    """
    mode = self.ensureCompiled(mode,inputs=inputs)
    Xval,trueYVal = self._ensureWrapped(X,Y,wrapped)
    trueY = tf.placeholder(tf.float32, shape=trueYVal.shape, name="tensorlog/trueY")
    fd = { self.getInputName(mode):Xval, trueY.name:trueYVal }
    (_, Y_) = self.inference(mode)
    correct_prediction = tf.equal(tf.argmax(trueY,1), tf.argmax(Y_,1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
    self.ensureSessionInitialized()
    with self.session.as_default():
      return accuracy.eval(fd)

  # higher-level training stuff

  def runExpt(self,prog=None,trainData=None,testData=None,targetMode=None,
              savedTestPredictions=None,savedTestExamples=None,savedTrainExamples=None,savedModel=None,
              optimizer=None, epochs=10, minibatchSize=0):
    """Similar to tensorlog.expt.Expt().run()
    """
    assert targetMode is not None,'targetMode must be specified'
    assert prog is not None,'prog must be specified'
    logging.debug('runExpt calling setAllWeights %.3f Gb' % util.memusage())
    prog.setAllWeights()
    logging.debug('runExpt finished setAllWeights %.3f Gb' % util.memusage())

    expt.Expt.timeAction('compiling and cross-compiling', lambda:self.ensureCompiled(targetMode,inputs=None))

    assert optimizer is None,'optimizers not supported yet'
    optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.1)
    train_step = optimizer.minimize(self._wsDict[targetMode].dataLossExpr, var_list=self.getParamVariables(targetMode))
    X = trainData.getX(targetMode)
    Y = trainData.getY(targetMode)
    X,Y = self._ensureWrapped(X,Y,False)
    TX = testData.getX(targetMode)
    TY = testData.getY(targetMode)
    TX,TY = self._ensureWrapped(TX,TY,False)

    lossFun = self.dataLossFunction(targetMode,wrapInputs=False,unwrapOutputs=False)
    def printLoss(msg,X,Y): print(msg,lossFun(X,Y))
    def printAccuracy(msg,X,Y): print(msg,self.accuracy(targetMode,X,Y,wrapped=True))

    expt.Expt.timeAction('computing train loss',lambda:printLoss('initial train loss',X,Y))
    expt.Expt.timeAction('computing test loss',lambda:printLoss('initial test loss',TX,TY))
    expt.Expt.timeAction('computing train accuracy',lambda:printAccuracy('initial train accuracy',X,Y))
    expt.Expt.timeAction('computing test accuracy',lambda:printAccuracy('initial test accuracy',TX,TY))

    expt.Expt.timeAction('training', lambda:self.optimizeDataLoss(targetMode,optimizer,X,Y,epochs=epochs,minibatchSize=minibatchSize,wrapped=True))

    expt.Expt.timeAction('computing train loss',lambda:printLoss('final train loss',X,Y))
    expt.Expt.timeAction('computing test loss',lambda:printLoss('final test loss',TX,TY))
    expt.Expt.timeAction('computing train accuracy',lambda:printAccuracy('final train accuracy',X,Y))
    expt.Expt.timeAction('computing test accuracy',lambda:printAccuracy('final test accuracy',TX,TY))

    if savedModel:
      self.exportAllLearnedParams()
      expt.Expt.timeAction('saving trained model', lambda:prog.db.serialize(savedModel))

    def savePredictions(fileName):
      inferenceFun = self.inferenceFunction(targetMode,wrapInputs=False,unwrapOutputs=True)
      Y_ = inferenceFun(TX)
      # Y_ is unwrapped, but need to get unwrapped version of TX from testData
      expt.Expt.predictionAsProPPRSolutions(fileName,targetMode.functor,prog.db,testData.getX(targetMode),Y_)

    if savedTestPredictions:
      expt.Expt.timeAction('saving test predictions', lambda:savePredictions(savedTestPredictions))
    if savedTestExamples:
      expt.Expt.timeAction('saving test examples', lambda:testData.saveProPPRExamples(savedTestExamples,prog.db))
    if savedTrainExamples:
      expt.Expt.timeAction('saving train examples',lambda:trainData.saveProPPRExamples(savedTrainExamples,prog.db))
    if savedTestPredictions and savedTestExamples:
      print('ready for commands like: proppr eval %s %s --metric auc --defaultNeg' % (savedTestExamples,savedTestPredictions))

  # debug stuff

  @staticmethod
  def pprintExpr(expr,previouslySeen=None,depth=0,maxdepth=20):
    """ Print debug-level information on a tensorlog expression """
    if previouslySeen is None:
      previouslySeen=set()
    if depth>maxdepth:
      print('...')
    else:
      tab = '| '*(depth+1),
      op = expr.op
      print('%sexpr:' % tab,expr,'type','op',op.name,'optype',op.type)
    if not expr in previouslySeen:
      previouslySeen.add(expr)
      for inp in op.inputs:
        TensorFlowCrossCompiler.pprintExpr(inp,previouslySeen,depth=depth+1,maxdepth=maxdepth)

  @staticmethod
  def pprintAndLocateGradFailure(expr,vars,previouslySeen=None,depth=0,maxdepth=20):
    """ Print debug-level information on a tensorlog expression, and also
    give an indication of where a gradient computation failed.  """
    if previouslySeen is None:
      previouslySeen=set()
    def hasGrad(expr):
      try:
        return all([g is not None for g in tf.gradients(expr,vars)]),'ok'
      except Exception as ex:
        return False,ex
    if depth>maxdepth:
      print('...')
    else:
      op = expr.op
      stat,ex = hasGrad(expr)
      tab = '+ ' if stat else '| '
      print(tab*(depth+1),expr,op.name,ex)
      if not expr in previouslySeen:
        previouslySeen.add(expr)
        for inp in op.inputs:
          TensorFlowCrossCompiler.pprintAndLocateGradFailure(
            inp,vars,previouslySeen,depth=depth+1,maxdepth=maxdepth)

  #
  # standard xcomp interface
  #

  def _finalizeCompile(self,mode):
    #TODO does this work for multi-mode compilation?
    if self.summaryFile:
      self.summaryMergeAll = tf.summary.merge_all()

  def _buildLossExpr(self,mode):
    target_y = self._createPlaceholder(xcomp.TRAINING_TARGET_VARNAME,'vector',self._wsDict[mode].inferenceOutputType)
    self._wsDict[mode].dataLossArgs = self._wsDict[mode].inferenceArgs + [target_y]
    # we want to take the log of the non-zero entries and leave the
    # zero entries alone, so add 1 to all the zero indices, then take
    # a log of that.
    inferenceReplacing0With1 = tf.where(
        self._wsDict[mode].inferenceExpr>0.0,
        self._wsDict[mode].inferenceExpr,
        tf.ones(tf.shape(self._wsDict[mode].inferenceExpr), tf.float32))
    self._wsDict[mode].dataLossExpr = tf.reduce_sum(-target_y * tf.log(inferenceReplacing0With1))
    self._wsDict[mode].dataLossGradExprs = tf.gradients(self._wsDict[mode].dataLossExpr,self.getParamVariables(mode))

  def _asOneInputFunction(self,arg1,expr,wrapInputs,unwrapOutputs):
    def closure(rawInput1,session=None):
      input1 = self._wrapMsg(rawInput1) if wrapInputs else rawInput1
      bindings = {arg1:input1}
      return self._callAndUnwrap(expr,bindings,unwrapOutputs,session)
    return closure

  def _asTwoInputFunction(self,arg1,arg2,expr,wrapInputs,unwrapOutputs):
    def closure(rawInput1,rawInput2,session=None):
      input1 = self._wrapMsg(rawInput1) if wrapInputs else rawInput1
      input2 = self._wrapMsg(rawInput2) if wrapInputs else rawInput2
      bindings = {arg1:input1,arg2:input2}
      return self._callAndUnwrap(expr,bindings,unwrapOutputs,session)
    return closure

  def _exprListAsUpdateFunction(self,arg1,arg2,exprList,wrapInputs,unwrapOutputs):
    def closure(rawInput1,rawInput2,session=None):
      input1 = self._wrapMsg(rawInput1) if wrapInputs else rawInput1
      input2 = self._wrapMsg(rawInput2) if wrapInputs else rawInput2
      bindings = {arg1:input1,arg2:input2}
      if session is None:
        self.ensureSessionInitialized()
        with self.session.as_default():
          rawUpdates = [expr.eval(feed_dict=bindings) for expr in exprList]
      else:
        with session.as_default():
          rawUpdates = [expr.eval(feed_dict=bindings) for expr in exprList]
      if unwrapOutputs:
        return list(map(lambda key,rawUpdate:(key,self._unwrapUpdate(key,rawUpdate)), self.prog.getParamList(), rawUpdates))
      else:
        return list(zip(self.prog.getParamList(), rawUpdates))
    return closure

  def _callAndUnwrap(self,expr,bindings,unwrapOutputs,session):
    # helper for _asXInputFunction's
    if session is None:
      self.ensureSessionInitialized()
      with self.session.as_default():
        tmp = expr.eval(feed_dict=bindings)
    else:
      with session.as_default():
        tmp = expr.eval(feed_dict=bindings)
    return self._unwrapOutput(tmp) if unwrapOutputs else tmp

  def show(self,verbose=0):
    """ Print a summary of current workspace to stdout """
    print('exprArgs',self.ws.inferenceArgs)
    print('expr',self.ws.inferenceExpr,'type',type(self.ws.inferenceExpr))
    if verbose>=1:
      TensorFlowCrossCompiler.pprintExpr(self.ws.inferenceExpr)

  def getLearnedParam(self,key,session=None):
    if session is None:
      self.ensureSessionInitialized()
      with self.session.as_default():
        varVal = self._handleExpr[key].eval()
    else:
      with session.as_default():
        varVal = self._handleExpr[key].eval()
    # same logic works for param values as param updates
    functor,arity = key
    if arity==1:
      return self._unwrapDBVector(key,varVal)
    else:
      return self._unwrapDBMatrix(key,varVal)

###############################################################################
# implementation for dense messages, dense relation matrices
###############################################################################

class DenseMatDenseMsgCrossCompiler(TensorFlowCrossCompiler):

  def __init__(self,db,summaryFile=None):
    super(DenseMatDenseMsgCrossCompiler,self).__init__(db,summaryFile=summaryFile)

  def _softPlusInverse(self,y):
    approxLogInf = 50.0
    # be careful, because you can't apply softplus inverse to zero
    # values, so first add 1 to all the zeros, by adding to all and
    # subtracting from the nonzeros.  nonzeros here must be the values
    # that are not zero, and don't underflow to zero when we add and
    # subtract
    nonzeroIndices = np.nonzero((y-approxLogInf) + approxLogInf)
    tmp = y + 1.0
    tmp[nonzeroIndices] -= 1.0
    result = np.log(np.exp(tmp)-1.0)
    # finally put something back in the zero positions which
    # is small enough that softplus brings it back to zero
    result -= approxLogInf
    result[nonzeroIndices] += approxLogInf
    return result

  def _createPlaceholder(self,name,kind,typeName):
    assert kind=='vector'
    result = tf.placeholder(tf.float32, shape=[None,self.db.dim(typeName)], name="tensorlog/"+name)
    return result

  def _insertHandleExpr(self,key,name,val,broadcast=False):
    # parameters are passed through softplus so they don't get pushed to zero
    # we need to undo this transformation when you load them into variables...
    isTrainable = (key in self.db.paramSet)
    v = self._reparameterizeAndRecordVar(val,name,isTrainable)
    self._handleExprVar[key] = v
    self._handleExpr[key] = self._reparameterizedVarExpr(v,isTrainable)

  def _reparameterizeAndRecordVar(self,val,name,isTrainable):
    initVal = self._softPlusInverse(val) if (isTrainable and xcomp.conf.reparameterizeMatrices) else val
    v = tf.Variable(initVal, name="tensorlog/"+name, trainable=isTrainable)
    #v = tf.Variable(val, name="tensorlog/"+name, trainable=isTrainable)
    self.summarize(name,v)
    self.tfVarsToInitialize.append(v)
    return v

  def _reparameterizedVarExpr(self,var,isTrainable):
    return tf.nn.softplus(var) if (isTrainable and xcomp.conf.reparameterizeMatrices) else var

  def summarize(self,name,v):
    if self.summaryFile:
      tf.summary.scalar('summaries/%s/size' % name, tf.size(v))

  def _wrapMsg(self,vec):
    """ Convert a vector from the DB into a vector value used by the
    target language """
    return vec.todense()

  def _wrapDBVector(self,vec):
    """ Convert a vector from the DB into a vector value used by the
    target language """
    return vec.todense()

  def _wrapDBMatrix(self,mat):
    """ Convert a matrix from the DB into a vector value used by the
    target language """
    return mat.todense()

  def _unwrapUpdate(self,key,up):
    return self._unwrapOutput(up)

  def _unwrapOutput(self,x):
    """Convert a matrix produced by the target language to the usual
    sparse-vector output of tensorlog"""
    sx = ss.csr_matrix(x)
    sx.eliminate_zeros()
    return sx

  def _unwrapDBVector(self,key,vec):
    return self._unwrapOutput(vec)

  def _unwrapDBMatrix(self,key,mat):
    return self._unwrapOutput(vec)

  def _softmaxFun2Expr(self,subExpr,typeName):
    # zeros are actually big numbers for the softmax,
    # so replace them with a big negative number
    subExprReplacing0WithNeg10 = tf.where(
        subExpr>0.0,
        subExpr,
        tf.ones(tf.shape(subExpr), tf.float32)*(-10.0))
    return tf.nn.softmax(subExprReplacing0WithNeg10 + self._nullSmoother[typeName])

  def _transposeMatrixExpr(self,m):
    return tf.transpose(m)

  def _vecMatMulExpr(self,v,m):
    return tf.matmul(v,m)

  def _componentwiseMulExpr(self,v1,v2):
    return tf.multiply(v1,v2)

  def _weightedVecExpr(self,vec,weighter):
    return tf.multiply(vec, tf.reduce_sum(weighter, axis=1, keep_dims=True))

###############################################################################
# implementation for dense messages, sparse relation matrices
###############################################################################

class SparseMatDenseMsgCrossCompiler(DenseMatDenseMsgCrossCompiler):

  def __init__(self,db,summaryFile=None):
    logging.debug('SparseMatDenseMsgCrossCompiler calling %r %.3f Gb' % (super(SparseMatDenseMsgCrossCompiler,self).__init__,util.memusage()))
    super(SparseMatDenseMsgCrossCompiler,self).__init__(db,summaryFile=summaryFile)
    logging.debug('SparseMatDenseMsgCrossCompiler finished super.__init__ %.3f Gb' % util.memusage())
    # we will need to save the original indices/indptr representation
    # of each sparse matrix
    self.sparseMatInfo = {}
    logging.debug('SparseMatDenseMsgCrossCompiler initialized %.3f Gb' % util.memusage())

  def _insertHandleExpr(self,key,name,val,broadcast=False):
    (functor,arity) = key
    isTrainable = (key in self.db.paramSet)
    if arity<2:
      initVal = self._softPlusInverse(val) if isTrainable else val
      v = self._reparameterizeAndRecordVar(val,name,isTrainable)
      self._handleExprVar[key] = v
      self._handleExpr[key] = self._reparameterizedVarExpr(v,isTrainable)
    else:
      # matrixes are sparse so we need to convert them into
      # a handle expression that stores a SparseTensor, and
      # do some additional bookkeeping.

      # first convert from scipy csr format of indices,indptr,data to
      # tensorflow's format, where the sparseindices are a 2-D tensor.
      sparseIndices = []
      (nRows,nCols) = val.shape
      for i in range(nRows):
        for j in val.indices[val.indptr[i]:val.indptr[i+1]]:
          sparseIndices.append([i,j])
      logging.debug('%d sparseIndices for %d x %d relation %s: sparsity %g' %
                    (len(sparseIndices),nRows,nCols,functor,len(sparseIndices)/float(nRows*nCols)))
      # save the old shape and indices for the scipy matrix so we can
      # reconstruct a scipy matrix in unwrapUpdate.
      self.sparseMatInfo[key] = (val.indices,val.indptr,val.shape)
      # create the handle expression, and save a link back to the
      # underlying varable which will be optimized, ie., the 'values'
      # of the SparseTensor,
      indiceVar = tf.Variable(np.array(sparseIndices), name="tensorlog/%s_indices" % name)
      valueVar = self._reparameterizeAndRecordVar(val.data,name,isTrainable)
      # note: the "valueVar+0.0" seems to be necessary to get a non-zero
      # gradient, but I don't understand why.  w/o this there is no "read"
      # node in for the variable in the graph and the gradient fails
      self._handleExpr[key] = tf.SparseTensor(indiceVar,self._reparameterizedVarExpr(valueVar,isTrainable)+0.0,[nRows,nCols])
      self._handleExprVar[key] = valueVar
      # record the index variable, which also needs to be initialized
      self.tfVarsToInitialize.append(indiceVar)
      self.summarize("%s_indices" % name,indiceVar)

  def _unwrapDBVector(self,key,vec):
    return ss.csr_matrix(vec)

  def _unwrapDBMatrix(self,key,mat):
    (indices,indptr,shape) = self.sparseMatInfo[key]
    # note mat will be a SparseTensor in this case.....
    return ss.csr_matrix((mat.values,indices,indptr),shape=shape)

  def _unwrapUpdate(self,key,up):
    # we will optimize by updating the _handleExprVar's, which are,
    # for a SparseTensor, the value expressions.  to check gradients
    # and such we will need to convert the value updates to tensorlog
    # sparse matrix updates.
    (functor,arity) = key
    if arity==1:
      return ss.csr_matrix(up)
    elif arity==2:
      (indices,indptr,shape) = self.sparseMatInfo[key]
      return ss.csr_matrix((up,indices,indptr),shape=shape)
    else:
      assert False

  #
  # override the dense-matrix operations with sparse ones
  #

  def _wrapDBMatrix(self,mat):
    return mat

  def _transposeMatrixExpr(self,m):
    return tf.sparse_transpose(m)

  def _vecMatMulExpr(self,v,m):
    # TODO: the sparse_transpose below throws two op_scope deprecation warning
    mT = tf.sparse_transpose(m)
    vT = tf.transpose(v)
    result = tf.transpose(tf.sparse_tensor_dense_matmul(mT,vT))
    return result

class FixedRateGDLearner(learnxcomp.BatchEpochsLearner):
    """ A gradient descent learner.
    """

    def __init__(self,prog,xc=None,compilerClass=DenseMatDenseMsgCrossCompiler,epochs=20,rate=0.1,regularizer=None,tracer=None,epochTracer=None):
        super(FixedRateGDLearner,self).__init__(prog,xc,compilerClass=compilerClass,regularizer=regularizer,tracer=tracer,epochTracer=epochTracer)
        self.epochs=epochs
        self.rate=rate
        self.optimizer = tf.train.GradientDescentOptimizer(learning_rate=rate)

    def trainMode(self,mode,X,Y,epochs=-1):
        if epochs<0: epochs=self.epochs
        if isinstance(mode,str):mode=declare.asMode(mode)
        self.xc.optimizeDataLoss(mode,self.optimizer,X,Y,epochs=epochs)


#!/usr/bin/env python
# -*- coding:utf-8 -*-

import csv
import os
import sys
import time
from datetime import datetime

# date time format
DATE_TIME_FORMAT = '%Y/%m/%d %H:%M:%S'

# all the available rent type
RENT_TYPE_SHARED_BIKE = 1
RENT_TYPE_ELECTRIC_MOPED = 8
RENT_TYPE_SHARED_CAR = 7
RENT_TYPE_PUBLIC_BIKE = 2

# file path for each rent type
PATH_TO_SHARED_BIKE = '/home/haozhexu/data/shared_bike.dat'
PATH_TO_ELECTRIC_MOPED = '/home/haozhexu/data/electric_moped.dat'
PATH_TO_SHARED_CAR = '/home/haozhexu/data/shared_car.dat'
PATH_TO_PUBLIC_BIKE = '/home/haozhexu/data/public_bike.dat'
PATH_TO_ALL_RENT_TYPE = '/home/haozhexu/data/all.dat'

def check_arg():
    if len(sys.argv) != 2:
        print("You need to specify the csv file source. Run it like the following \
        python data_cleaner.py path_to_your_csv_file")

def get_path_to_file():
    """
    return the path to the csv file
    """
    return sys.argv[1]

def read_csv_file_and_write_to_files():
    """
    read the csv file and then print each line to console
    """
    path_to_file = get_path_to_file()
    with open(path_to_file, newline='') as csvfile:

        shared_bike_lines_to_write = []
        electric_moped_lines_to_write = []
        shared_car_lines_to_write = []
        public_bike_lines_to_write = []

        reader = csv.DictReader(csvfile)
        for row in reader:

            lend_time = get_datetime_from_string_csv_field(row['lend_time'])
            rent_long = get_float_number_from_string_csv_field(row['rent_long'])
            rent_lat = get_float_number_from_string_csv_field(row['rent_lat'])
            bike_type = int(row['bike_type'])

            if check_valid_rent_long_and_rent_lat(rent_long, rent_lat) and check_valid_rent_type(bike_type):
                line_to_append = str(lend_time) + '\t' + str(rent_long) + '\t' + str(rent_lat)
                if bike_type == RENT_TYPE_SHARED_BIKE:
                    shared_bike_lines_to_write.append(line_to_append)
                elif bike_type == RENT_TYPE_ELECTRIC_MOPED:
                    electric_moped_lines_to_write.append(line_to_append)
                elif bike_type == RENT_TYPE_SHARED_CAR:
                    shared_car_lines_to_write.append(line_to_append)
                elif bike_type == RENT_TYPE_PUBLIC_BIKE:
                    public_bike_lines_to_write.append(line_to_append)

        print('write file into: {}'.format(PATH_TO_SHARED_BIKE))
        write_to_file(PATH_TO_SHARED_BIKE, shared_bike_lines_to_write)

        print('write file into: {}'.format(PATH_TO_ELECTRIC_MOPED))
        write_to_file(PATH_TO_ELECTRIC_MOPED, electric_moped_lines_to_write)

        print('write file into: {}'.format(PATH_TO_SHARED_CAR))
        write_to_file(PATH_TO_SHARED_CAR, shared_car_lines_to_write)

        print('write file into: {}'.format(PATH_TO_PUBLIC_BIKE))
        write_to_file(PATH_TO_PUBLIC_BIKE, public_bike_lines_to_write)

        print('write file into: {}'.format(PATH_TO_ALL_RENT_TYPE))
        write_to_file(PATH_TO_ALL_RENT_TYPE, shared_bike_lines_to_write
        + electric_moped_lines_to_write
         + shared_car_lines_to_write
          + public_bike_lines_to_write)

def get_float_number_from_string_csv_field(string_field_csv):
    """convert the raw field from csv file to float type, in case the field is
    empty, just assign 0
    """
    if string_field_csv == '':
        return 0
    else:
        return float(string_field_csv)

def get_datetime_from_string_csv_field(string_field_csv):
    """convert the raw field from csv file to datetime type.
    one example of the date fime from csv file is "2018/12/2 12:54:34"
    """
    return datetime.strptime(string_field_csv, DATE_TIME_FORMAT)

def check_valid_rent_type(rent_type):
    """check if the given rent_type falls into the pre-defined rent type set
    """
    valid_rent_types = {RENT_TYPE_ELECTRIC_MOPED, RENT_TYPE_PUBLIC_BIKE, RENT_TYPE_SHARED_BIKE, RENT_TYPE_SHARED_CAR}
    return rent_type in valid_rent_types

def check_valid_rent_long_and_rent_lat(rent_long, rent_lat):
    """checks if any given long or lat is 0 or not.
    A valid (long, lat) should be (long>0, lat>0) return false if not valid
    """
    return not(rent_long == 0 or rent_lat == 0)

def write_to_file(file_path, contents):
    """create a file based on the file path if it does not exist yet.
    delete the file if it has already exists and then create a new one.
    """
    try:
        if os.path.exists(file_path):
            os.remove(file_path)

        with open(file_path, 'w+') as file:
            if contents:
                file.write('\n'.join(contents))
    except FileNotFoundError:
        print('Something went wrong when creating: {}'.format(file_path))

if __name__ == '__main__':
    check_arg()
    read_csv_file_and_write_to_files()


# -*- coding: utf-8 -*-
# import module snippets
from __future__ import print_function
import httplib2
from apiclient.discovery import build
from oauth2client.service_account import ServiceAccountCredentials

class Base(object):
    user = None
    http = None

    def __init__(
        self, credential_path: str, scopes: list, delegate_user: str = None
    ):
        """
        @params[in]     credentail_path GoogleのClient作成時のjsonのパス
        @params[in]     scopes          アクセスのスコープ一覧
        @params[in]     delegate_user   アクセスユーザーを指定
        """
        credentials = ServiceAccountCredentials.from_json_keyfile_name(
            credential_path,
            scopes=scopes
        )
        if delegate_user is not None:
            self.user = delegate_user
            credentials = credentials.create_delegated(delegate_user)
        self.http = credentials.authorize(httplib2.Http())


from pydot import Node, Edge, Dot
from collections import defaultdict

import htmlentitydefs
import re
import string
import database as db

# this pattern matches substrings of reserved and non-ASCII characters
from database.model import Team, RouteDistance

pattern = re.compile(r"[&<>\"\x80-\xff]+")

# create character map
entity_map = {}

for i in range(256):
    entity_map[chr(i)] = "&#%d;" % i

for entity, char in htmlentitydefs.entitydefs.items():
    if entity_map.has_key(char):
        entity_map[char] = "&%s;" % entity

def escape_entity(m, get=entity_map.get):
    return string.join(map(get, m.group()), "")

def escape(string):
    return pattern.sub(escape_entity, string)

colors = {0: "#ffbbbb",
          1: "#bbffbb",
          2: "#bbbbff"}

types = {0: "Vorspeise",
         1: "Hauptgang",
         2: "Nachspeise"}

def make_nodes(plan, teams, names=True):
    nodes = {}
    for team in plan:
        team_part = plan[team].index(team)
        if names:
            teamname = escape(teams[team].name.encode("latin1"))
        else:
            teamname = "Team %s" % team
        nodes[team] = Node(team,
                           label=u"%s\\n%s" % (teamname, types[team_part]),
                           style="filled",
                           fillcolor=colors[team_part])
    return nodes

def graph_way(team, team_plan, nodes, distances=None, with_dist=False, with_label=False):
    edges = []
    for station in team_plan:
        if station != team:
            edge_args = {}
            length = distances[team][station] / 1000.0
            if with_dist:
                edge_args.update(dict(len=length*2, w=100))
            if with_label:
                edge_args.update(dict(label="%0.1fkm" % length,
                                      fontcolor="gray"))

            edge = Edge(nodes[team], nodes[station], **edge_args)
            edges.append(edge)
    return edges

def fetch_teams():
    teams = {}
    for team in db.session.query(Team).filter_by(deleted=False).filter_by(confirmed=True, backup=False):
        teams[str(team.id)] = team
    return teams

def fetch_distances():
    distances = defaultdict(dict)
    for dist in db.session.query(RouteDistance):
        distances[str(dist.location_from.team_id)][str(dist.location_to.team_id)] = dist.distance

    return distances

def process_plan(filename, plan, names=True, with_dist=False, with_label=False):
    graph = Dot(overlap="false", splines="true", esep=.2)
    teams = fetch_teams()

    distances = fetch_distances()

    nodes = make_nodes(plan, teams, names)

    for team_node in nodes:
        graph.add_node(nodes[team_node])

    for team in plan:
        edges = graph_way(team, plan[team], nodes, distances, with_dist, with_label)
        for edge in edges:
            graph.add_edge(edge)

    if with_dist:
        graph.write_png(filename, prog="neato")
    else:
        graph.write_png(filename, prog="dot")


# --- VTK-PYTHON SCRIPT FOR READING MESH VTP AND DACH1 VTI TO MAP DACH1 VALUES ONTO MESH SURFACE
# --- WRITTEN BY SUHAAS ANBAZHAKAN
# --- BASED ON A SCRIPT BY AEKAANSH VERMA

import sys
import vtk
import numpy as np
from vtk.util import numpy_support as VN

vtk_file_prefix = 'all_results'
WSS_CONV_FACTOR = 1
P_CONV_FACTOR = .133333333333333
INTERPOLATING_RADIUS = 5
GLOBAL_AVERAGE = 80
THRESHOLD = 20
a = 50
b = 50
# First, read in the .vtp file for mesh and Dach1
datareader=vtk.vtkXMLImageDataReader()
datareader.SetFileName('test.vti')
datareader.Update()

ref = vtk.vtkXMLImageDataReader()
ref.SetFileName('E175_H2_Dach1_ImageData_Combined.vti')
ref.Update()

# Read your data into another polydata variable for reading
mesh=vtk.vtkPolyData()
mesh=datareader.GetOutput()

imageData=vtk.vtkPolyData()
imageData=ref.GetOutput()

numPts=imageData.GetNumberOfPoints()

normalizedDach1Array = vtk.vtkDoubleArray()

ImageData_numpy = VN.vtk_to_numpy(imageData.GetPointData().GetArray('Dach1'))
GLOBAL_AVERAGE = np.mean(ImageData_numpy)
GLOBAL_max = np.max(ImageData_numpy)

sum = 0
count = 0
for imageDataPointID in xrange(0, numPts):
	imageDataPointCoordinate = imageData.GetPoint(imageDataPointID)
	x,y,z = imageData.GetPoint(imageDataPointID)
	kernel_size = np.array([a,b]);
	if(imageData.GetPointData().GetArray('Dach1').GetValue(imageDataPointID)>THRESHOLD):
		sum = sum + imageData.GetPointData().GetArray('Dach1').GetValue(imageDataPointID)
		count = count + 1
GLOBAL_AVERAGE = sum/numPts

for imageDataPointID in xrange(0, numPts):
	imageDataPointCoordinate = imageData.GetPoint(imageDataPointID)
	x,y,z = imageData.GetPoint(imageDataPointID)
	kernel_size = np.array([a,b]);
	initialDach1Value = imageData.GetPointData().GetArray('Dach1').GetValue(imageDataPointID)
	if(imageDataPointID % 100 ==0):
		print(initialDach1Value)
	sum = 0
	count = 1
	percentDone = float(imageDataPointID/numPts*100)
	for kernel_x in xrange(-a, a):
		for kernel_y in xrange(-b, b):
			if(x-a>0 and x+a<1000 and y-b>0 and y+b<1000):
				x = x - a
				y = y - b
				kernelPointCoordinate = x,y,z
				kernelPointID = imageData.FindPoint(kernelPointCoordinate)
				sum = sum +  imageData.GetPointData().GetArray('Dach1').GetValue(kernelPointID)
				count = count + 1

	local_average = sum/count
	if(local_average>0):
		normalizedDach1 = GLOBAL_AVERAGE/local_average*initialDach1Value
		normalizedDach1Array.InsertNextValue(normalizedDach1)

# Convert Dach1 and Pressure to micron units and new arrays to point data
normalizedDach1Array.SetName('Normalized Dach1 (' + str(a) + ',' + str(b) + ')')
imageData.GetPointData().AddArray(normalizedDach1Array)

# Write a new .vtp file that has all converted values and mapped values.
w = vtk.vtkXMLImageDataWriter()
w.SetInputData(imageData)
w.SetFileName('E175_H2_Dach1_ImageData_Normalized.vti')
w.Write()


from tests import unittest
import time

import mock
import dateutil.parser

import botocore.parameters
import botocore.exceptions

class FakeService(object):

    def __init__(self, timestamp_format):
        self.timestamp_format = timestamp_format

class FakeOperation(object):

    def __init__(self, timestamp_format):
        self.service = FakeService(timestamp_format)

class TestParameters(unittest.TestCase):

    def test_iso_timestamp_from_iso(self):
        op = FakeOperation('iso8601')
        p = botocore.parameters.TimestampParameter(op, name='paramname',
                                                   type='timestamp')
        d = {}
        ts = '2012-10-12T00:00'
        dt = dateutil.parser.parse(ts)
        p.build_parameter_query(ts, d)
        self.assertEqual(d['paramname'], dt.isoformat())
        with self.assertRaisesRegexp(botocore.exceptions.ValidationError,
                                     'timestamp:paramname'):
            p.build_parameter_query(value='not a date string',
                                    built_params=d)

    def test_iso_timestamp_from_epoch(self):
        op = FakeOperation('iso8601')
        p = botocore.parameters.TimestampParameter(op, name='foo')
        d = {}
        iso = '2013-04-11T23:23:15'
        epoch = 1365722595
        p.build_parameter_query(epoch, d)
        self.assertEqual(d['foo'], iso)
        with self.assertRaises(botocore.exceptions.ValidationError):
            p.build_parameter_query(value='not a date string',
                                    built_params=d)

    def test_epoch_timestamp_from_iso(self):
        op = FakeOperation('unixTimestamp')
        p = botocore.parameters.TimestampParameter(op, name='foo')
        d = {}
        iso = '2013-04-11T23:23:15'
        epoch = 1365722595
        p.build_parameter_query(iso, d)
        self.assertEqual(d['foo'], str(epoch))
        with self.assertRaises(botocore.exceptions.ValidationError):
            p.build_parameter_query(value='not a date string',
                                    built_params=d)

    def test_epoch_timestamp_from_epoch(self):
        op = FakeOperation('unixTimestamp')
        p = botocore.parameters.TimestampParameter(op, name='foo')
        d = {}
        epoch = 1365722595
        p.build_parameter_query(epoch, d)
        self.assertEqual(d['foo'], str(epoch))
        with self.assertRaises(botocore.exceptions.ValidationError):
            p.build_parameter_query(value='not a date string',
                                    built_params=d)

    def test_string(self):
        p = botocore.parameters.StringParameter(None, name='foo')
        d = {}
        value = 'This is a test'
        p.build_parameter_query(value, d)
        self.assertEqual(d['foo'], value)
        with self.assertRaises(botocore.exceptions.ValidationError):
            p.build_parameter_query(value=1,
                                    built_params=d)

    def test_integer(self):
        p = botocore.parameters.IntegerParameter(None, name='foo')
        d = {}
        p.build_parameter_query(123, d)
        self.assertEqual(d['foo'], '123')
        p.build_parameter_json(123, d)
        self.assertEqual(d['foo'], 123)
        with self.assertRaises(botocore.exceptions.ValidationError):
            p.build_parameter_query(value=123.4,
                                    built_params=d)

    def test_integer_range(self):
        p = botocore.parameters.IntegerParameter(None, name='foo',
                                                 min=0, max=10)
        d = {}
        p.build_parameter_query(9, d)
        self.assertEqual(d['foo'], '9')
        with self.assertRaises(botocore.exceptions.ValidationError):
            p.build_parameter_query(value=8.4,
                                    built_params=d)
        with self.assertRaisesRegexp(botocore.exceptions.RangeError,
                                     'foo'):
            p.build_parameter_query(value=100,
                                    built_params=d)

    def test_float(self):
        p = botocore.parameters.FloatParameter(None, name='foo')
        d = {}
        p.build_parameter_query(123.4, d)
        self.assertEqual(d['foo'], '123.4')
        p.build_parameter_json(123.4, d)
        self.assertEqual(d['foo'], 123.4)
        with self.assertRaises(botocore.exceptions.ValidationError):
            p.build_parameter_query(value=True,
                                    built_params=d)

    def test_float_range(self):
        p = botocore.parameters.FloatParameter(None, name='foo',
                                               min=0, max=10)
        d = {}
        p.build_parameter_query(9.0, d)
        self.assertEqual(d['foo'], '9.0')
        p.build_parameter_json(9.0, d)
        self.assertEqual(d['foo'], 9.0)
        with self.assertRaises(botocore.exceptions.RangeError):
            p.build_parameter_query(value=100.0,
                                    built_params=d)

    def test_boolean(self):
        p = botocore.parameters.BooleanParameter(None, name='foo')
        d = {}
        p.build_parameter_query('true', d)
        self.assertEqual(d['foo'], 'true')
        p.build_parameter_query('True', d)
        self.assertEqual(d['foo'], 'true')
        p.build_parameter_query('TRUE', d)
        self.assertEqual(d['foo'], 'true')
        p.build_parameter_query('false', d)
        self.assertEqual(d['foo'], 'false')
        p.build_parameter_query('False', d)
        self.assertEqual(d['foo'], 'false')
        p.build_parameter_query('FALSE', d)
        self.assertEqual(d['foo'], 'false')
        p.build_parameter_query(True, d)
        self.assertEqual(d['foo'], 'true')
        p.build_parameter_json(True, d)
        self.assertEqual(d['foo'], True)
        with self.assertRaises(botocore.exceptions.ValidationError):
            p.build_parameter_query(value='100',
                                    built_params=d)

    def test_list_fails_member_validation(self):
        p = botocore.parameters.ListParameter(operation=None, name='foo',
                                              members={'type': 'string'})
        d = {}
        # None is not of type string.
        value = ['This', 'is', None, 'test']
        with self.assertRaises(botocore.exceptions.ValidationError):
            p.build_parameter('query', value, d)

    def test_list_type_validation(self):
        p = botocore.parameters.ListParameter(operation=None, name='foo',
                                              members={'type': 'string'})
        d = {}
        # None is not of type string.
        value = 'not a list'
        with self.assertRaises(botocore.exceptions.ValidationError):
            p.build_parameter('query', value, d)

    def test_list_struct_fails_validation(self):
        members = {
            "shape_name": "Foo",
            "type": "structure",
            "members": {
                "foo": {
                    "shape_name": "String",
                    "type": "string",
                    "required": True,
                },
            },
        }
        p = botocore.parameters.ListParameter(operation=None, name='foo',
                                              members=members)
        d = {}
        # 'notfoo' is an unknown key so this will raise an exception
        # when we try to validate this value.
        value = [{'foo': 'bar'}, {'foo': 'baz'}, {'notfoo': 'bad'}]
        with self.assertRaisesRegexp(botocore.exceptions.ValidationError,
                                     'element of None:foo'):
            p.build_parameter('query', value, d)

    def test_plain_list(self):
        # Test a plain vanilla list.
        p = botocore.parameters.ListParameter(operation=None, name='foo',
                                              members={'type': 'string'})
        d = {}
        value = ['This', 'is', 'a', 'test']
        p.build_parameter_query(value, d)
        self.assertEquals(d, {'foo.member.4': 'test',
                              'foo.member.2': 'is',
                              'foo.member.3': 'a',
                              'foo.member.1': 'This'})

    def test_list_with_xmlname(self):
        # Test a list where the member specifies an xmlname.
        # This should be ignored.
        p = botocore.parameters.ListParameter(
            operation=None, name='foo',
            members={'type': 'string', 'xmlname': 'bar'})
        d = {}
        value = ['This', 'is', 'a', 'test']
        p.build_parameter_query(value, d)
        self.assertEquals(d, {'foo.member.4': 'test',
                              'foo.member.2': 'is',
                              'foo.member.3': 'a',
                              'foo.member.1': 'This'})
        d = {}
        p.build_parameter_json(value, d)
        self.assertEquals(d, {'foo': ['This', 'is', 'a', 'test']})

    def test_flattened_list(self):
        # Test a flattened list.  Member should always define
        # and xmlname attribute.
        p = botocore.parameters.ListParameter(
            operation=None, name='foo', flattened=True,
            members={'type': 'string', 'xmlname': 'bar'})
        d = {}
        value = ['This', 'is', 'a', 'test']
        p.build_parameter_query(value, d)
        self.assertEquals(d, {'bar.4': 'test',
                              'bar.1': 'This',
                              'bar.2': 'is',
                              'bar.3': 'a'})

    def test_type_blob_validation(self):
        p = botocore.parameters.BlobParameter(
            operation=None, name='Foo', payload=True)
        bytearray_type = bytearray(b'foo')
        byte_type = bytes(b'foo')
        str_type = str('foo')
        self.assertEqual(p.validate(str_type), str_type)
        self.assertEqual(p.validate(byte_type), byte_type)
        self.assertEqual(p.validate(bytearray_type), bytearray_type)

    def test_blob_conversion_to_base64(self):
        # Blobs that are not in the payload should be base64-encoded
        p = botocore.parameters.BlobParameter(
            operation=None, name='Foo', payload=False)
        self.assertEqual(p.validate('foo'), 'Zm9v')

class TestStructureParamaters(unittest.TestCase):

    def setUp(self):
        self.members = {
            "AvailabilityZone": {
              "shape_name": "String",
              "type": "string",
              "documentation": "",
            },
            "GroupName": {
              "shape_name": "String",
              "type": "string",
              "documentation": "",
            },
            "Tenancy": {
              "shape_name": "String",
              "type": "string",
              "documentation": "",
            }
        }
        self.p = botocore.parameters.StructParameter(
            operation=None, name='Foo',
            members=self.members)

    def test_struct_parameter_query_build(self):
        d = {}
        self.p.build_parameter_query({'AvailabilityZone': 'foo',
                                      'GroupName': 'bar',
                                      'Tenancy': 'baz'}, d)
        self.assertEqual(d, {
            'Foo.AvailabilityZone': 'foo',
            'Foo.GroupName': 'bar',
            'Foo.Tenancy': 'baz',
        })

    def test_struct_type_validation(self):
        self.p.validate({'AvailabilityZone': 'foo',
                         'GroupName': 'bar',
                         'Tenancy': 'baz'})
        with self.assertRaisesRegexp(
                botocore.exceptions.ValidationError,
                ("Invalid value \('not a dict'\) for param "
                 "None:Foo of type structure")):
            self.p.validate('not a dict')

    def test_struct_unknown_keys_validation(self):
        # Unknown key 'group_name' key
        with self.assertRaises(botocore.exceptions.UnknownKeyError):
            # Missing the 'GroupName' key
            self.p.validate({'AvailabilityZone': 'foo',
                             'group_name': 'unknown',
                             'Tenancy': 'baz'})

if __name__ == "__main__":
    unittest.main()


from PIL import Image
from torchvision.transforms import transforms
import numpy as np

# convert Tensor to img
def from_array(inp):
    inp = inp.numpy().transpose((1, 2, 0))
    mean = np.array([0.485, 0.456, 0.406])
    std = np.array([0.229, 0.224, 0.225])
    inp = (std * inp + mean) * 255
    return inp.astype(np.uint8)

def load_img(img):
    image = Image.open(img).convert('RGB')
    image.load()
    return image

def prepare_sample(img):
    image = img.resize((244, 244))
    return np.array(image)

def pre_process_img(img, mode):
    transform = transforms.Compose([
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ])
    x = load_img(img) if mode != 'train' else img
    x = prepare_sample(x) if mode != 'train' else img
    x = transform(x)
    return x


import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import numpy as np
import random

r = pd.read_csv("MuchasR.csv", delimiter=',')

plt.plot(r['y'],r['x'],'g.', label='Crecimiento en de M')

legend = plt.legend(loc='upper center', shadow=True)
frame = legend.get_frame()
frame.set_facecolor('0.90')
for label in legend.get_texts() :
    label.set_fontsize('medium')

for label in legend.get_lines():
    label.set_linewidth(1.5)

plt.ylabel("Tiempo en segundos")
plt.xlabel("Cantidad de Rutas total")
plt.show()