import pandas
import numpy as np
import random

from .recommender_system_base import RecommenderSystemBase

class HashTable:
    def __init__(self, hash_size, inp_dimensions):
        self.hash_size = hash_size
        self.inp_dimensions = inp_dimensions
        self.hash_table = dict()
        self.projections = np.random.randn(self.hash_size, inp_dimensions)

    def generate_hash(self, inp_vector):
        bools = (np.dot(inp_vector, self.projections.T) > 0).astype('int')
        return ''.join(bools.astype('str'))

    def __setitem__(self, inp_vec, label):
        #print("generating hash value for hash table generation")
        hash_value = self.generate_hash(inp_vec)
        #print("completed generating hash value for hash table generation")
        self.hash_table[hash_value] = self.hash_table.get(hash_value, list()) + [label]
        #print("completed Setting values to hash table")
    def __getitem__(self, inp_vec):
        #print("generating hash value for searching hash table")
        hash_value = self.generate_hash(inp_vec)
        return self.hash_table.get(hash_value, [])

class LSH:
    def __init__(self, num_tables, hash_size, inp_dimensions):
        self.num_tables = num_tables
        self.hash_size = hash_size
        self.inp_dimensions = inp_dimensions
        self.hash_tables = list()
        for i in range(self.num_tables):
            self.hash_tables.append(HashTable(self.hash_size, self.inp_dimensions))

    def __setitem__(self, inp_vec, label):
        #print("generating hash tables")
        for table in self.hash_tables:
            table[inp_vec] = label
        #print("hash table generation complete")

    def __getitem__(self, inp_vec):
        results = list()
        for table in self.hash_tables:
            #print("looping through hash tables")
            results.extend(table[inp_vec])
            #print("suggestions from table ",results)

        return list(set(results))

class ItemItemRecommenderSystem(RecommenderSystemBase):
    """
    Attributes
    ----------

    Methods
    -------
    compute_movie_embeddings
        Computes the movie embeddings.

    recommend_similar_movies
        Recommends the k most similar of the movie with the id 'movie_id'.

    recommend_movies_to_user
        Given a user with a watch history, it recommends the k movies that he will most likely watch.

    get_movies_embeddings
        Returns the embedding of the movies with movie_id in movie_ids.

    Notes
    -----
    - You can add other attributes and methods to this class.
    - In the constructor parameters, you can add other datasets if you need them.

    Examples
    --------
    >>> rec_sys = ItemItemRecommenderSystem(**kwargs)
    >>> ...
    >>> rec_sys.recommend_similar_movies(movie_id='the_promise-das_versprechen-en-1995', k=10)
    ...
    >>> rec_sys.recommend_movies_to_user(user_id=25, k=10)
    ...
    >>> movie_embeddings = rec_sys.get_movies_embeddings(movie_ids)
    >>> visualize_embeddings(movie_embeddings)
    ...
    """

    def __init__(self, ratings_dataframe: pandas.DataFrame, movies_metadata_dataframe: pandas.DataFrame,
                 keywords_dataframe: pandas.DataFrame, credits_dataframe: pandas.DataFrame) -> None:
        """Sets the movie_embeddings attribute.

        Parameters
        ----------
        ratings_dataframe : pandas.DataFrame
            The movie ratings of users.
        movies_metadata_dataframe : pandas.DataFrame
            The movies metadata.
        keywords_dataframe : pandas.DataFrame
            The movies keywords.
        credits_dataframe : pandas.DataFrame
            The movies credits.
        """
        print("starting init")
        super().__init__(ratings_dataframe, movies_metadata_dataframe, keywords_dataframe, credits_dataframe)
        self.movie_embeddings = self.make_embeddings(self.movies_dataframe, 'movie')
        self.user_embeddings = self.make_embeddings(self.movies_dataframe, 'user')
        print("ending init")

    def make_embeddings(self, merged_movie_data, emb_type):
        print("make embedding")

        if emb_type == 'movie':
            ratings_matrix = merged_movie_data.pivot_table(columns='userId',index='movie_id',values='rating')
        else:
            ratings_matrix = merged_movie_data.pivot_table(columns='movie_id',index='userId',values='rating')

        ratings_matrix.dropna(axis=1, how='all', inplace=True)
        ratings_matrix.fillna( 0, inplace = True )
        print("ending embedding")
        return ratings_matrix

    def getJaccardSim(self, movie1,movie2):
        movie1=''.join((movie1 > 3).astype(int).astype('str'))
        movie2=''.join((movie2 > 3).astype(int).astype('str'))
        N = 0
        D = 0
        for i in range(len(movie1)):
            sum_ = int(movie1[i]) + int(movie2[i])
            if(sum_ >= 1):
                flag = 1
                D += 1
                if(sum_ == 2):
                    N += 1
        if D == 0:
            return 0
        return(float(N)/D)

    def getCosineSim(self, movie1,movie2):
        return np.dot(movie1,movie2)/(np.linalg.norm(movie1)*np.linalg.norm(movie2))

    def recommend_movies_to_user(self, user_id: int, k: int, algo) -> pandas.DataFrame:
        """Given a user with a watch history, it recommends the k movies that he will most likely watch.

        user_favourite_movies = the set of movies that the user watched and liked.

        If len(user_favourite_movies) = 0:
            Recommend k random movies from the set of highly rated movies in the dataset.
            These k movies should be chosen randomly. So if the function is executed 2 times, it should
            return different results.

        If k < len(user_favourite_movies):
            Select a random set of movies from the user_favourite_movies set and recommend a movie for each item.

        If k > len(user_favourite_movies):
            Select n movies for each movie the user liked.
            Example :
                k = 10 and len(user_favourite_movies) = 1
                    Recommend 10 movies that are similar to the movie the user watched.
                k = 10 and len(user_favourite_movies) = 3
                    Recommend:
                        3 movies that are similar the 1st movie the user liked.
                        3 movies that are similar the 2nd movie the user liked.
                        4 movies that are similar the 3rd movie the user liked.

        Parameters
        ----------
        user_id : int
            The id of the user
        k : int
            The number of movies to recommend

        Returns
        -------
        pandas.DataFrame
            A subset of the movies_dataframe with the k movies that the user may like.
        """

        from scipy.sparse import csr_matrix
        embeddings_sparse = csr_matrix(self.movie_embeddings.values)

        from sklearn.neighbors import NearestNeighbors

        user_favourite_movies = self.movies_dataframe[self.movies_dataframe.userId == user_id][self.movies_dataframe.rating >= 3].movie_id.tolist()
        #print("favorite movies",user_favourite_movies)

        if len(user_favourite_movies) == 0:
            return self.movies_dataframe[self.movies_dataframe.rating >= 4].sample(k)

        elif algo == 'KNN':

            if k < len(user_favourite_movies):
                user_favourite_movies = random.sample(user_favourite_movies, k)
                model = NearestNeighbors(n_neighbors=k,algorithm='brute',metric='cosine')
                model.fit(embeddings_sparse)

                movie_embeddings = self.get_movies_embeddings(user_favourite_movies)
                distances,suggestions=model.kneighbors(movie_embeddings.values)

                movies = []
                distance = []
                for i in user_favourite_movies:
                    movie_embeddings = self.get_movies_embeddings(i)
                    distances,suggestions=model.kneighbors(movie_embeddings.values.reshape(1, -1),2)
                    distances= distances.flatten()
                    suggestions= suggestions.flatten()

                    for i in range(1,len(suggestions)):
                        movie_id=self.movie_embeddings.index[suggestions[i]]
                        movies.append(movie_id)
                        distance.append(distances[i])

                return self.movies_dataframe.loc[self.movies_dataframe['movie_id'].isin(movies)].drop_duplicates(subset=['movie_id'])

            elif k > len(user_favourite_movies):
                n = len(user_favourite_movies)
                q = k//n
                r = k%n
                k_values = []
                for _ in range(n):
                    k_values.append(q)
                k_values[-1] += r

                movies = []
                distance = []

                model = NearestNeighbors(n_neighbors=k_values[-1],algorithm='brute',metric='cosine')

                model.fit(embeddings_sparse)

                for idx,i in enumerate(k_values):
                    movie_embeddings = self.get_movies_embeddings(user_favourite_movies[idx])
                    distances,suggestions=model.kneighbors(movie_embeddings.values.reshape(1, -1),i+1)
                    distances= distances.flatten()
                    suggestions= suggestions.flatten()

                    for i in range(1,len(suggestions)):
                        movie_id=self.movie_embeddings.index[suggestions[i]]
                        movies.append(movie_id)
                        distance.append(distances[i])

                return self.movies_dataframe.loc[self.movies_dataframe['movie_id'].isin(movies)].drop_duplicates(subset=['movie_id'])

        else:
            if k < len(user_favourite_movies):
                user_favourite_movies = random.sample(user_favourite_movies, k)
                movies = []
                for user_fav in user_favourite_movies:
                    res = self.recommend_similar_movies(user_fav, 1, algo)
                    movies.append(res.movie_id.values[0])
                return self.movies_dataframe.loc[self.movies_dataframe['movie_id'].isin(movies)].drop_duplicates(subset=['movie_id'])

            elif k > len(user_favourite_movies):
                n = len(user_favourite_movies)
                q = k//n
                r = k%n
                k_values = []
                for _ in range(n):
                    k_values.append(q)
                k_values[-1] += r

                movies = []
                for i in range(n):
                    res = self.recommend_similar_movies(user_favourite_movies[i], k_values[i], algo)
                    movies.extend(res.movie_id.tolist())

                return self.movies_dataframe.loc[self.movies_dataframe['movie_id'].isin(movies)].drop_duplicates(subset=['movie_id'])

    def recommend_similar_movies(self, movie_id: str, k: int, algo) -> pandas.DataFrame:
        """Recommends the k most similar movies of the movie with the id 'movie_id'.

        Parameters
        ----------
        movie_id : str
            The id of the movie.
        k : int
            The number of similar movies to recommend.

        Returns
        -------
        pandas.DataFrame
            A subset of the movies_dataframe with the k similar movies of the target movie (movie_id).
        """

        if algo == 'knn':

            from scipy.sparse import csr_matrix
            embeddings_sparse = csr_matrix(self.movie_embeddings)

            from sklearn.neighbors import NearestNeighbors
            model = NearestNeighbors(n_neighbors=k,algorithm='brute',metric='cosine')

            model.fit(embeddings_sparse)

            #condition = self.movies_dataframe['movie_id']==movie_id
            #idVal= self.movies_dataframe[condition].drop_duplicates(subset=['movie_id'])['movieId']
            #print("Movie id", idVal)
            movie_embeddings = self.get_movies_embeddings(movie_id)
            distances,suggestions=model.kneighbors(movie_embeddings.values.reshape(1,-1),k+1)
            suggestions= suggestions.flatten()

            print(suggestions)
            movies = []
            for i in range(1,len(suggestions)):
                movies.append(self.movie_embeddings.index[suggestions[i]])

            return self.movies_dataframe.loc[self.movies_dataframe['movie_id'].isin(movies)].drop_duplicates(subset=['movie_id'])

        else:
            nusers = self.movie_embeddings.columns
            nmovies = self.movie_embeddings.index

            hash_table = LSH(num_tables=20,hash_size=10, inp_dimensions=len(nusers))

            for i in range(len(nmovies)):
                hash_table[self.movie_embeddings.loc[nmovies[i]]]=nmovies[i]

            inp_vec=self.movie_embeddings.loc[movie_id]
            # print("Movie_id" ,nmovies[movie_id])
            similar_movies = hash_table[inp_vec]
            cos_sim_values =[]
            jac_sim_values=[]

            for a in similar_movies:
                if a== movie_id:
                    continue
                out_vec = self.movie_embeddings.loc[a]
                cos_sim_values.append(self.getCosineSim(inp_vec,out_vec))
                jac_sim_values.append(self.getJaccardSim(inp_vec,out_vec))

            if algo == 'LSH-C':
                ranked_cos_sim = np.argsort(np.array(cos_sim_values))
                movies_id_cos = ranked_cos_sim[::-1][:k]
                cos_sugg = []
                for i in range(0,k):
                    movie_sugg_cos = similar_movies[movies_id_cos[i]]
                    cos_sugg.append(self.movies_dataframe[self.movies_dataframe["movie_id"]==str(movie_sugg_cos)]["movie_id"].values[0])
                return self.movies_dataframe.loc[self.movies_dataframe["movie_id"].isin(cos_sugg)].drop_duplicates(subset=['movie_id'])

            elif algo == 'LSH-J':
                ranked_jac_sim = np.argsort(np.array(jac_sim_values))
                movies_id_jac = ranked_jac_sim[::-1][:k]
                jac_sugg = []
                for i in range(0,k):
                    movie_sugg_jac= similar_movies[movies_id_jac[i]]
                    jac_sugg.append(self.movies_dataframe[self.movies_dataframe["movie_id"]==str(movie_sugg_jac)]["movie_id"].values[0])
                return self.movies_dataframe.loc[self.movies_dataframe["movie_id"].isin(jac_sugg)].drop_duplicates(subset=['movie_id'])

    def get_movies_embeddings(self, movie_ids: [str]) -> pandas.DataFrame:
        """Returns the embedding of the movies with movie_id in movie_ids.

        Parameters
        ----------
        movie_ids : [str]
            List of the movies movie_id.

        Returns
        -------
        pandas.DataFrame
            The embeddings of the movies with movie_id in movie_ids.
        """

        return self.movie_embeddings.loc[movie_ids,:]


import click
import json
import os
from pathlib import Path
from typing import Optional, Tuple
from zipfile import ZipFile, BadZipFile

from cookiecutter_project_upgrader.logic import update_project_template_branch

@click.command(context_settings={"help_option_names": ["-h", "--help"]})
@click.option('--context-file', '-c', type=click.Path(file_okay=True, readable=True, allow_dash=True),
              default="docs/cookiecutter_input.json", help="Default: docs/cookiecutter_input.json")
@click.option('--branch', '-b', default="cookiecutter-template", help="Default: cookiecutter-template")
@click.option('--upgrade-branch', '-u',
              help="Optional branch name of cookiecutter template to checkout before upgrading.")
@click.option('--zip-file', '-f', help="Zip file Path/URL for Cookiecutter templates.")
@click.option('--interactive', '-i', is_flag=True,
              help="Enter interactive mode. Default behaviour: skip questions, use defaults.")
@click.option('--merge-now', '-m', type=bool, default=None,
              help="Execute a git merge after a successful update, default: ask if interactive, otherwise false.")
@click.option('--push-template-branch-changes', '-p', type=bool, default=None,
              help="Push changes to the remote Git branch on a successful update, "
                   "default: ask if interactive, otherwise false.")
@click.option('--exclude', '-e', type=str, multiple=True,
              help='Git pathspecs to exclude from the update commit, e.g. -e "*.py" -e "tests/".')
def main(context_file: str, branch: str, upgrade_branch: Optional[str],
         zip_file: Optional[str], interactive: bool, merge_now: Optional[bool],
         push_template_branch_changes: Optional[bool],
         exclude: Tuple[str, ...]):
    """Upgrade projects created from a Cookiecutter template"""
    context = _load_context(context_file)
    if zip_file is not None and _is_valid_file(zip_file):
        click.echo(f"Using zip-file: {zip_file} for upgrade.")
        context['_template'] = zip_file
    project_directory = os.getcwd()
    update_project_template_branch(context,
                                   project_directory,
                                   branch,
                                   upgrade_branch,
                                   merge_now,
                                   push_template_branch_changes,
                                   exclude,
                                   interactive)

def _load_context(context_file: str):
    try:
        context_str = Path(context_file).read_text(encoding="utf-8")
    except FileNotFoundError:
        click.echo(f"Context file not found at: {context_file}\n"
                   f"Make sure you are in the right directory")
        exit(1)
    context = json.loads(context_str)
    return context

def _is_valid_file(zip_file: str):
    # we check only for local files for valid, for remote URLs, cookiecutter can do the test if its valid.
    zip_path = Path(zip_file)
    try:
        if zip_path.exists():
            ZipFile(Path(zip_file)).testzip()
    except BadZipFile:
        click.echo(f"Bad Zip file at: {zip_file}\n"
                   f"Make sure your zip file is correct.")
        exit(1)
    return True


from django.shortcuts import render
from account.forms import LoginForm,RegistrationForm,UserProfileForm
from django.http import HttpResponse
from django.contrib.auth.decorators import login_required
from .models import UserProfile,UserInfo
from django.contrib.auth.models import User
from django.http import HttpResponseRedirect
from .forms import UserProfileForm,UserInfoForm,UserForm
from django.core.urlresolvers import reverse

# Create your views here.

def register(request):
    if request.method == 'POST':
        user_form = RegistrationForm(request.POST)
        userprofile_form = UserProfileForm(request.POST)
        if user_form.is_valid()*userprofile_form.is_valid():
            new_user = user_form.save(commit=False)
            new_user.set_password(user_form.cleaned_data['password'])
            new_user.save()
            new_profile = userprofile_form.save(commit=False)
            new_profile.user = new_user
            new_profile.save()
            UserInfo.objects.create(user=new_user)
            return HttpResponseRedirect(reverse('account:user_login'))
        else:
            return HttpResponse('sorry,you can not register')
    else:
        user_form = RegistrationForm()
        userprofile_form = UserProfileForm()
        return render(request,'account/register.html',{'form':user_form,'profile':userprofile_form})

@login_required(login_url='/account/login/')
def myself(request):
    user = User.objects.get(username=request.user.username)
    userprofile=UserProfile.objects.get(user=user)
    userinfo = UserInfo.objects.get(user=user)
    return render(request,'account/myself.html',{'user':user,'userinfo':userinfo,'userprofile':userprofile})

@login_required(login_url='/account/login/')
def myself_edit(request):
    user = User.objects.get(username=request.user.username)
    userprofile=UserProfile.objects.get(user=request.user)
    userinfo = UserInfo.objects.get(user=request.user)

    if request.method == 'POST':
        user_form = UserForm(request.POST)
        userprofile_form = UserProfileForm(request.POST)
        userinfo_form = UserInfoForm(request.POST)
        if user_form.is_valid() * userprofile_form.is_valid() * userinfo_form.is_valid():
            user_cd =user_form.cleaned_data
            userprofile_cd =userprofile_form.cleaned_data
            userinfo_cd =userinfo_form.cleaned_data
            print(user_cd['email'])
            user.email = user_cd['email']
            userprofile.birth=userprofile_cd['birth']
            userprofile.phone=userprofile_cd['phone']
            userinfo.school=userinfo_cd['school']
            userinfo.company=userinfo_cd['company']
            userinfo.profession=userinfo_cd['profession']
            userinfo.address=userinfo_cd['address']
            userinfo.aboutme=userinfo_cd['aboutme']
            user.save()
            userprofile.save()
            userinfo.save()
        return HttpResponseRedirect('/account/my-information/')
    else:
        user_form=UserForm(instance=request.user)
        userprofile_form=UserProfileForm(initial={'birth':userprofile.birth,'phone':userprofile.phone})
        userinfo_form=UserInfoForm(initial={'school':userinfo.school,'company':userinfo.company,
                                            'profession':userinfo.profession,
                                            'address':userinfo.address,
                                            'aboutme':userinfo.aboutme})
        return render(request,'account/myself_edit.html',{'user_form':user_form,'userprofile_form':userprofile_form,'userinfo_form':userinfo_form})


#!/usr/bin/python3
# Purpose:      Save a Youtube live-stream to an M3U playlist
# Author:       cgomesu
# Date:         April 30th, 2020
# Disclaimer:   Use at your own discretion.
#               Be mindful of the API daily quota.
#               The author does not provide any sort warranty whatsoever.

import requests

class YoutubeHandler:
    def __init__(self,
                 apiurl,
                 apikey,
                 channelid,
                 channelname,
                 channellogo):
        self.apiurl = apiurl
        self.apikey = apikey
        self.channelid = channelid
        self.channelname = channelname
        self.channellogo = channellogo

    def find_chinfo(self):
        # Returns the ID of the channel that best matches the NAME provided and its LOGO
        try:
            # Check https://developers.google.com/youtube/v3/docs
            resource = "search"
            parameters = {
                "key": self.apikey,
                "part": "snippet",
                "type": "channel",
                "maxResults": 1,
                "q": self.channelname
            }
            response = requests.get(self.apiurl + resource, params=parameters)
            # Parse JSON for key status
            if response.status_code != 200:
                if response.json()["error"]["errors"][0]["reason"] == "keyInvalid":
                    print("The Youtube API key is not valid. "
                          "Review your credentials. Key provided: {}".format(self.apikey))
                    # Exit program if the API key is invalid because it's pointless to continue
                    exit()
                print("Unable to use the Youtube API key. Reason: {}".
                      format(response.json()["error"]["errors"][0]["reason"]))
                raise Exception
            # Get channelId from json
            self.channelid = response.json()["items"][0]["snippet"]["channelId"]
            self.channellogo = response.json()["items"][0]["snippet"]["thumbnails"]["high"]["url"]
            print("The channel ID is: {}".format(self.channelid))
            print("The URL of the channel's logo is: {}".format(self.channellogo))
            return self.channelid, self.channellogo
        except Exception as err:
            print("There was an error while retrieving the channel info: {}".format(err))
            return None, None

    def find_stream(self):
        # Retrieves info from the live-stream of a specified channelId
        try:
            # Check https://developers.google.com/youtube/v3/docs
            # If multiple streams, prioritize highest view count
            resource = "search"
            parameters = {
                "key": self.apikey,
                "part": "id,snippet",
                "channelId": self.channelid,
                "type": "video",
                "eventType": "live",
                "order": "viewCount"
            }
            response = requests.get(self.apiurl + resource, params=parameters)
            # Parse JSON for key status
            if response.status_code != 200:
                if response.json()["error"]["errors"][0]["reason"] == "keyInvalid":
                    print("The Youtube API key is invalid. "
                          "Review your credentials. Key provided: {}".format(self.apikey))
                    # Exit program if the API key is invalid because it's pointless to continue
                    exit()
                print("Unable to use the Youtube API key. Reason: {}".
                      format(response.json()["error"]["errors"][0]["reason"]))
                raise Exception
            # Check if there's a live-stream available. Raise exception otherwise.
            if not response.json()["items"]:
                print("Unable to find a live-stream on channel ID {}".format(self.channelid))
                raise Exception
            print("A live-stream was found!  Extracting info from it...")
            # Parse video info from json
            # Make sure to encode(utf-8 or -16) because otherwise, we'll get some unicode error,
            # owing to the presence of special characters in title, description, etc.
            video = {
                "title": response.json()["items"][0]["snippet"]["title"].encode("utf-8"),
                "description": response.json()["items"][0]["snippet"]["description"].encode("utf-8"),
                "id": response.json()["items"][0]["id"]["videoId"],
                "url": "https://www.youtube.com/watch?v=" + response.json()["items"][0]["id"]["videoId"],
                "date": response.json()["items"][0]["snippet"]["publishedAt"].encode("utf-8"),
                "region": response.json()["regionCode"].encode("utf-8")
            }
            print("Done extracting info from the live-stream!")
            return video
        except Exception as err:
            print("There was an error while trying to retrieve the videoId from the live-stream: {}".format(err))
            return None


#!/usr/bin/env python
import time
import picamera
import requests
from time import sleep
import cv2
import base64
import os
try:
    from pathlib import Path
except (ImportError, AttributeError):
    from pathlib2 import Path
Path().expanduser()

def upload_photo(image, filename):
    api_url = "https://20mission.org/photoboo/api/photos/"
    image_bytestring = cv2.imencode('.jpg', image)[1].tostring()
    payload = {
        "name": filename,
        "data": base64.encodestring(image_bytestring).decode("utf-8")
    }
    print("Uploading {} to {}".format(filename, api_url))
    response = requests.put(api_url, json=payload)
    print("done: response: {}".format(str(response.status_code)))

def display(image_data, title="", time_s=None):
        cv2.imshow(title, image_data)
        cv2.waitKey(0)
        if time_s is not None:
            time.sleep(time_s)
        cv2.destroyAllWindows()

camera = picamera.PiCamera()
camera.resolution = (800, 600)
camera.shutter_speed = 5000
camera.exposure_compensation = 25
camera.exposure_mode = "night"
camera.awb_mode = "auto"
sleep(1)

save_folder = Path("/tmp/images")
does_folder_exist = save_folder.exists()
if does_folder_exist is False:
    save_folder.mkdir()

for i in range(0, 50):
    filename = "test_{}.jpg".format(str(int(round(time.time()))))
    print("Taking photo: {}".format(filename))
    camera.capture('/tmp/images/{}'.format(filename))
    print("Pausing...")
    time.sleep(2)

camera.close()
image = cv2.imread('/tmp/images/{}'.format(filename))
print("filename: {}".format(filename))
upload_photo(image, filename)


from itertools import combinations
class Sumzero:
	def __init__(self):
		self.lst = [float(a) for a in input('Enter real numbers: ').split()]
	def compute_Sum(self):
		ans = []
		for c in combinations(self.lst,3):
			if sum(c) == 0:
				ans.append(list(c))
		print(ans)
s = Sumzero()
s.compute_Sum()


# uncompyle6 version 3.7.4
# Python bytecode 3.7 (3394)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: /Users/philewels/GitHub/MultiQC/multiqc/modules/homer/homer.py
# Compiled at: 2019-11-13 05:22:42
# Size of source mod 2**32: 2235 bytes
""" MultiQC module to parse output from HOMER """
from __future__ import print_function
from collections import OrderedDict
import logging
from multiqc.modules.base_module import BaseMultiqcModule
from .findpeaks import FindPeaksReportMixin
from .tagdirectory import TagDirReportMixin
log = logging.getLogger(__name__)

class MultiqcModule(BaseMultiqcModule, FindPeaksReportMixin, TagDirReportMixin):
    __doc__ = ' HOMER has a number of different commands and outputs.\n    This MultiQC module supports some but not all. The code for\n    each script is split into its own file and adds a section to\n    the module output if logs are found. '

    def __init__(self):
        super(MultiqcModule, self).__init__(name='HOMER', anchor='homer', href='http://homer.ucsd.edu/homer/',
          info='is a suite of tools for Motif Discovery and next-gen sequencing analysis.')
        self.general_stats_headers = OrderedDict()
        self.general_stats_data = dict()
        n = dict()
        self.tagdir_data = {'GCcontent':{},  'restriction':{},  'restriction_norm':{},  'length':{},  'taginfo_total':{},  'taginfo_total_norm':{},  'taginfo_uniq':{},  'taginfo_uniq_norm':{},  'FreqDistribution':{},  'header':{},  'interChr':{}}
        n['findpeaks'] = self.parse_homer_findpeaks()
        if n['findpeaks'] > 0:
            log.info('Found {} findPeaks reports'.format(n['findpeaks']))
        n['Homer_tagDir'] = self.homer_tagdirectory()
        if n['Homer_tagDir'] > 0:
            log.info('Found {} reports'.format(n['Homer_tagDir']))
        if sum(n.values()) == 0:
            raise UserWarning
        self.general_stats_addcols(self.general_stats_data, self.general_stats_headers)


#!/usr/bin/env python
from pathlib import Path

from gin.errors import ManifestNotFound, ParseError
from gin.parser import Parser
from gin.project import Project

class Gin:
    _manifest: Path
    _project: Project

    def __init__(self):
        pass

    def _parse(self):
        try:
            parser = Parser(self._manifest)
            self._project = parser.get_project()
        except ParseError:
            print("Failed to parse the manifest")

    def manifest(self, manifest):
        path = Path(manifest)
        if not path.exists():
            raise ManifestNotFound(f"Manifest {manifest} not found")
        self._manifest = Path(manifest)
        self._parse()


#!/usr/bin/env python

"""
Installs the flowstats module for Kaluza data analysis
"""

# Always prefer setuptools over distutils
from setuptools import setup, find_packages
# To use a consistent encoding
from codecs import open
from os import path

here = path.abspath(path.dirname(__file__))

# Get the long description from the README file
with open(path.join(here, 'README.txt'), encoding='utf-8') as f:
    long_description = f.read()

setup(
    name='sample',
    version='0.0.1',
    description='Flowstats: a module to analyze Kaluza flow cytometry data.',
    long_description=long_description,
    url='https://github.com/pypa/sampleproject',
    author='katdev',
    author_email='katdev@users.noreply.github.com',
    license='Apache',

    # See https://pypi.python.org/pypi?%3Aaction=list_classifiers
    classifiers=[
        'Development Status :: 3 - Alpha',
        'Intended Audience :: End users/Desktop',
        'Topic :: Software Development :: Build Tools',
        'License :: OSI Approved :: Apache',
        'Programming Language :: Python :: 3.5',
    ],
    scripts=['bin/create_database.py', 'bin/flowstats_gui.py'],
    keywords=['Kaluza', 'flow cytometry'],
    packages=find_packages(exclude=['contrib', 'docs', 'tests']),
    install_requires=['sqlalchemy'],
)


from collections import deque

def print_orders(tasks, prerequisites):
    sorted_order = []

    if tasks <= 0:
        return False

    in_degree = {i: 0 for i in range(tasks)}
    graph = {i: [] for i in range(tasks)}

    for prerequisite in prerequisites:
        parent, child = prerequisite[0], prerequisite[1]
        graph[parent].append(child)
        in_degree[child] += 1

    sources = deque()
    for key in in_degree:
        if in_degree[key] == 0:
            sources.append(key)

    print_all_top_sorts(graph, in_degree, sources, sorted_order)

def print_all_top_sorts(graph, in_degree, sources, sorted_order):
    if sources:
        for vertex in sources:
            sorted_order.append(vertex)
            sources_for_next_call = deque(sources)

            sources_for_next_call.remove(vertex)

            for child in graph[vertex]:
                in_degree[child] -= 1
                if in_degree[child] == 0:
                    sources_for_next_call.append(child)

            print_all_top_sorts(graph, in_degree, sources_for_next_call, sorted_order)

            sorted_order.remove(vertex)
            for child in graph[vertex]:
                in_degree[child] += 1

    if len(sorted_order) == len(in_degree):
        print(sorted_order)

print("Task Orders: ")
print_orders(3, [[0, 1], [1, 2]])

print("Task Orders: ")
print_orders(4, [[3, 2], [3, 0], [2, 0], [2, 1]])

print("Task Orders: ")
print_orders(6, [[2, 5], [0, 5], [0, 4], [1, 4], [3, 2], [1, 3]])


from __future__ import division
from __future__ import print_function

import sys
import argparse
import cv2
import editdistance
from DataLoader import DataLoader, Batch
from Model import Model, DecoderType
from SamplePreprocessor import preprocess
import os

class FilePaths:
	"filenames and paths to data"
	fnCharList = '../model/charList.txt'
	fnAccuracy = '../model/accuracy.txt'
	fnTrain = '../data/'
	#fnTrain = '/content/' # for colab, copy the data from drive.
	fnInfer = '../data/test.png'
	fnCorpus = '../data/corpus.txt'

def train(model, loader, datablock_size):
	"train NN"
	model.trainBatch(loader, FilePaths.fnAccuracy, datablock_size)

def infer(model, batch):
	"recognize text in image provided by file path"
	if DecoderType == DecoderType.WordBeamSearch:
		recognized = model.inferBatch(batch)
		print('result is ')
		print(recognized)
		return recognized
	else:
		(recognized, probability) = model.inferBatch(batch)
		print(recognized)
		print('Probability:', probability)
		return recognized

def main():
	"main function"
	# optional command line args
	parser = argparse.ArgumentParser()
	parser.add_argument('--train', help='train the NN', action='store_true')
	parser.add_argument('--beamsearch', help='use beam search instead of best path decoding', action='store_true')
	parser.add_argument('--wordbeamsearch', help='use word beam search instead of best path decoding', action='store_true')
	parser.add_argument('--batch_size', help='batch size? default is 100', type=int, default=100)
	parser.add_argument('--block_size', help='datablock size? default is 10,000. decrease if RAM is lower than 16gigs', \
		type=int, default=10000)

	args = parser.parse_args()

	decoderType = DecoderType.BestPath
	if args.beamsearch:
		decoderType = DecoderType.BeamSearch
	elif args.wordbeamsearch:
		decoderType = DecoderType.WordBeamSearch

	batch_size = args.batch_size
	datablock_size = args.block_size

	# train on IAM dataset
	if args.train:
		# load training data, create TF model
		print('Setting up loader')
		loader = DataLoader(FilePaths.fnTrain, Model.imgSize, Model.maxTextLen)

		# save characters of model for inference mode
		print('Writing charlist')
		open(FilePaths.fnCharList, 'w').write(str().join(loader.charList))

		# save words contained in dataset into file
		print('Writing words')
		open(FilePaths.fnCorpus, 'w').write(str(' ').join(loader.words))

		# execute training
		if args.train:
			print('Setting up model')
			model = Model(loader.charList, decoderType, batch_size)
			# train(model, loader)
			train(model, loader, datablock_size)

	# infer text on test image
	else:
		file = open(FilePaths.fnInfer + 'preds.txt', 'w')
		model = Model(open(FilePaths.fnCharList).read(), decoderType, batch_size=100)
		imgs = []
		for filename in os.listdir(FilePaths.fnInfer):
			img = FilePaths.fnInfer + filename
			imgs.append(preprocess(cv2.imread(img, cv2.IMREAD_GRAYSCALE), Model.imgSize))
		size = len(imgs)
		dummy_txts = [None for i in range(size)]
		batch = Batch(dummy_txts, imgs)
		preds = infer(model, batch)
		for word in preds:
			file.write(word+' ')
		file.close()

if __name__ == '__main__':
	main()


import webapp2
import MySQLdb
import json

from contextlib import closing

from lib.hitchcfg import CONFIG
from lib.util import connectdb, errorMessage, writeOut
from lib.util import checkCntType, checkJsonStruct, checkLogin, timeBorders
from lib.geo import *
from lib.sql import SQLqueries

class GetSpotsHandler(webapp2.RequestHandler):
	def get(self):
		writeOut(self, 405, "BAD_REQ")

	def post(self):
		if not checkCntType(self):
			return

		try:
			jsonData = json.loads(self.request.body)
		except ValueError:
			writeOut(self, 400, "BAD_JSON")
			return

		if not checkValues(self, jsonData):
			return

		if hasDestination(jsonData) and not isValidWithDestination(self, jsonData):
			return

		with closing(connectdb()) as db:
			if not checkLogin(self, db, jsonData['userid'], jsonData['hash']):
				return

			with closing(db.cursor()) as cursor:
				try:
					uid = jsonData['userid']
					lat = jsonData['pos']['lat']
					lng = jsonData['pos']['lng']
					radius = jsonData['radius']

					borders = geocalc(lat, lng, radius)
					data = []
					queryType = ""

					if hasDestination(jsonData) and isValidWithDestination(self, jsonData):

						dest_lat = jsonData['dest']['lat']
						dest_lng = jsonData['dest']['lng']
						hitchtime = jsonData['hitchtime']

						dist = geodistance(jsonData['pos'], jsonData['dest'])
						dest_radius = clamp(dist/5.0, 1, 50)
						dest_borders = geocalc(dest_lat, dest_lng, dest_radius)
						tb = timeBorders(hitchtime, CONFIG['time_interval'])
						vars = borders2tuple(dest_borders) + tb + tb + (hitchtime,) + borders2tuple(borders)

						cursor.execute(SQLqueries.select_spots_relevant_time, vars)

						for row in cursor.fetchall():
							data.append({
								'spotid': row[0],
								'pos': {
									'lat': round(row[1], 6),
									'lng': round(row[2], 6)
								},
								'nrides': row[3],
								'nmatching': row[4],
								'best_time': round(float(row[5]), 2),
								'timefactor': round(float(row[6]), 3),
								'score': round(float(row[7]), 3)
							})

						queryType = "complex"
					else:
						cursor.execute(SQLqueries.select_spots_all,
							borders2tuple(borders))

						for row in cursor.fetchall():
							data.append({
								'spotid': row[0],
								'pos': {
									'lat': round(row[1], 6),
									'lng': round(row[2], 6)
								},
								'nrides': row[3],
								'nmatching': 0,
								'best_time': 0,
								'timefactor': 0,
								'score': 0
							})

						queryType = "simple"

					returnJson = errorMessage(200, "OK")
					returnJson['spots'] = data
					returnJson['query'] = queryType
					self.response.write(json.dumps(returnJson))

				except MySQLdb.Error as e:
					writeOut(self, 501, "MYSQL_ERROR")

def clamp(value, min, max):
	if value < min:
		return min
	elif value > max:
		return max
	else:
		return value

def checkValues(self, jsonData):
	expected = {
		'userid': 'int',
		'hash': 'string',
		'pos': {
			'lat': 'float',
			'lng': 'float'
		},
		'radius': 'float'
	}

	if not checkJsonStruct(expected, jsonData):
		writeOut(self, 400, "INVAL_STRUCT")
		return False

	if jsonData['userid'] < 1:
		writeOut(self, 400, "INVAL_UID")
		return False

	if not len(jsonData['hash'])==32:
		writeOut(self, 400, "INVAL_HASH")
		return False

	if jsonData['pos']['lat'] < -90 or jsonData['pos']['lat'] > 90:
		writeOut(self, 400, "INVAL_LAT:pos")
		return False

	if jsonData['pos']['lng'] < -180 or jsonData['pos']['lng'] > 180:
		writeOut(self, 400, "INVAL_LNG:pos")
		return False

	if jsonData['radius'] <= 0 or jsonData['radius'] > 25:
		writeOut(self, 400, "INVAL_RAD")
		return False

	return True

def hasDestination(jsonData):
	expected = {
		'dest': {
			'lat': 'float',
			'lng': 'float'
		},
		'hitchtime': 'int'
	}

	return checkJsonStruct(expected, jsonData)

def isValidWithDestination(self, jsonData):
	if jsonData['dest']['lat'] < -90 or jsonData['dest']['lat'] > 90:
		writeOut(self, 400, "INVAL_LAT:dest")
		return False

	if jsonData['dest']['lng'] < -180 or jsonData['dest']['lng'] > 180:
		writeOut(self, 400, "INVAL_LNG:dest")
		return False

	if jsonData['hitchtime'] < 0 or jsonData['hitchtime'] >= CONFIG['max_time']:
		writeOut(self, 400, "INVAL_TIME")
		return False

	return True


from fms.models.project import SkillTag, MilestoneProgress, ProjectCategory, ProjectType, ProjectRole, Project, WorkspaceUserLog, Message
from fms.models.core import MenuItem
from django.utils import timezone
from datetime import datetime, date
from fms.service import projects_service
from django.contrib.auth.models import User

def init(company):
	if SkillTag.objects.filter(company=company).count() == 0:
		skilltag = SkillTag.objects.create(company=company, tagcode='DEFAULT', tagdesc='Default skill tag')
		skilltag.save()

	if MilestoneProgress.objects.all().count() == 0:
		progress = MilestoneProgress.objects.create(code='NS', description='Not started')
		progress.save()
		progress = MilestoneProgress.objects.create(code='IP', description='In progress')
		progress.save()
		progress = MilestoneProgress.objects.create(code='CP', description='Completed')
		progress.save()
		progress = MilestoneProgress.objects.create(code='RJ', description='Rejected')
		progress.save()

	if MenuItem.objects.all().count() == 0:
		MenuItem.objects.all().delete()
		menu = MenuItem.objects.create(menuItem='Dashboard', menuCss='fa-dashboard', menuLink='/fms/', menuOrder=0)
		menu.save()
		menu = MenuItem.objects.create(menuItem='Workspace', menuCss='fa-object-group', menuLink='/fms/workspaces/', menuOrder=3)
		menu.save()
		menu = MenuItem.objects.create(menuItem='Project Category', menuCss='fa-wrench', menuLink='/fms/category/', menuOrder=4)
		menu.save()
		menu = MenuItem.objects.create(menuItem='Project Type', menuCss='fa-wrench', menuLink='/fms/projecttype/', menuOrder=6)
		menu.save()
		menu = MenuItem.objects.create(menuItem='Skill Tags', menuCss='fa-wrench', menuLink='/fms/skilltag/', menuOrder=8)
		menu.save()
		menu = MenuItem.objects.create(menuItem='Maven Prophet', menuCss='fa-bar-chart-o', menuLink='/fms/prophet/', menuOrder=9)
		menu.save()

	if ProjectCategory.objects.filter(company=company).count() == 0:
		category = ProjectCategory.objects.create(company=company, category_code='DEFAULT', category_desc='Default project category')
		category.save()

	if ProjectType.objects.filter(company=company).count() == 0:
		project_type = ProjectType.objects.create(company=company, typecode='DEFAULT', typedesc='Default project type', category=category)
		project_type.save()

	if ProjectRole.objects.all().count() == 0:
		ProjectRole.objects.all().delete()
		role = ProjectRole.objects.create(code='PM', desc='Project Manager')
		role.save()
		role = ProjectRole.objects.create(code='CO', desc='Collaborator')
		role.save()
		role = ProjectRole.objects.create(code='OB', desc='Observer')
		role.save()

def get_menu():
	items = []
	links = []
	css = []

	itemlist = MenuItem.objects.filter(is_visible=True).order_by('menuOrder')
	for item in itemlist:
		items.append(item.menuItem)
		links.append(item.menuLink)
		css.append(item.menuCss)

	return zip(items, links, css)

def get_meta_data(request):
	projects = projects_service.get_involved_project_list(user=request.user)
	notifications = []

	for project in projects:
		count = 0
		last_seen = None
		messages = Message.objects.filter(project=project)
		workspaceuserlog = WorkspaceUserLog.objects.filter(project=project, user=request.user)

		if workspaceuserlog is not None and len(workspaceuserlog) == 1:
			last_seen = workspaceuserlog[0].last_seen

		if messages is not None and len(messages) > 0:
			for message in messages:
				if last_seen is None:
					last_seen = timezone.make_aware(datetime.combine(date(2000, 1, 1), datetime.min.time()))

				if last_seen < message.created_date:
					count += 1

			if count > 0:
				notification = {
					'project_name' : project.title,
					'count' : count,
					'project_id' : project.id
				}

				notifications.append(notification)

	is_disabled = False
	try:
		is_disabled = MenuItem.objects.get(menuLink=request.get_full_path()).is_disabled
	except Exception as e:
		print('MenuItem for ', request.get_full_path(), ' does not exist')
		print(str(e))

	context = {
		'menus' : get_menu(),
		'current_user' : request.user.username,
		'notifications' : notifications,
		'notifications_count' : len(notifications),
		'is_disabled' : is_disabled
	}

	return context

def is_fms_user(username):
	users = User.objects.filter(username=username).count()
	return (users > 0)


# -*- coding: utf-8 -*-
"""
fish_multi.py
画像データとメタデータをconcatする
CNNとMLPの多入力モデル

@author: hanano
"""

from keras.preprocessing.image import ImageDataGenerator
from sklearn.metrics import confusion_matrix
from keras.callbacks import EarlyStopping
from keras.preprocessing.image import img_to_array, load_img
from keras.models import Model, Sequential
from keras.layers import Dense, Dropout, Flatten, Input, BatchNormalization
from keras.applications.vgg16 import VGG16
import numpy as np
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from keras import optimizers
from keras.utils import np_utils
import os
import random
import re
import itertools
import pandas as pd
from keras.layers.core import Activation
from scipy import sparse
from keras.layers.merge import concatenate

#サイズを指定
im_rows = 224
im_cols = 224
im_color = 3
in_shape = (im_rows,im_cols,im_color)
nb_classes = 9

#ImageDataGenerator と　画像を水増しする関数---------------------------------------#
datagen = ImageDataGenerator(
                            rotation_range=20,
                            width_shift_range=0.01,
                            height_shift_range=0.01,
                            brightness_range=(0.5, 1.0),
                            zoom_range=0.2,#拡大縮小
                            shear_range=5,#せん断(引き伸ばし)
                            channel_shift_range=5,
                            horizontal_flip = True,
                            vertical_flip = True
                            )

def images_gen(x_list,y_list):
    output_dir = 'test'
    x_list_add = []
    y_list_add = []
    if os.path.isdir(output_dir) == False:
        os.mkdir(output_dir)
    for x ,y in zip(x_list,y_list):#xは(3, width, height)で受け取る
        x = x.reshape((1,) + x.shape)  #(1, 3, width, height)に変換する

        i = 0
        for batch in datagen.flow(x, batch_size=32, save_to_dir=output_dir, save_prefix='img', save_format='jpg'):
            batch = batch.astype(np.uint8)#データ型を揃える
            batch = batch.reshape((224, 224, 3))
            x_list_add.append(batch)
            y_list_add.append(y)
            i += 1
            if i > 4:#１枚から5枚作る
                break
    x_np_add = np.array(x_list_add)
    y_np_add = np.array(y_list_add)

    return x_np_add,y_np_add
#------------------------------------------------------------------------------#

#写真データを読み込みの関数--------------------------------------------------------#
def list_pictures(directory, ext='jpg|jpeg|bmp|png|ppm'):
    return [os.path.join(root, f)
            for root, _, files in os.walk(directory) for f in files
            if re.match(r'([\w]+\.(?:' + ext + '))', f.lower())]
#------------------------------------------------------------------------------#

X = []
Y = []
count = 0

filelist = ["LateolabraxJaponicus_resize_224","LateolabraxLatus_resize_224","LateolabraxMaculatus_resize_224","ThunnusOrientalis_resize_224","ThunnusAlbacares_resize_224","ThunnusTonggol_resize_224","PagrusMajor_resize_224","EvynnisTumifrons_resize_224","DentexHypselosomusBleeker_resize_224"]
for dir in filelist:
    for picture in list_pictures(dir + '/'):
        img = img_to_array(load_img(picture, target_size=(im_rows,im_cols)))
        X.append(img)
        Y.append(count)
    count += 1

# arrayに変換
X = np.asarray(X)
Y = np.asarray(Y)

# 画素値を0から1の範囲に変換
#X = X.astype('float32')
#X = X / 255.0 #最大値で割ることでデータを正規化する

# クラスの形式を変換
Y = np_utils.to_categorical(Y, 9)

# 学習用データとテストデータ
X_train, X_test, y_train, y_test = train_test_split(X, Y, train_size=0.8, random_state=111)
#inputtensorをかく

#trainデータを水増しする
X_train_add,y_train_add = images_gen(X_train,y_train)

X_train_add2 = np.concatenate([X_train_add, X_train], axis=0)

y_train_add2 = np.concatenate([y_train_add, y_train], axis=0)

X_train_add2 = X_train_add2.astype("float32")/255
X_test = X_test.astype("float32")/255

# メタデータ用の処理---------------------------------------------------------------#
#One-hotベクトル→index番号
y_train_add2_meta = np.argmax(y_train_add2,axis=1)
y_test_meta = np.argmax(y_test,axis=1)

X_train_meta = np.zeros((6090, 4))
X_test_meta = np.zeros((254, 4))
fish_data4 = np.loadtxt("fish_data4.csv",       # (900,4)
                  delimiter=",",
                  skiprows=0,                   # 先頭の何行を無視するか（指定した行数までは読み込まない）
                  usecols=(0,1,2,3)             # 読み込みたい列番号
                 )
from sklearn import preprocessing
mm = preprocessing.MinMaxScaler()
fish_data4_norm = mm.fit_transform(fish_data4)

fish_data_0 = fish_data4_norm[0:100]
fish_data_1 = fish_data4_norm[100:200]
fish_data_2 = fish_data4_norm[200:300]
fish_data_3 = fish_data4_norm[300:400]
fish_data_4 = fish_data4_norm[400:500]
fish_data_5 = fish_data4_norm[500:600]
fish_data_6 = fish_data4_norm[600:700]
fish_data_7 = fish_data4_norm[700:800]
fish_data_8 = fish_data4_norm[800:900]

for i in range(6090):
    randomnum = random.randint(0, 99)
    if y_train_add2_meta[i] == 0:
        X_train_meta[i] = fish_data_0[randomnum]
    if y_train_add2_meta[i] == 1:
        X_train_meta[i] = fish_data_1[randomnum]
    if y_train_add2_meta[i] == 2:
        X_train_meta[i] = fish_data_2[randomnum]
    if y_train_add2_meta[i] == 3:
        X_train_meta[i] = fish_data_3[randomnum]
    if y_train_add2_meta[i] == 4:
        X_train_meta[i] = fish_data_4[randomnum]
    if y_train_add2_meta[i] == 5:
        X_train_meta[i] = fish_data_5[randomnum]
    if y_train_add2_meta[i] == 6:
        X_train_meta[i] = fish_data_6[randomnum]
    if y_train_add2_meta[i] == 7:
        X_train_meta[i] = fish_data_7[randomnum]
    if y_train_add2_meta[i] == 8:
        X_train_meta[i] = fish_data_8[randomnum]

for i in range(254):
    randomnum = random.randint(0, 99)
    if y_test_meta[i] == 0:
        X_test_meta[i] = fish_data_0[randomnum]
    if y_test_meta[i] == 1:
        X_test_meta[i] = fish_data_1[randomnum]
    if y_test_meta[i] == 2:
        X_test_meta[i] = fish_data_2[randomnum]
    if y_test_meta[i] == 3:
        X_test_meta[i] = fish_data_3[randomnum]
    if y_test_meta[i] == 4:
        X_test_meta[i] = fish_data_4[randomnum]
    if y_test_meta[i] == 5:
        X_test_meta[i] = fish_data_5[randomnum]
    if y_test_meta[i] == 6:
        X_test_meta[i] = fish_data_6[randomnum]
    if y_test_meta[i] == 7:
        X_test_meta[i] = fish_data_7[randomnum]
    if y_test_meta[i] == 8:
        X_test_meta[i] = fish_data_8[randomnum]
# ----------------------------------------------------------------------------#

def create_mlp(dim, regress=False):
	# define our MLP network
    model = Sequential()
    model.add(Dense(128, input_dim=dim, activation="relu"))
    model.add(Dense(128, activation="relu"))
    model.add(Dense(128, activation="relu"))

    return model

mlp = create_mlp(X_train_meta.shape[1], regress=False)
input_tensor = Input(shape=(im_rows, im_cols, im_color)) #224,224,3

# functionalな書き方をする
base_model = VGG16(include_top=False, weights="imagenet", input_tensor=input_tensor)
x = base_model.output
x = Flatten(input_shape = base_model.output_shape[1:])(x)
x = Dropout(0.5)(x)
x = Dense(32, activation="relu", kernel_initializer="he_normal")(x)
x = Dropout(0.5)(x)
x = Dense(32, activation="relu")(x)
x = Dropout(0.5)(x)

#MLPとCNNの出力をconcatenateする
combinedInput = concatenate([mlp.output, x])

x = Dense(64, activation="relu")(combinedInput)

x = Dense(64, activation="relu")(x)
x = Dense(64, activation="relu")(x)
x = Dense(9, activation="softmax")(x)

model = Model(inputs=[mlp.input, base_model.input], outputs=x)#新しく学習するとき
for layer in model.layers[:15]:
    layer.trainable = False
model.summary()

optimizer = optimizers.SGD(lr=0.001, momentum=0.95)

#多クラス分類なら'categorical_crossentropy'
model.compile(loss="categorical_crossentropy",#多クラス分類なので
              optimizer=optimizer,
              metrics=["accuracy"])

early_stopping = EarlyStopping(monitor="acc",min_delta=0.01, patience=15, verbose=1)

hist=model.fit([X_train_meta,X_train_add2],y_train_add2,
              batch_size=32,
              epochs=200,#nb_epochからepochsへ　https://qiita.com/cvusk/items/aa6270301ff2d14fb989
              verbose=1,
              callbacks=[early_stopping],
              validation_split=0.1)

#モデルを評価
score=model.evaluate([X_test_meta,X_test],y_test,verbose=1)
print("正解率=",score[1],"loss=",score[0])

num = 18 #何回目の施行か

fig, (axL, axR) = plt.subplots(ncols=2, figsize=(10,4))

# loss
def plot_history_loss(fit):
    # Plot the loss in the history
    axL.plot(fit.history['loss'],label="loss for training")
    axL.plot(fit.history['val_loss'],label="loss for validation")
    axL.set_title('model loss')
    axL.set_xlabel('epoch')
    axL.set_ylabel('loss')
    axL.legend(loc='upper right')
#    axL.grid()
    axL.text(20, 2, "test loss = %ls"%score[0], size = 10)

# acc
def plot_history_acc(fit):
    # Plot the loss in the history
    axR.plot(fit.history['acc'],label="acc for training")
    axR.plot(fit.history['val_acc'],label="acc for validation")
    axR.set_title('model accuracy')
    axR.set_xlabel('epoch')
    axR.set_ylabel('accuracy')
    axR.legend(loc='lower right')
#    axR.grid()
    axR.text(20, 0.3, "test acc = %ls"%score[1], size = 10)

plot_history_loss(hist)
plot_history_acc(hist)
fig.savefig('result//lossacc_meta%i.png'%num)
plt.close()

#混同行列を表示
def plot_confusion_matrix(cm, classes,
                          normalize=False,
                          title='Confusion matrix',
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

    print(cm)

    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    fmt = '.2f' if normalize else 'd'
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt),
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")
    #plt.figure(figsize=(10,10))
    plt.ylabel('True label')
    plt.xlabel('Predicted label')
    plt.tight_layout()

preds = model.predict([X_test_meta,X_test]).argmax(axis=1)
cm = confusion_matrix(y_test.argmax(axis=1),preds)
plt.figure(figsize=(6,5))
plot_confusion_matrix(cm, classes=['0','1','2','3','4','5','6','7','8'], title='Confusion matrix, without normalization')
plt.savefig('result//cm_meta%i.png'%num)
plt.figure(figsize=(6,5))
plot_confusion_matrix(cm, classes=['0','1','2','3','4','5','6','7','8'], normalize=True,title='Normalized confusion matrix')
plt.savefig('result//cm_norm_meta%i.png'%num)

#データフレーム型に変換
labels = np.array(['LateolabraxJaponicus','LateolabraxLatus','LateolabraxMaculatus','ThunnusOrientalis','ThunnusAlbacares','ThunnusTonggol','PagrusMajor','EvynnisTumifrons','DentexHypselosomusBleeker'])
cm_labeled = pd.DataFrame(cm, columns=labels, index=labels)
cm_labeled.to_csv("result//cm_df_meta%i.csv"%num)

#学習モデルの保存
json_string = model.to_json()
#モデルのファイル名　拡張子.json
open('result//fish_cnn3_meta%i.json'%num, 'w').write(json_string)
#重みファイルの保存 拡張子がhdf5
model.save_weights('result//fish_cnn3_meta%i.hdf5'%num)


import xml.etree.ElementTree as et
import time
from debug import printMsg

def getGraphFromFile(file, showMessage = True):
    printMsg("=================== START FILE CONVERSION ===================", showMessage)
    startTime = time.time()
    graph = createGraph(file)
    printMsg(("Took : " + str(time.time() - startTime) + " seconds to make a graph from the file."), showMessage)
    printMsg("=============================================================", showMessage)
    return graph

def createGraph(file):
    graph = {}
    nodes = {}
    links = []
    tree = None
    root = None

    with open(file, encoding='utf-8') as data:
        tree = et.parse(data)
        root = tree.getroot()

    for rawNode in root.findall("node"):
        nodes[int(rawNode.get("id"))] = (float(rawNode.get("lat")), float(rawNode.get("lon")))
        graph[nodes[int(rawNode.get("id"))]] = []

    for rawLink in root.findall("way"):
        neighbors = []
        i = 0
        for neighbor in rawLink.iter("nd"):
            neighbors.append(nodes[int(neighbor.attrib["ref"])])
        links.append(neighbors)

    for link in links:
        length = len(link)

        for i in range(length):
            l = graph.get(link[i], [])

            if length > 1 and i == 0:
                if link[1] not in l:
                    graph[link[0]].append(link[1])

            elif i == (length - 1):
                if link[length - 2] not in l:
                    graph[link[length - 1]].append(link[length - 2])

            else:
                if link[i - 1] not in l:
                    graph[link[i]].append(link[i - 1])
                if link[i + 1] not in l:
                    graph[link[i]].append(link[i + 1])

    return graph


"""
Module for receiving LOGs via API.
"""
from datetime import datetime
from os import path
from typing import Dict, List, Optional, Union

import requests

from log_processing_demo.log_item import LogItem
from log_processing_demo.sort import sort

LOG_LIST_NAME = "logs"
ERROR_NAME = "error"

class RequestError(Exception):
    """Exception for wrapping all possible exceptions of 'requests' module."""

    pass

class LogReceiver:
    """Class for receiving LOGs via API.
    See methods docstrings.
    """

    def __init__(self, base_url: str):
        """Constructor remembers a base API URL.

        Parameters
        ----------
        base_url : str
            Base API URL.
        """
        self.base_url: str = base_url

    def __call__(
        self, date_string: str, sort_by_time: Optional[bool] = False
    ) -> List[Dict[str, Union[datetime, str]]]:
        """Call an instance of class to retreive a list of LOG messages for desired date.

        Parameters
        ----------
        date_string : str
            Date to get LOG messages for. Format: YYMMDD
        sort_by_time : Optional[bool]
            Whether or not the result should be sorted by record creation time.

        Returns
        -------
        List[Dict[str, Union[datetime, str]]]
            Either sorted or unsorted list of LOG records.

        Raises
        ------
        RequestError
            Re-rised for any 'requests' module exception.
        ValueError
            Rised when API returns an error message.

        """
        try:
            response = requests.get(path.join(self.base_url, date_string))
            response.raise_for_status()
        except Exception as error:
            raise RequestError(str(error))

        data = response.json()
        if data[ERROR_NAME]:
            raise ValueError(data[ERROR_NAME])

        log_list = list(map(LogItem.parse_obj, data[LOG_LIST_NAME]))

        if sort_by_time:
            sort(log_list, key=lambda x: x.created_at)

        return log_list