import time

from django.core.checks import Tags
from django.http import JsonResponse
from django.shortcuts import render

import markdown
# Create your views here.
from django.template.context_processors import request
from blog.models import Blog, Type, Tag, BlogTags, User, Comment, Clipboard
import json

import datetime

class DateEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, datetime.datetime):
            return obj.strftime("%Y-%m-%d %H:%M:%S")
        else:
            return json.JSONEncoder.default(self, obj)

# 获取typeName
def get_type_name_by_id(param):
    type = Type.objects.filter(id=param).values()
    if type:
        return list(type)[0]

# 获取tagName
def get_tag_name_by_id(param):
    tags = list(BlogTags.objects.filter(blogs=param).values())
    res_list = []
    if tags:
        for item in tags:
            # print(item)
            tmp_tag=list(Tag.objects.filter(id=item["tags_id"]).values())[0]
            res_list.append({"id":tmp_tag["id"],"name":tmp_tag["name"]})
    # print(res_list)
    # print(res)
    return res_list

# 获取userName
def get_user_name_by_id(param):
    user = User.objects.filter(id=param).values()
    return list(user)[0]["nickname"]

# api:GET获取blog_top列表
def get_blog(request):
    print(type(request))
    if request.method == "GET":
        res = {}
        new_list_blog = []
        page = request.GET.get('page', '')
        mode = request.GET.get('mode', '')
        id = request.GET.get('id', '')
        if not id:
            blog = Blog.objects.filter(published=True).order_by('-update_time').values()
            for item in list(blog):
                item.pop("content")
                item["type_id"] = get_type_name_by_id(item["type_id"])
                item["tags"] = get_tag_name_by_id(item["id"])
                item["user_id"] = get_user_name_by_id(item["user_id"])
                new_list_blog.append(item)
            res["total"] = len(list(blog))
            if not page:
                res["page"] = "all"
                res["list"] = new_list_blog
            elif page == "admin":
                new_list_blog=[]
                blog = Blog.objects.all().order_by('-update_time').values()
                for item in list(blog):
                    item.pop("content")
                    item["type_id"] = get_type_name_by_id(item["type_id"])
                    item["tags"] = get_tag_name_by_id(item["id"])
                    item["user_id"] = get_user_name_by_id(item["user_id"])
                    new_list_blog.append(item)
                res["page"] = "admin"
                res["list"] = new_list_blog
            else:
                res["page"] = page
                res["list"] = new_list_blog[int(page) * 10:(int(page) + 1) * 10]
            res_json = json.dumps(res, cls=DateEncoder)
            return JsonResponse(json.loads(res_json))
        else:
            blog = Blog.objects.filter(id=id).values()[0]
            if mode == "markdown":
                pass
            else:
                blog["content"] = markdown.markdown(blog["content"], extensions=['markdown.extensions.fenced_code'])
            blog["user_id"] = get_user_name_by_id(blog["user_id"])
            blog["type_id"] = get_type_name_by_id(blog["type_id"])
            blog["tags"] = get_tag_name_by_id(blog["id"])
            res_json = json.dumps(blog, cls=DateEncoder)
            return JsonResponse(json.loads(res_json))

# api:GET获取type_top列表
def get_type(request):
    if request.method == "GET":
        top_num = request.GET.get('top', '')
        res = {}
        types = Type.objects.all()
        total = len(list(types))
        res["type_total"] = total
        type_item = []
        sort_list = {}
        if top_num:
            for item in types:
                tmp_item = {}
                tmp_item["type_id"] = item.id
                tmp_item["type_name"] = item.name
                tmp_list = list(Blog.objects.filter(published=True, type=item.id).order_by('-update_time').values())
                tmp_item["blog_total"] = len(tmp_list)
                sort_list[item.id] = len(tmp_list)
                new_tmp_list = []
                for item2 in tmp_list:
                    item2.pop("content")
                    new_tmp_list.append(item2)
                tmp_item["blog_list"] = new_tmp_list
                type_item.append(tmp_item)
            # print(sort_list)
            new_sort = sorted(sort_list.items(), key=lambda x: x[1], reverse=True)
            # print(new_sort)
            new_type_item = []
            for item in new_sort:
                for item2 in type_item:
                    if item2["type_id"] == item[0]:
                        new_type_item.append(item2)
            res["type_item"] = new_type_item[:int(top_num)]
        else:
            for item in types:
                tmp_item = {}
                tmp_item["type_id"] = item.id
                tmp_item["type_name"] = item.name
                tmp_list = list(Blog.objects.filter(published=True, type=item.id).order_by('-update_time').values())
                tmp_item["blog_total"] = len(tmp_list)
                sort_list[item.id] = len(tmp_list)
                new_tmp_list = []
                if tmp_list:
                    for item2 in tmp_list:
                        item2.pop("content")
                        item2["user_id"] = get_user_name_by_id(item2["user_id"])
                        item2["type_id"] = get_type_name_by_id(item2["type_id"])
                        item2["tags"] = get_tag_name_by_id(item2["id"])
                        new_tmp_list.append(item2)
                    tmp_item["blog_list"] = new_tmp_list
                    type_item.append(tmp_item)
                    res["type_item"] = type_item
        res_json = json.dumps(res, cls=DateEncoder)
        return JsonResponse(json.loads(res_json))

# api:GET获取tag_top列表
def get_tag(request):
    start = time.time()
    if request.method == "GET":
        top_num = request.GET.get('top', '')
        res = {}
        tags = Tag.objects.all()
        total = len(list(tags))
        res["tag_total"] = total
        tag_item = []
        sort_list = {}
        if top_num:
            for item in tags:
                tmp_item = {}
                tmp_item["tag_id"] = item.id
                tmp_item["tag_name"] = item.name
                tmp_list = list(BlogTags.objects.filter(tags=item.id).values())
                tmp_item["blog_total"] = len(tmp_list)
                sort_list[item.id] = len(tmp_list)
                new_tmp_list = []
                for item2 in tmp_list:
                    if list(Blog.objects.filter(published=True, id=item2["blogs_id"]).order_by(
                            '-update_time').values()):
                        tmp=list(Blog.objects.filter(published=True, id=item2["blogs_id"]).order_by(
                            '-update_time').values())[0]
                        tmp["content"] = ""
                        new_tmp_list.append(tmp)
                tmp_item["blog_list"] = new_tmp_list
                tag_item.append(tmp_item)
            new_sort = sorted(sort_list.items(), key=lambda x: x[1], reverse=True)
            # print(new_sort)
            new_tag_item = []
            for item in new_sort:
                for item2 in tag_item:
                    if item2["tag_id"] == item[0]:
                        new_tag_item.append(item2)
            res["tag_item"] = new_tag_item[:int(top_num)]
        else:
            for item in tags:
                tmp_item = {}
                tmp_item["tag_id"] = item.id
                tmp_item["tag_name"] = item.name
                tmp_list = list(BlogTags.objects.filter(tags=item.id).order_by('-blogs').values())
                tmp_item["blog_total"] = len(tmp_list)
                sort_list[item.id] = len(tmp_list)
                new_tmp_list = []
                tmp_blog_list=[]
                for item in tmp_list:
                    try:
                        tmp_blog_list=list(Blog.objects.filter(published=True, id=item["blogs_id"]).values())
                        if tmp_blog_list:
                            tmp =tmp_blog_list[0]
                            tmp["content"] = ""
                            tmp["user_id"] = get_user_name_by_id(tmp["user_id"])
                            tmp["type_id"] = get_type_name_by_id(tmp["type_id"])
                            tmp["tags"] = get_tag_name_by_id(tmp["id"])
                            new_tmp_list.append(tmp)
                    except Exception as error:
                        new_tmp_list.append(str(error))
                tmp_item["blog_list"] = new_tmp_list
                tag_item.append(tmp_item)
                res["tag_item"] = tag_item
        end = time.time()
        res["elapsed_time"]=end - start
        res_json = json.dumps(res, cls=DateEncoder)
        # print(res["tag_item"][3]["blog_list"][0]["update_time"])
        # print(res["tag_item"][3]["blog_list"][1]["update_time"])
        return JsonResponse(json.loads(res_json))

# api:GET获取recommend_top列表
def get_recommend(request):
    if request.method == "GET":
        res = {}
        blog = Blog.objects.filter(recommend=True).order_by('-update_time').values()
        res["total"] = len(list(blog))
        top_recommend = request.GET.get('top', '')
        if not top_recommend:
            res["top"] = "all"
            res["list"] = list(blog)
        else:
            res["top"] = top_recommend
            res["list"] = list(blog[:int(top_recommend)])
        res_json = json.dumps(res, cls=DateEncoder)
        # print(type(res_json))
        return JsonResponse(json.loads(res_json))

# api:GET获取commend_top列表
def get_comment(request):
    if request.method == "GET":
        res = {}
        comment = Comment.objects.filter().order_by('-create_time').values()
        res["total"] = len(list(comment))
        new_comment_list = []
        for item in list(comment):
            print(item)
            tmp_blog = Blog.objects.filter(id=item["blog_id"]).values()[0]
            tmp_blog["content"] = ""
            item["blog_id"] = tmp_blog
            item["content"] = markdown.markdown(item["content"])
            new_comment_list.append(item)
        top_comment = request.GET.get('top', '')
        if not top_comment:
            res["top"] = "all"
            res["list"] = list(new_comment_list)
        else:
            res["top"] = top_comment
            res["list"] = list(new_comment_list[:int(top_comment)])
        res_json = json.dumps(res, cls=DateEncoder)
        # print(type(res_json))
        return JsonResponse(json.loads(res_json))

# api:GET获取clipboard_top列表
def get_clipboard(request):
    if request.method == "GET":
        res = {}
        clipboard = Clipboard.objects.filter().order_by('-create_time').values()
        res["total"] = len(list(clipboard))
        new_comment_list = list(clipboard)
        # new_comment_list=[]
        # for item in list(comment):
        #     print(item)
        #     tmp_blog=Blog.objects.filter(id=item["blog_id"]).values()[0]
        #     tmp_blog["content"]=""
        #     item["blog_id"]=tmp_blog
        #     item["content"]=markdown.markdown(item["content"])
        #     new_comment_list.append(item)
        top_comment = request.GET.get('top', '')
        if not top_comment:
            res["top"] = "all"
            res["list"] = list(new_comment_list)
        else:
            res["top"] = top_comment
            res["list"] = list(new_comment_list[:int(top_comment)])
        res_json = json.dumps(res, cls=DateEncoder)
        # print(type(res_json))
        return JsonResponse(json.loads(res_json))

# api:GET获取achieve列表
def get_archive(request):
    if request.method == "GET":
        res = {}
        data = []
        year = []
        for yy in Blog.objects.extra(select={"year": "DATE_FORMAT(create_time, '%%Y')"},
                                     # where={"t_blog b group by date_format(create_time,'%Y') order by year"},
                                     order_by={"create_time"}).values():
            year.append(yy["year"])
        year = list(set(year))
        for yy in year:
            tmp_list = []
            for blog in Blog.objects.extra(where={"DATE_FORMAT(create_time, '%%Y')=%s"},
                                           params={int(yy)}).values():
                print("blog", blog)
                tmp_list.append(blog)
            data.append({"year": yy, "list": tmp_list})
        res["data"] = data
        res["blog_total"] = len(Blog.objects.all())
        res_json = json.dumps(res, cls=DateEncoder)
        return JsonResponse(json.loads(res_json))

# 导入,可以使此次请求忽略csrf校验
from django.views.decorators.csrf import csrf_exempt

# 跨域 在处理函数加此装饰器即可 @csrf_exempt
# api:POST提交admin登录验证
@csrf_exempt
def admin_login(request):
    if request.method == "POST":
        res = {}
        res["data"] = "succeed"
        res["recode"] = "200"
        res_json = json.dumps(res)
        return JsonResponse(json.loads(res_json))
    if request.method == "GET":
        res = {}
        res["data"] = "error"
        res["recode"] = "403"
        res_json = json.dumps(res)
        return JsonResponse(json.loads(res_json))

# api：POST提交添加一条新的blog记录
@csrf_exempt
def add_blog(request):
    if request.method == "POST":
        blog_data = request.body.decode()
        # print(type(request.body),request.body.decode())
        blog_json_data = json.loads(blog_data)["data"]
        blog_id=blog_json_data["id"]

        res = {}
        if not blog_id:
            # 创建新的blog数据
            newBlog=Blog.objects.create(
                appreciation=blog_json_data["appreciation"],
                commentabled=blog_json_data["commentabled"],
                content=blog_json_data["content"],
                create_time=datetime.datetime.now(),
                description=blog_json_data["description"],
                first_picture=blog_json_data["first_picture"],
                flag=blog_json_data["flag"],
                published=blog_json_data["published"],
                recommend=blog_json_data["recommend"],
                share_statement=blog_json_data["share_statement"],
                title=blog_json_data["title"],
                update_time=datetime.datetime.now(),
                type_id=blog_json_data["type_id"]["id"],
                user_id=1
            )
            # 删除原有tag
            tmp_blog = Blog.objects.get(id=newBlog.id)
            tags_item=BlogTags.objects.filter(blogs_id=newBlog.id)
            for item in tags_item:
                item.delete()
            # 添加tags
            tags = blog_json_data["tags"]
            for item in tags:
                tag_item=Tag.objects.get(id=item["id"])
                BlogTags(blogs=tmp_blog,tags=tag_item).save()
                print("tag_id", item["id"])

            res["data"] = {
                "msg":"succeed to add new blog.",
                "blogId":str(newBlog.id)
            }
        elif blog_id:
            tmp_blog=Blog.objects.get(id=blog_id)
            tmp_blog.appreciation = blog_json_data["appreciation"]
            tmp_blog.commentabled = blog_json_data["commentabled"]
            tmp_blog.content = blog_json_data["content"]
            tmp_blog.description = blog_json_data["description"]
            tmp_blog.first_picture = blog_json_data["first_picture"]
            tmp_blog.flag = blog_json_data["flag"]
            print("before",tmp_blog.published)
            tmp_blog.published = blog_json_data["published"]
            print("after", tmp_blog.published)
            tmp_blog.recommend = blog_json_data["recommend"]
            tmp_blog.share_statement = blog_json_data["share_statement"]
            tmp_blog.title = blog_json_data["title"]
            tmp_blog.update_time = datetime.datetime.now()
            tmp_blog.type_id = blog_json_data["type_id"]["id"]
            tmp_blog.user_id = 1
            tmp_blog.save()
            # 删除原有tag
            tmp_blog = Blog.objects.get(id=tmp_blog.id)
            tags_item = BlogTags.objects.filter(blogs_id=tmp_blog.id)
            for item in tags_item:
                item.delete()
            # 添加tags
            tags = blog_json_data["tags"]
            tmp_blog = Blog.objects.get(id=blog_id)
            for item in tags:
                tag_item=Tag.objects.get(id=item["id"])

                BlogTags(blogs=tmp_blog,tags=tag_item).save()
                print("tag_id", item["id"])

            res["data"] = {
                "msg": "succeed to update the blog.",
                "blogId": str(blog_id)
            }
        res["status"] = "200"
        res_json = json.dumps(res)
        return JsonResponse(json.loads(res_json))
    if request.method == "GET":
        res = {}
        res["data"] = "error"
        res["status"] = "403"
        res_json = json.dumps(res)
        return JsonResponse(json.loads(res_json))

def test(request):
    if request.method == "POST":
        res = {}
        res["data"] = "succeed"
        res["recode"] = "200"
        res_json = json.dumps(res)
        return JsonResponse(json.loads(res_json))
    if request.method == "GET":
        tags_item = BlogTags.objects.filter(blogs_id=66)
        for item in tags_item:
            print(item.tags)
        print(list(tags_item))
        res = {}
        res["data"] = "error"
        res["recode"] = "403"
        res_json = json.dumps(res)
        return JsonResponse(json.loads(res_json))


import numpy as np
import requests
from bs4 import BeautifulSoup

def get_data(countrylabel):
    """take html source code from link and read information from it as string. Read dates and total cases
	return dictionary. Read corona data from http://www.worldometers.info/coronavirus/country/countrylabel.
	:param: countrylabel str"""
    country = dict({'countrylabel':countrylabel})       # return dictionary with data to this country

    link = "http://www.worldometers.info/coronavirus/country/%s/" % countrylabel  # read this website
    r = requests.get(link)                              # response from website
    soup = BeautifulSoup(r.content, 'html.parser')      # data structure representing a parsed HTML
    charts = soup.find_all("script",
                           attrs={'type': 'text/javascript',
                                  'src':'', 'class':'', 'data-cfasync':''})  # find all <script type='text/javascript' in html and store them to a list
    for i in range(len(charts)):
        if 'coronavirus-cases-linear' in str(charts[i]):                     # find the chart with data wanted
            rawdata = str(charts[i]).replace(" ", "")                        # html to string and omit whitspaces
            time = rawdata.split('xAxis:{\ncategories:[')[1].split("]}")[0]  # split string to time data
            time = np.array(time.replace("\"", "").split(','))               # make array of strings
            total = rawdata.split('TotalCoronavirusCases')[1].split('data:[')[1].split("]}")[0]  # total cases to time data
            total = np.array(total.split(','), dtype=int)
            country['time'] = time
            country['total'] = total

        elif 'graph-active-cases-total' in str(charts[i]):
            rawdata = str(charts[i]).replace(" ", "")                        # html to string and omit whitspaces
            active = rawdata.split('TotalCoronavirusCurrentlyInfected')[1].split('data:[')[1].split("]}")[0] # currently infected
            active = np.array(active.split(','), dtype=int)
            country['active'] = active
    return country

def get_data_berlin():
    """ Load from https://datawrapper.dwcdn.net/fIw01/56/ to get Covid19 data of Berlin. Source is Senatsverwaltung
    fuer Gesundheit und Pflege.
    :return dictionary
    time (str-array), total (int-array), inhosp (int-array), sever (int-array), death (int-array)"""
    berlin = dict({'countrylabel':'Berlin'})       # return dictionary with data to Berlin

    link = 'https://datawrapper.dwcdn.net/fIw01/56/'

    r = requests.get(link)                              # response from website
    soup = BeautifulSoup(r.content, 'html.parser')      # data structure representing a parsed HTML
    sourcecode = str(soup.contents)                        # get source code as string
    rawdata = sourcecode.split('gestorben\\\\n')[1].split('\\",\\"')[0]   # crop string to data
    string = rawdata.replace('\\\\t', ',').replace('\\\\n', ',')          # change utf8 delimiter to comma
    arraystr = np.array(string.split(','))                                # make string
    arraystr = np.reshape(arraystr, (len(arraystr)//5, 5))                # change to table (2darray) with 5 columns

    # extract columns in dictionary time=dates (str) rest is integer
    berlin['time'] = np.array([translate_time_format(date) for date in arraystr[:, 0]])  # dd.02.yyy -> Febdd etc
    berlin['total'] = np.array(arraystr[:, 1], dtype=int)
    berlin['inhosp'] = np.array(arraystr[:, 2], dtype=int)
    berlin['severe'] = np.array(arraystr[:, 3], dtype=int)
    berlin['death'] = np.array(arraystr[:, 4], dtype=int)
    berlin['active'] = np.zeros(len(berlin['time']))

    return berlin

def translate_time_format(ddmmyyy):
    """
    :param: ddmmyyyy (str) as 'dd.mm.yyyy'  with mm = 01,02,...,12
    :return date (str) as 'MMdd'        with MM=Jan,Feb,...,Dec
    """
    months = dict({'01': 'Jan', '02': 'Feb', '03': 'Mar', '04': 'Apr', '05': 'May', '06': 'Jun',
                   '07': 'Jul', '08': 'Aug', '09': 'Sep', '10': 'Oct', '11': 'Nov', '12': 'Dec'})
    dmy = ddmmyyy.split('.')
    return months[dmy[1]] + dmy[0]

if __name__ == '__main__':
    print(get_data('germany'))


from django.conf.urls import patterns, include, url

from piston.resource import Resource
from piston.authentication import HttpBasicAuthentication

from handler.todoapi import TodosHandler

# Uncomment the next two lines to enable the admin:
from django.contrib import admin

admin.autodiscover()

urlpatterns = patterns('',
    # Examples:
    url(r'^$', 'todolist.handler.views.index', name='index'),
    url(r'^register$', 'todolist.handler.views.register', name='register'),
    # Uncomment the admin/doc line below to enable admin documentation:
    # url(r'^admin/doc/', include('django.contrib.admindocs.urls')),

    # Uncomment the next line to enable the admin:
    url(r'^admin/', include(admin.site.urls)),
    (r'^accounts/login/$', 'django.contrib.auth.views.login', {'template_name': 'login.html'}),
    (r'^accounts/logout/$', 'django.contrib.auth.views.logout', {'next_page': '/'}),
)

#these are the calls which need authentication of the user
auth = HttpBasicAuthentication(realm="API")
todoresource = Resource(TodosHandler, authentication=auth)
urlpatterns += patterns('',
   url(r'^todo/list$', todoresource),
   url(r'^todo/task/(?P<todoitem_id>[^/]+)?$', todoresource),
)


"""Day 7: The Treachery of Whales"""

from typing import List, Dict
from util import file_input, text_input, ints

example_input = "16,1,2,0,4,2,7,1,2,14"

def min_consumption(positions: List[int], fixed_burn_rate: bool = True) -> int:
    num_positions = max(positions) + 1
    fuel_required = [0] * num_positions
    for from_pos in positions:
        for to_pos in range(num_positions):
            fuel_spent = abs(to_pos - from_pos)
            if not fixed_burn_rate:
                fuel_spent = int((fuel_spent * fuel_spent + fuel_spent) / 2)
            fuel_required[to_pos] += fuel_spent
    return min(fuel_required)

def day7_1(nums: List[int]) -> int:
    return min_consumption(nums)

assert day7_1(text_input(example_input, ints)[0]) == 37
assert day7_1(file_input(7, ints)[0]) == 345035

def day7_2(nums: List[int]) -> int:
    return min_consumption(nums, fixed_burn_rate=False)

assert day7_2(text_input(example_input, ints)[0]) == 168
assert day7_2(file_input(7, ints)[0]) == 97038163


##this code will help you extract tweets per day(i have done this way)
##the hashtags or keywords related to coronavirus have been taken from twitter site(a full list of all the keywords were available in the site)
import pandas as pd
import tweepy

#the query length is limited(nearly 500 characters)
covid19_keywords = '#CoronaAlert OR #coronavirusdelhi OR #coronavirusinindia OR #covid_19ind OR #covid19india OR #coronavirusindia OR #QuarantineLife OR #CONVID19 OR #lockdown OR #pandemic OR #quarantine OR #QuarantineAndChill OR #remoteworking OR #socialdistancing OR #stayhome OR #stayhomesavelives OR #workfromhome OR #StaySafeStayHome OR #WorkingFromHome OR #IndiaFightsCorona -filter:retweets'

#Twitter credentials for the app
consumer_key = XXXXXXXXXXX
consumer_secret = XXXXXXXXXXX
access_key= XXXXXXXXXXX
access_secret = XXXXXXXXXXX

#geocode='28.60476,77.38276,50km' for delhi region

#pass twitter credentials to tweepy
auth = tweepy.AppAuthHandler(consumer_key, consumer_secret)
#auth.set_access_token(access_key, access_secret)
api = tweepy.API(auth,wait_on_rate_limit=True,wait_on_rate_limit_notify=True)

covid19_tweets= "covid19_tweets_22.csv"

#columns of the csv file
COLS = ['created_at','original_text','user_location']

##pulling tweets of one day
#set two date variables for date range
start_date = '2020-05-22'
end_date = '2020-05-23'

#method write_tweets()
def write_tweets(keyword,file):

    df = pd.DataFrame(columns=COLS)
    #page attribute in tweepy.cursor and iteration(until=before the given date)
    for page in tweepy.Cursor(api.search,q=keyword ,geocode='28.60476,77.38276,50km',lang='en',since=start_date,until=end_date,result_type='mixed',count=100,tweet_mode='extended').pages(2000):
        for status in page:
            new_entry = []
            status = status._json
            #new entry append
            new_entry += [status['created_at'],status['full_text'],status['user']['location']]
            single_tweet_df = pd.DataFrame([new_entry], columns=COLS)
            df = df.append(single_tweet_df, ignore_index=True)

    df.to_csv(file,columns = COLS, index=False, encoding="utf-8")

#call main method passing keywords and file path
write_tweets(covid19_keywords,covid19_tweets)


from abc import ABC, abstractmethod
import os

from FileHandler import CSVHandler, parse_types
import pandas as pd
import numpy as np
from sklearn.metrics import roc_auc_score
from collections import defaultdict

class QueryHandlerInterface(ABC):
    """Интерфейс обработчика запросов"""

    @abstractmethod
    def get_sources(self, d_type, folder):
        """Функция должна возвращать список файлов, которые можно прочитать."""
        pass

    @abstractmethod
    def get_metrics(self, d_type, source, target=''):
        """Функция должна возвращать список метрик, котороые возможно вернуть."""
        pass

    @abstractmethod
    def get_data(self, request):
        """Функция должна возвращать данные в необходиомом формате."""
        pass

def dataframe_to_response(target, df):
    """Возвращает список временных рядов. Временной ряд:
    {
    "target":"pps in",
    "datapoints":[
      [622,1450754160000],
      [365,1450754220000]
    ]
  }"""
    response = []
    if df.empty:
        return response

    # if freq is not None:
    #    orig_tz = df.index.tz
    #    df = df.tz_convert('UTC').resample(rule=freq, label='right', closed='right', how='mean').tz_convert(orig_tz)

    if isinstance(df, pd.Series):
        response.append(_series_to_response(df))
    elif isinstance(df, pd.DataFrame):
        for col in df.columns:
            response.append(_series_to_response(df[col]))
    else:
        abort(404, Exception('Received object is not a dataframe or series.'))
    return response

def _series_to_response(df):
    sorted_df = df.dropna().sort_index()
    timestamps = sorted_df.index
    values = sorted_df.values.tolist()
    return {'target': '%s' % df.name,
            'datapoints': list(zip(values, timestamps))}

def dataframe_to_json_table(df):
    """Возвращает датафрейм в виде таблицы. Таблица:
    {
    "columns":[
      {"text":"Time","type":"time"},
      {"text":"Country","type":"string"},
      {"text":"Number","type":"number"}
    ],
    "rows":[
      [1234567,"SE",123],
      [1234567,"DE",231],
      [1234567,"US",321]
    ],
    "type":"table"
  }"""
    response = []
    if df.empty:
        return response
    if isinstance(df, pd.DataFrame):
        response.append({'type': 'table',
                         'columns': df.columns.map(lambda col: {"text": col}).tolist(),
                         'rows': df.where(pd.notnull(df), None).values.tolist()})
    else:
        abort(404, Exception('Received object is not a dataframe.'))
    return response

def parse_optional(target):
    res = defaultdict(lambda: None)
    if target.get('data', None) is not None:
        res['index_col'] = target['data'].get('index_col', None)
        res['log_scale'] = target['data'].get('log_scale', None)
    return res

class QueryHandler(QueryHandlerInterface):
    """Обработчик запросов для курсовой :3"""
    special_metrics = ['agreement_count', 'default_rate', #'client_count',
                            'avg_PD', 'gini']

    def __init__(self, path, filetypes: list):
        """filetypes - список обрабатываемых типов файлов"""
        self.path = path
        self.filetypes = filetypes
        self.type_class_dict = parse_types(filetypes)  # словарь {тип: класс_обработчика}
        self.file_class_dict = {}  # словарь {файл: класс_обработчика}

    def get_sources(self, d_type: str, folder: str) -> list:
        """Возвращает список ресурсов, а заодно заполняет словарь file_class_dict"""
        results = []
        for filename in os.listdir(os.path.join(self.path, folder)):
            for filetype in self.filetypes:
                if filetype in filename:
                    results.append(filename)
                    self.file_class_dict[filename] = self.type_class_dict[filetype](filename, os.path.join(self.path, folder))
                    break
        return results

    def get_metrics(self, d_type: str, source: str, target: str = '') -> list:
        columns = self.file_class_dict[source].get_columns(target)
        all_metrics = list(map(lambda x: 'special:'+x, self.special_metrics)) + columns
        return all_metrics

    def special_metric_handle(self, metric, source, options: dict):
        if metric == 'agreement_count':
            data = self.file_class_dict[source].get_data_by_column('id', options)
            data.sort_index()
            res = data.groupby(data.index).count()
            res.name = metric
            return res
        # elif metric == 'client_count':
        elif metric == 'default_rate':
            data_def = self.file_class_dict[source].get_data_by_column('default_12m', options)
            data_non_def = self.file_class_dict[source].get_data_by_column('cur_default', options)
            res = data_def.groupby(data_def.index).sum() / data_non_def.groupby(data_non_def.index).sum()
            res.name = metric
            return res
        elif metric == 'avg_PD':
            data = self.file_class_dict[source].get_data_by_column('pd', options)
            res = data.groupby(data.index).mean()
            res.name = metric
            return res
        elif metric == 'gini':
            data = self.file_class_dict[source].get_data_by_column(['pd', 'default_12m'], options)
            res = 2 * data.groupby(data.index).apply(lambda group: roc_auc_score(group.default_12m, group.pd)) - 1
            res.name = metric
            return res

    def get_data(self, request) -> list:
        results = []
        for target in request['targets']:
            source = target['source']
            if source == "":
                continue
            metric = target['target']
            if ':' in metric:
                metric = metric.split(':')[1]
            options = parse_optional(target)
            if metric in self.special_metrics:
                data = self.special_metric_handle(metric, source, options)
            else:
                data = self.file_class_dict[source].get_data_by_column(metric, options)
            if target.get('type', 'ts') == 'table':
                results.extend(dataframe_to_json_table(data))
            else:
                results.extend(dataframe_to_response(target, data))
        return results


import firebase_admin
from firebase_admin import credentials, firestore
import time
import threading
import urllib.request
from os import system

print('Initializing Firestore connection...')
# Credentials and Firebase App initialization. Always required
firCredentials = credentials.Certificate('')
firApp = firebase_admin.initialize_app (firCredentials)

# Get access to Firestore
db = firestore.client()
print('Connection initialized')

# Create an Event for notifying main thread.
callback_done = threading.Event()

# Create a callback on_snapshot function to capture changes
def on_snapshot1(col_snapshot, changes, read_time):

    print(u'Callback received query snapshot.')

    if not col_snapshot:
        return

    for change in changes:
        if change.type.name == 'ADDED':
            # print(f'uid : {change.document.id}')
            uid = f'{change.document.id}'
            #print(uid)

            user_ref = db.collection(u'result').document(uid)
            doc = user_ref.get()

            if doc.exists:
                # print(f'Document data: {doc.to_dict()}')
                url_front = doc.get('imageurl_front')
            else:
                print(u'No such document!')

            urllib.request.urlretrieve(url_front, "front.png")

            system("python DataWriteFront.py")

    callback_done.set()

def on_snapshot2(col_snapshot, changes, read_time):

    print(u'Callback received query snapshot.')

    if not col_snapshot:
        return

    for change in changes:
        if change.type.name == 'ADDED':
            # print(f'uid : {change.document.id}')
            uid = f'{change.document.id}'
            #print(uid)

            user_ref = db.collection(u'result').document(uid)
            doc = user_ref.get()

            if doc.exists:
                # print(f'Document data: {doc.to_dict()}')
                url_side = doc.get('imageurl_side')
            else:
                print(u'No such document!')

            urllib.request.urlretrieve(url_side, "side.png")

            system("python DataWriteSide.py")

    callback_done.set()

while True:
    print('processing...')
    col_query1 = db.collection(u'result').where(u'flag_front', u'==', 1)
    col_query2 = db.collection(u'result').where(u'flag_side', u'==', 1)

    query_watch = col_query1.on_snapshot(on_snapshot1)
    query_watch = col_query2.on_snapshot(on_snapshot2)

    time.sleep(4)


from django.shortcuts import render
from django.views.generic import TemplateView

from apps.accounts.mixins import LoggedInRequiredMixin
from .models import Painter
from django.views.generic.edit import CreateView
from django.http import HttpResponse, HttpResponseBadRequest, HttpResponseServerError
from .forms import UploadFileForm
from django.views.generic.edit import FormView
from django.shortcuts import render, redirect
from django.contrib.messages import get_messages
from django.contrib import messages
import json

# Create your views here.
class PainterCreate(LoggedInRequiredMixin, TemplateView):
    # template_name = 'painter.html'
    template_name = 'editor_painter.html'

    def get_context_data(self, **kwargs):
        context = super(PainterCreate, self).get_context_data(**kwargs)
        projects = Painter.get_all(self.request)
        context['projects'] = projects

        storage = get_messages(self.request)
        kv = None
        for message in storage:
            kv = message
            break
        # parse
        if kv:
            kv = kv.message
            kv_dict = json.loads(kv)
            context.update(kv_dict)
        return context

class CreateProject(LoggedInRequiredMixin, CreateView):

    def post(self, request, *args, **kwargs):
        name = request.POST.get('name')
        if not name:
            return HttpResponseBadRequest()
        try:
            user = request.user
            p = Painter(name=name, user=user)
            p.save()
        except:
            import traceback
            return HttpResponseServerError(traceback.format_exc())

        return HttpResponse(content='Created project `{0}`.'.format(name))

class UploadFile(LoggedInRequiredMixin, FormView):
    template_name = 'editor_painter.html'

    def post(self, request, *args, **kwargs):
        content = request.FILES['file']
        filename = request.FILES['file'].name
        with open(filename, 'wb') as f:
            for chunk in content.chunks():
                f.write(chunk)
        context = {
            'upload_success': True,
            'hide_upload_dialog': True,
        }
        messages.add_message(request, messages.INFO, json.dumps(context))
        return redirect('/painters/painter/')


from django.contrib.auth import get_user
from django.utils.functional import SimpleLazyObject
from drf_yasg.utils import swagger_auto_schema
from rest_framework import status
from rest_framework.decorators import action
from rest_framework.generics import GenericAPIView
from rest_framework.response import Response
from rest_framework.viewsets import ViewSetMixin
from rest_framework_simplejwt.exceptions import TokenError, InvalidToken
from rest_framework_simplejwt.serializers import TokenObtainPairSerializer, TokenRefreshSerializer

from .. import serializers as srz

class TokenViewSet(ViewSetMixin,
                   GenericAPIView):
    def get_serializer_class(self):
        if self.action == 'create':
            return TokenObtainPairSerializer
        if self.action == 'refresh':
            return TokenRefreshSerializer
        return TokenObtainPairSerializer

    # noinspection PyTypeChecker
    @swagger_auto_schema(responses={status.HTTP_201_CREATED: srz.TokenObtainPairResponseSerializer})
    def create(self, request, *args, **kwargs):
        serializer = self.get_serializer(data=request.data)

        try:
            serializer.is_valid(raise_exception=True)
        except TokenError as e:
            raise InvalidToken(e.args[0])

        return Response({
                **serializer.validated_data,
                'role': serializer.user.role,
                'branch': serializer.user.branch.name,
                'fullname': serializer.user.fullname,
                'inn': serializer.user.inn,
        }, status=status.HTTP_200_OK)

    #noinspection PyTypeChecker
    @action(['post'], False, 'refresh')
    @swagger_auto_schema(responses={status.HTTP_200_OK: srz.TokenRefreshResponseSerializer})
    def refresh(self, request, *args, **kwargs):
        serializer = self.get_serializer(data=request.data)

        try:
            serializer.is_valid(raise_exception=True)
        except TokenError as e:
            raise InvalidToken(e.args[0])

        return Response({
            **serializer.validated_data,
            'role': serializer.user.role,
            'branch': serializer.user.branch.name,
            'fullname': serializer.user.fullname,
        }, status=status.HTTP_200_OK)


import contextlib
import hashlib
import logging
import os
import re
from datetime import datetime, timedelta
from fnmatch import fnmatch

import click
import click_odoo
from click_odoo import odoo

from ._dbutils import advisory_lock, db_exists, pg_connect
from .manifest import expand_dependencies
from .update import _save_installed_checksums

_logger = logging.getLogger(__name__)

EXCLUDE_PATTERNS = ("*.pyc", "*.pyo")

def check_dbname(dbname):
    if not re.match("^[A-Za-z][A-Za-z0-9-_.]*$", dbname):
        raise click.ClickException("Invalid database name '{}'".format(dbname))

def check_cache_prefix(cache_prefix):
    if not re.match("^[A-Za-z][A-Za-z0-9-]{0,7}$", cache_prefix):
        raise click.ClickException(
            "Invalid cache prefix name '{}'".format(cache_prefix)
        )

@contextlib.contextmanager
def _patch_ir_attachment_store(force_db_storage):
    @odoo.api.model
    def _db_storage(self):
        return "db"

    if not force_db_storage:
        yield
    else:
        # make sure attachments created during db initialization
        # are stored in database, so we get something consistent
        # when recreating the db by copying the cached template
        if odoo.release.version_info >= (12, 0):
            from odoo.addons.base.models.ir_attachment import IrAttachment
        else:
            from odoo.addons.base.ir.ir_attachment import IrAttachment
        orig = IrAttachment._storage
        IrAttachment._storage = _db_storage
        try:
            yield
        finally:
            IrAttachment._storage = orig

def odoo_createdb(dbname, demo, module_names, force_db_storage):
    with _patch_ir_attachment_store(force_db_storage):
        odoo.service.db._create_empty_database(dbname)
        odoo.tools.config["init"] = dict.fromkeys(module_names, 1)
        odoo.tools.config["without_demo"] = not demo
        odoo.modules.registry.Registry.new(dbname, force_demo=demo, update_module=True)
        _logger.info(click.style(f"Created new Odoo database {dbname}.", fg="green"))
        with odoo.sql_db.db_connect(dbname).cursor() as cr:
            _save_installed_checksums(cr)
        odoo.sql_db.close_db(dbname)

def _fnmatch(filename, patterns):
    for pattern in patterns:
        if fnmatch(filename, pattern):
            return True
    return False

def _walk(top, exclude_patterns=EXCLUDE_PATTERNS):
    for dirpath, dirnames, filenames in os.walk(top):
        dirnames.sort()
        reldir = os.path.relpath(dirpath, top)
        if reldir == ".":
            reldir = ""
        for filename in sorted(filenames):
            filepath = os.path.join(reldir, filename)
            if _fnmatch(filepath, exclude_patterns):
                continue
            yield filepath

def addons_hash(module_names, with_demo):
    h = hashlib.sha1()
    h.update("!demo={}!".format(int(bool(with_demo))).encode("utf8"))
    for module_name in sorted(expand_dependencies(module_names, True, True)):
        module_path = odoo.modules.get_module_path(module_name)
        h.update(module_name.encode("utf8"))
        for filepath in _walk(module_path):
            h.update(filepath.encode("utf8"))
            with open(os.path.join(module_path, filepath), "rb") as f:
                h.update(f.read())
    return h.hexdigest()

def refresh_module_list(dbname):
    with click_odoo.OdooEnvironment(database=dbname) as env:
        env["ir.module.module"].update_list()

class DbCache:
    """Manage a cache of db templates.

    Templates are named prefix-YYYYmmddHHMM-hashsum, where
    YYYYmmddHHMM is the date and time when the given hashsum has last been
    used for that prefix.
    """

    HASH_SIZE = hashlib.sha1().digest_size * 2
    MAX_HASHSUM = "f" * HASH_SIZE

    def __init__(self, prefix, pgcr):
        check_cache_prefix(prefix)
        self.prefix = prefix
        self.pgcr = pgcr

    def _lock(self):
        return advisory_lock(self.pgcr, self.prefix)

    def _make_pattern(self, dt=None, hs=None):
        if dt:
            dtpart = dt.strftime("%Y%m%d%H%M")
        else:
            dtpart = "_" * 12
        if hs:
            hspart = hs
        else:
            hspart = "_" * self.HASH_SIZE
        pattern = self.prefix + "-" + dtpart + "-" + hspart
        # 63 is max postgres db name, so we may truncate the hash part
        return pattern[:63]

    def _make_new_template_name(self, hashsum):
        return self._make_pattern(dt=datetime.utcnow(), hs=hashsum)

    def _create_db_from_template(self, dbname, template):
        _logger.info(
            click.style(
                f"Creating database {dbname} from template {template}",
                fg="green",
            )
        )
        self.pgcr.execute(
            f"""
            CREATE DATABASE "{dbname}"
            ENCODING 'unicode'
            TEMPLATE "{template}"
            """
        )

    def _rename_db(self, dbname_from, dbname_to):
        self.pgcr.execute(
            f"""
            ALTER DATABASE "{dbname_from}"
            RENAME TO "{dbname_to}"
            """
        )

    def _drop_db(self, dbname):
        _logger.info(f"Dropping database {dbname}")
        self.pgcr.execute(
            f"""
            DROP DATABASE "{dbname}"
            """
        )

    def _find_template(self, hashsum):
        """search same prefix and hashsum, any date"""
        pattern = self.prefix + "-____________-" + hashsum
        self.pgcr.execute(
            """
            SELECT datname FROM pg_database
            WHERE datname like %s
            ORDER BY datname DESC  -- MRU first
            """,
            (pattern,),
        )
        r = self.pgcr.fetchone()
        if r:
            return r[0]
        else:
            return None

    def _touch(self, template_name, hashsum):
        # change the template date (MRU mechanism)
        assert template_name.endswith(hashsum)
        new_template_name = self._make_new_template_name(hashsum)
        if template_name != new_template_name:
            self._rename_db(template_name, new_template_name)

    def create(self, new_database, hashsum):
        """Create a new database from a cached template matching hashsum"""
        with self._lock():
            template_name = self._find_template(hashsum)
            if not template_name:
                return False
            else:
                self._create_db_from_template(new_database, template_name)
                self._touch(template_name, hashsum)
                return True

    def add(self, new_database, hashsum):
        """Create a new cached template"""
        with self._lock():
            template_name = self._find_template(hashsum)
            if template_name:
                self._touch(template_name, hashsum)
            else:
                new_template_name = self._make_new_template_name(hashsum)
                self._create_db_from_template(new_template_name, new_database)

    @property
    def size(self):
        with self._lock():
            pattern = self._make_pattern()
            self.pgcr.execute(
                """
                SELECT count(*) FROM pg_database
                WHERE datname like %s
            """,
                (pattern,),
            )
            return self.pgcr.fetchone()[0]

    def purge(self):
        with self._lock():
            pattern = self._make_pattern()
            self.pgcr.execute(
                """
                SELECT datname FROM pg_database
                WHERE datname like %s
            """,
                (pattern,),
            )
            for (datname,) in self.pgcr.fetchall():
                self._drop_db(datname)

    def trim_size(self, max_size):
        with self._lock():
            pattern = self._make_pattern()
            self.pgcr.execute(
                """
                SELECT datname FROM pg_database
                WHERE datname like %s
                ORDER BY datname DESC
                OFFSET %s
            """,
                (pattern, max_size),
            )
            for (datname,) in self.pgcr.fetchall():
                self._drop_db(datname)

    def trim_age(self, max_age):
        with self._lock():
            pattern = self._make_pattern()
            max_name = self._make_pattern(
                dt=datetime.utcnow() - max_age, hs=self.MAX_HASHSUM
            )
            self.pgcr.execute(
                """
                SELECT datname FROM pg_database
                WHERE datname like %s
                  AND datname <= %s
                ORDER BY datname DESC
            """,
                (pattern, max_name),
            )
            for (datname,) in self.pgcr.fetchall():
                self._drop_db(datname)

@click.command()
@click_odoo.env_options(
    default_log_level="warn",
    with_database=False,
    with_rollback=False,
    with_addons_path=True,
)
@click.option(
    "--new-database",
    "-n",
    required=False,
    help="Name of new database to create, possibly from cache. "
    "If absent, only the cache trimming operation is executed.",
)
@click.option(
    "--modules",
    "-m",
    default="base",
    show_default=True,
    help="Comma separated list of addons to install.",
)
@click.option(
    "--demo/--no-demo", default=True, show_default=True, help="Load Odoo demo data."
)
@click.option(
    "--cache/--no-cache",
    default=True,
    show_default=True,
    help="Use a cache of database templates with the exact "
    "same addons installed. Disabling this option "
    "also disables all other cache-related operations "
    "such as max-age or size. Note: when the cache is "
    "enabled, all attachments created during database "
    "initialization are stored in database instead "
    "of the default Odoo file store.",
)
@click.option(
    "--cache-prefix",
    default="cache",
    show_default=True,
    help="Prefix to use when naming cache template databases "
    "(max 8 characters). CAUTION: all databases named like "
    "{prefix}-____________-% will eventually be dropped "
    "by the cache control mechanism, so choose the "
    "prefix wisely.",
)
@click.option(
    "--cache-max-age",
    default=30,
    show_default=True,
    type=int,
    help="Drop cache templates that have not been used for "
    "more than N days. Use -1 to disable.",
)
@click.option(
    "--cache-max-size",
    default=5,
    show_default=True,
    type=int,
    help="Keep N most recently used cache templates. Use "
    "-1 to disable. Use 0 to empty cache.",
)
@click.option(
    "--unless-exists",
    is_flag=True,
    help="Don't report error if database already exists.",
)
def main(
    env,
    new_database,
    modules,
    demo,
    cache,
    cache_prefix,
    cache_max_age,
    cache_max_size,
    unless_exists,
):
    """Create an Odoo database with pre-installed modules.

    Almost like standard Odoo does with the -i option,
    except this script manages a cache of database templates with
    the exact same addons installed. This is particularly useful to
    save time when initializing test databases.

    Cached templates are identified by computing a sha1
    checksum of modules provided with the -m option, including their
    dependencies and corresponding auto_install modules.
    """
    if new_database:
        check_dbname(new_database)
    if unless_exists and db_exists(new_database):
        msg = "Database already exists: {}".format(new_database)
        click.echo(click.style(msg, fg="yellow"))
        return
    module_names = [m.strip() for m in modules.split(",")]
    if not cache:
        if new_database:
            odoo_createdb(new_database, demo, module_names, False)
        else:
            _logger.info(
                "Cache disabled and no new database name provided. " "Nothing to do."
            )
    else:
        with pg_connect() as pgcr:
            dbcache = DbCache(cache_prefix, pgcr)
            if new_database:
                hashsum = addons_hash(module_names, demo)
                if dbcache.create(new_database, hashsum):
                    _logger.info(
                        click.style(
                            "Found matching database template! âœ¨ ðŸ�° âœ¨",
                            fg="green",
                            bold=True,
                        )
                    )
                    refresh_module_list(new_database)
                else:
                    odoo_createdb(new_database, demo, module_names, True)
                    dbcache.add(new_database, hashsum)
            if cache_max_size >= 0:
                dbcache.trim_size(cache_max_size)
            if cache_max_age >= 0:
                dbcache.trim_age(timedelta(days=cache_max_age))

if __name__ == "__main__":  # pragma: no cover
    main()


from __future__ import unicode_literals

import datetime

from django.contrib.admin.util import (lookup_field, display_for_field,
    display_for_value, label_for_field)
from django.contrib.admin.views.main import (ALL_VAR, EMPTY_CHANGELIST_VALUE,
    ORDER_VAR, PAGE_VAR, SEARCH_VAR)
from django.contrib.admin.templatetags.admin_static import static
from django.core.exceptions import ObjectDoesNotExist
from django.db import models
from django.utils import formats
from django.utils.html import format_html
from django.utils.safestring import mark_safe
from django.utils import six
from django.utils.text import capfirst
from django.utils.translation import ugettext as _
from django.utils.encoding import smart_text, force_text
from django.template import Library
from django.template.loader import get_template
from django.template.context import Context

from django.contrib.admin.templatetags.admin_list import (_boolean_icon,
    result_hidden_fields, result_headers, ResultList)

register = Library()

DOT = '.'

def items_for_result(cl, result, form):
    """
    Generates the actual list of data.
    """
    first = True
    pk = cl.lookup_opts.pk.attname
    for field_name in cl.list_display:
        row_class = ''
        try:
            f, attr, value = lookup_field(field_name, result, cl.model_admin)
        except ObjectDoesNotExist:
            result_repr = EMPTY_CHANGELIST_VALUE
        else:
            if f is None:
                if field_name == 'action_checkbox':
                    row_class = mark_safe(' class="action-checkbox"')
                allow_tags = getattr(attr, 'allow_tags', False)
                boolean = getattr(attr, 'boolean', False)
                if boolean:
                    allow_tags = True
                result_repr = display_for_value(value, boolean)
                # Strip HTML tags in the resulting text, except if the
                # function has an "allow_tags" attribute set to True.
                if allow_tags:
                    result_repr = mark_safe(result_repr)
                if isinstance(value, (datetime.date, datetime.time)):
                    row_class = mark_safe(' class="nowrap"')
            else:
                if isinstance(f.rel, models.ManyToOneRel):
                    field_val = getattr(result, f.name)
                    if field_val is None:
                        result_repr = EMPTY_CHANGELIST_VALUE
                    else:
                        result_repr = field_val
                else:
                    result_repr = display_for_field(value, f)
                if isinstance(f, (models.DateField, models.TimeField, models.ForeignKey)):
                    row_class = mark_safe(' class="nowrap"')
        if force_text(result_repr) == '':
            result_repr = mark_safe(' ')
        # If list_display_links not defined, add the link tag to the first field
        if (first and not cl.list_display_links) or field_name in cl.list_display_links:
            table_tag = {True:'th', False:'td'}[first]
            first = False
            url = cl.url_for_result(result)
            # Convert the pk to something that can be used in Javascript.
            # Problem cases are long ints (23L) and non-ASCII strings.
            if cl.to_field:
                attr = str(cl.to_field)
            else:
                attr = pk
            value = result.serializable_value(attr)
            result_id = repr(force_text(value))[1:]
            yield format_html('<{0}{1}>{4}</{5}>',
                              table_tag,
                              row_class,
                              url,
                              format_html(' onclick="opener.dismissRelatedLookupPopup(window, {0}); return false;"', result_id)
                                if cl.is_popup else '',
                              result_repr,
                              table_tag)
        else:
            # By default the fields come from ModelAdmin.list_editable, but if we pull
            # the fields out of the form instead of list_editable custom admins
            # can provide fields on a per request basis
            if (form and field_name in form.fields and not (
                    field_name == cl.model._meta.pk.name and
                        form[cl.model._meta.pk.name].is_hidden)):
                bf = form[field_name]
                result_repr = mark_safe(force_text(bf.errors) + force_text(bf))
            yield format_html('{1}	', row_class, result_repr)
    if form and not form[cl.model._meta.pk.name].is_hidden:
        yield format_html('{0}	', force_text(form[cl.model._meta.pk.name]))

def results(cl):
    if cl.formset:
        for res, form in zip(cl.result_list, cl.formset.forms):
            yield ResultList(form, items_for_result(cl, res, form))
    else:
        for res in cl.result_list:
            yield ResultList(None, items_for_result(cl, res, None))

def result_hidden_fields(cl):
    if cl.formset:
        for res, form in zip(cl.result_list, cl.formset.forms):
            if form[cl.model._meta.pk.name].is_hidden:
                yield mark_safe(force_text(form[cl.model._meta.pk.name]))

@register.inclusion_tag("admin/orgs/organization/change_list_results.html")
def result_list_org(cl):
    """
    Displays the headers and data list together
    """
    headers = list(result_headers(cl))
    num_sorted_fields = 0
    for h in headers:
        if h['sortable'] and h['sorted']:
            num_sorted_fields += 1
    return {'cl': cl,
            'result_hidden_fields': list(result_hidden_fields(cl)),
            'result_headers': headers,
            'num_sorted_fields': num_sorted_fields,
            'results': list(results(cl))}


from django.urls import path
from . import views
urlpatterns = [
    path('', views.index, name='index'),
    path('aboutus', views.aboutus, name='aboutus'),
    path('login/', views.loginPage, name='login'),
    path('logout/', views.logoutUser, name='logout'),
    path('quiz/', views.get_questions, name='get_questions'),
    path('user_input/', views.process_user_input,
         name='user_input'),  # type: ignore

]


#!/usr/bin/env python
# -*- coding: utf-8 -*-
__title__ = 'ljg logging'
__author__ = 'ljg'
__mtime__ = '20190401'

import logging
import colorlog
import os
from logging.handlers import TimedRotatingFileHandler

class Logger:

    def __init__(self, clevel=logging.INFO, Flevel=logging.INFO):
        if not os.path.exists('logs'):
            os.mkdir('logs')
        self.logger = logging.Logger(__name__)  # logging.getLogger('log')

        fmt_sh = colorlog.ColoredFormatter('%(log_color)s%(asctime)s[%(levelname)-7s][%(module)s:%(lineno)04d]%(message)s', '%Y-%m-%d %H:%M:%S')
        # 设置CMD日志
        sh = logging.StreamHandler()
        sh.setFormatter(fmt_sh)
        sh.setLevel(clevel)

        # 设置文件日志
        # fh = logging.FileHandler(logfile_dir + '/log.log', 'a', encoding='utf-8')

        # 每天凌晨,备份之前的记录
        fh = TimedRotatingFileHandler(filename="logs/log", when="MIDNIGHT", interval=1, backupCount=30)
        fh.suffix = "%Y-%m-%d.log"  # 此格式必须与上面when对应,否则无法实现backcount设置的保留删除功能
        # fh.extMatch = re.compile(r"^\d{4}-\d{2}-\d{2}.log$")
        # fh.suffix = "%Y-%m-%d_%H-%M.log"
        # fh.extMatch = re.compile(r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}.log$")
        fmt = logging.Formatter('%(asctime)s[%(levelname)-7s][%(module)s:%(lineno)04d]%(message)s', '%Y-%m-%d %H:%M:%S')
        fh.setFormatter(fmt)
        fh.setLevel(Flevel)

        self.logger.addHandler(sh)
        self.logger.addHandler(fh)

        self.info = self.logger.info
        self.debug = self.logger.debug
        self.war = self.logger.warning
        self.error = self.logger.error
        self.cri = self.logger.critical

def main():
    log = Logger(logging.INFO)
    while True:
        log.info('test')
        log.war('test')
        log.error('test')

if __name__ == '__main__':
    main()


#!/usr/bin/env python
# code for python 3
import time
import serial
import io

ser = serial.Serial(
    port='/dev/ttyS0',
    baudrate = 19200,
    parity=serial.PARITY_NONE,
    stopbits=serial.STOPBITS_ONE,
    bytesize=serial.EIGHTBITS,
    timeout=0.005
)
sio = io.TextIOWrapper(io.BufferedRWPair(ser, ser))
print ('Serial Connection Ready!')
time.sleep(0.2)
count = 0
data_in = []
state = 0
iterator = 0
timer_start = time.time()
while True:
    data_out = "GET_OBS \n"
    sio.write(data_out)
    sio.flush()
    data_in = []
    while (not data_in):
        data_in=sio.readline()

    if (state == 0):
        count = count + 1
    else:
        count = count - 1

    if state == 0 and count > 100:
        state = 1
        count = 100
    if state == 1 and count < 0:
        state = 0
        count = 0
    #print (str(data_in))

    data_out = "CMD_VEL " + str(-count)  + " " + str(count) + "\n"
    #data_out = "CMD_VEL " + str(0)  + " " + str(0) + "\n"
    sio.write(data_out)
    sio.flush()
    time.sleep(0.02)
    iterator = iterator + 1
    if (iterator % 100) == 0:
       iterator = 1

       timer_now = time.time()
       print ("Running at ", 100.0/(timer_now - timer_start), " Hz")
       timer_start = time.time()


import random
from logging import getLogger

import numpy as np
import torch

logger = getLogger()

class HrObservationMatrixGenerator:
    def __init__(self, seed: int = 42):
        logger.info(f"obs matrix generator got seed = {seed}")
        self.rnd = random.Random(seed)
        self.obs_matrix = None

    def generate_obs_matrix(
        self,
        *,
        nx: int,
        ny: int,
        obs_interval: int = 34,
        device: str = "cpu",
        dtype: torch.dtype = torch.float64,
    ):
        obs_init_index = self.rnd.randint(0, obs_interval - 1)

        obs_indices = np.zeros(nx * ny)
        obs_indices[obs_init_index::obs_interval] = 1.0
        obs_indices = np.where(obs_indices == 1.0)[0]

        num_obs = len(obs_indices)

        obs_matrix = torch.zeros(num_obs, nx * ny, dtype=dtype)

        for i, j in enumerate(obs_indices):
            obs_matrix[i, j] = 1.0

        p = 100 * torch.sum(obs_matrix).item() / (nx * ny)
        logger.debug(f"observatio prob = {p} [%]")

        self.obs_matrix = obs_matrix.to(device)

        return self.obs_matrix

    def generate_projection_matrix(self):

        if self.obs_matrix is None:
            raise Exception()

        return self.obs_matrix.t().mm(self.obs_matrix)


import webiopi
import datetime
import sqlite3
import subprocess
import sys

GPIO = webiopi.GPIO
AUTO = 1
SCRIPT_PATH = "/home/pi/HydroP/python/"

LIGHT1 = 17 # GPIO pin using BCM numbering

dbconn = sqlite3.connect(SCRIPT_PATH + "hydro.db")
dbconn.row_factory = sqlite3.Row
dbcursor = dbconn.cursor()

# setup function is automatically called at WebIOPi startup
def setup():
    # set the GPIO used by the light to output
    GPIO.setFunction(LIGHT1, GPIO.OUT)

    # retrieve current datetime
    now = datetime.datetime.now()

    dbcursor.execute("SELECT status, interval FROM devices WHERE name = \"LIGHT1\"")
    lightDevice = dbcursor.fetchone()

    if (lightDevice != None) and (lightDevice["status"] == AUTO):
        setLightInterval(lightDevice['interval'])

# loop function is repeatedly called by WebIOPi
def loop():
    # retrieve current datetime
    now = datetime.datetime.now()

    # gives CPU some time before looping again
    webiopi.sleep(1)

# destroy function is called at WebIOPi shutdown
def destroy():
    GPIO.digitalWrite(LIGHT1, GPIO.LOW)

@webiopi.macro
def setLightInterval(interval):
	subprocess.Popen(["python",SCRIPT_PATH + "light_loop.py",str(interval)])


import matplotlib
import numpy as np
import pandas as pd
from os import listdir
from pathlib import Path
from typing import Tuple
import matplotlib.pyplot as plt
from os.path import isfile, join

class Data:
    def __init__(self, path: str) -> None:
        self.path = path
        self.onlyfiles = self.getDfNames()

    def getDfNames(self) -> Tuple[str, ...]:
        """
        Fetches all objects in a directory

        Returns:
            Tuple[str, ...]: list of pathnames
        """
        return [f for f in listdir(self.path) if isfile(join(self.path, f))]

    def addColumnToDf(self, df_name: str, column_name: str, value: list) -> None:
        """
        Adds a column to the dataframe.
        if only single entry in vlaue, it is assumed that this value is for all rows in df.

        Args:
            df_name (str): name + filetype for the given dataframe file, must be .hdf5
            column_name (str): name of the column
            value (list): list of value or values.

        """

        try:
            df_name in self.onlyfiles

        except ValueError:
            print("Dataframe is not in directory, check for typos")

        df = pd.read_hdf(self.path / df_name)

        try:
            df[column_name]

        except:

            if len(value) != len(df):
                value = value * len(df)

            df[column_name] = value
            df.to_hdf(self.path / df_name, "mini")

    def printDf(self, df_name: str) -> None:
        """prints the first 5 entries in the dataframe, and the length of its columns

        Args:
            df_name (str): name of a given dataframe
        """

        df = pd.read_hdf(self.path / df_name)
        print(df.head(), len(df.columns))

    def addChannelType(self):
        """
        Adds channetype to the dataframe
        """

        column_name = "Channeltype"

        for idx, file in enumerate(self.onlyfiles):
            name_idx = file.find("_3lep")
            value = [f"{file[:name_idx]}"]

            self.addColumnToDf(df_name=file, column_name=column_name, value=value)
            self.printDf(file)

if __name__ == "__main__":
    path = Path("/storage/William_Sakarias/Sakarias_Data")

    data = Data(path)
    data.addChannelType()


"""
On the fly generator. Crop out portions of a large image, and pass boxes and annotations. This follows the csv_generator template. Satifies the format in generator.py
"""
import pandas as pd
import h5py

from keras_retinanet.preprocessing.generator import Generator
from keras_retinanet.utils.image import read_image_bgr
from keras_retinanet.utils.visualization import draw_annotations

import numpy as np
from PIL import Image
from six import raise_from
import random

import csv
import sys
import os.path

import cv2
import slidingwindow as sw
import itertools

from DeepForest.utils import image_utils

class H5Generator(Generator):
    """ Generate data for a custom h5 dataset.
    """

    def __init__(
        self,
        data,
        DeepForest_config,
        group_method="none",
        name=None,
        **kwargs
    ):
        """ Initialize a data generator.

        """
        self.image_names = []
        self.image_data  = {}
        self.name = name
        self.windowdf = data
        self.DeepForest_config = DeepForest_config

        #Holder for the group order, after shuffling we can still recover loss -> window
        self.group_order = {}
        self.group_method=group_method

        #Holder for image path, keep from reloading same image to save time.
        self.previous_image_path=None

        #Turn off lidar checking during prediction for training sets.
        self.with_lidar=False

        #Read classes
        self.classes={"Tree": 0}

        #Create label dict
        self.labels = {}
        for key, value in self.classes.items():
            self.labels[value] = key

        #Set groups at first order.
        self.define_groups(shuffle=False)

        #report total number of annotations
        self.total_trees = self.total_annotations()

        super(H5Generator, self).__init__(**kwargs)

    def __len__(self):
        """Number of batches for generator"""
        return len(self.groups)

    def size(self):
        """ Size of the dataset.
        """
        image_data= self.windowdf.to_dict("index")
        image_names = list(image_data.keys())

        return len(image_names)

    def num_classes(self):
        """ Number of classes in the dataset.
        """
        return max(self.classes.values()) + 1

    def name_to_label(self, name):
        """ Map name to label.
        """
        return self.classes[name]

    def label_to_name(self, label):
        """ Map label to name.
        """
        return self.labels[label]

    def total_annotations(self):
        """ Find the total number of annotations for the dataset
        """
        #Find matching annotations
        tiles = self.windowdf[["tile","site"]].drop_duplicates()
        total_annotations = 0

        #Select annotations
        #Optionally multiple h5 dirs
        for index, row in tiles.iterrows():
            h5_dir = self.DeepForest_config[row["site"]]["h5"]
            tilename = row["tile"]
            csv_name = os.path.join(h5_dir, os.path.splitext(tilename)[0]+'.csv')

            try:
                annotations = pd.read_csv(csv_name)
            except Exception as e:
                print(e)
                print("The csv named {} from tilename {} encountered an error when counting annotations".format(csv_name, tilename))
                continue

            selected_annotations = pd.merge(self.windowdf, annotations)
            total_annotations += len(selected_annotations)

        print("There are a total of {} tree annotations in the {} generator".format(total_annotations, self.name))

        return(total_annotations)

    def define_groups(self, shuffle=False):
        '''
        Define image data and names based on grouping of tiles for computational efficiency
        '''
        #group by tile
        print(self.windowdf.tile.unique()[0:5])

        if shuffle:
            groups = [df for _, df in self.windowdf.groupby('tile')]

            #Shuffle order of windows within a tile
            groups = [x.sample(frac=1) for x in groups]

            #Shuffle order of tiles
            random.shuffle(groups)
            newdf = pd.concat(groups,sort=False,ignore_index=True)
        else:
            newdf = self.windowdf

        #Bring pandas frame back together
        image_data = newdf.to_dict("index")
        image_names = list(image_data.keys())

        return(image_data, image_names)

    def load_image(self, image_index):
        """ Load an image at the image_index.
        """
        #Select sliding window and tile
        try:
            image_name = self.image_names[image_index]
        except Exception as e:
            print("Failed on image index {}".format(image_index))
            print("There are {} names in the image names object".format(len(self.image_names)))

        self.row = self.image_data[image_name]

        #Open image to crop
        ##Check if tile the is same as previous draw from generator, this will save time.
        if not self.row["tile"] == self.previous_image_path:

            #Set directory based on site
            h5_dir = self.DeepForest_config[self.row["site"]]["h5"]

            #tilename for h5 and csv files
            tilename = os.path.split(self.row["tile"])[-1]
            tilename = os.path.splitext(tilename)[0]
            h5_name = os.path.join(h5_dir, tilename+'.h5')
            csv_name = os.path.join(h5_dir, tilename+'.csv')

            #Read h5
            self.hf = h5py.File(h5_name, 'r')

            #Read corresponding csv labels
            self.annotations = pd.read_csv(csv_name)

        #read image from h5
        window = self.row["window"]
        image = self.hf["train_imgs"][window,...]

        #Save image path for next evaluation to check
        self.previous_image_path = self.row["tile"]

        return image

    def load_annotations(self, image_index):
        '''
        Load annotations from csv file
        '''
        #Select sliding window and tile
        image_name = self.image_names[image_index]
        self.row = self.image_data[image_name]

        #Find annotations
        annotations = self.annotations.loc[(self.annotations["tile"] == self.row["tile"]) & (self.annotations["window"] == self.row["window"])]

        return annotations[["0","1","2","3","4"]].values

    def compute_windows(self):
        ''''
        Create a sliding window object for reference
        '''
        #Load tile
        site = self.annotation_list.site.unique()[0]
        base_dir = self.DeepForest_config[site][self.name]["RGB"]
        image = os.path.join(base_dir, self.annotation_list.rgb_path.unique()[0])
        im = Image.open(image)
        numpy_image = np.array(im)

        #Generate sliding windows
        windows = sw.generate(numpy_image, sw.DimOrder.HeightWidthChannel,  self.DeepForest_config["patch_size"], self.DeepForest_config["patch_overlap"])

        return(windows)


from datetime import datetime

from airflow.decorators import dag
from airflow.operators.empty import EmptyOperator
from airflow.operators.python import BranchPythonOperator
from datarobot import TARGET_TYPE
from datarobot.enums import DEPLOYMENT_IMPORTANCE

from datarobot_provider.operators.ai_catalog import UploadDatasetOperator
from datarobot_provider.operators.custom_models import CreateCustomInferenceModelOperator
from datarobot_provider.operators.custom_models import CreateCustomModelDeploymentOperator
from datarobot_provider.operators.custom_models import CreateCustomModelVersionOperator
from datarobot_provider.operators.custom_models import CreateExecutionEnvironmentOperator
from datarobot_provider.operators.custom_models import CreateExecutionEnvironmentVersionOperator
from datarobot_provider.operators.custom_models import CustomModelTestOperator
from datarobot_provider.operators.custom_models import GetCustomModelTestOverallStatusOperator

@dag(
    schedule=None,
    start_date=datetime(2023, 1, 1),
    tags=['example', 'custom model'],
    params={
        "execution_environment_name": "Execution Env Airflow",
        "execution_environment_description": "Demo Execution Environment created by Airflow provider",
        "programming_language": "python",
        "required_metadata_keys": [],  # example: [{"field_name": "test_key", "display_name": "test_display_name"}],
        "custom_model_description": 'This is a custom model created by Airflow',
        "environment_version_description": "created by Airflow provider",
        "custom_model_name": "Airflow Custom model Demo",
        "target_type": TARGET_TYPE.REGRESSION,
        "target_name": 'Grade 2014',
        "is_major_update": True,
        "is_training_data_for_versions_permanently_enabled": True,
        "docker_context_path": "/usr/local/airflow/dags/datarobot-user-models/public_dropin_environments/python3_pytorch/",
        "custom_model_folder": "/usr/local/airflow/dags/datarobot-user-models/model_templates/python3_pytorch/",
        "test_dataset_file_path": "/usr/local/airflow/dags/datarobot-user-models/tests/testdata/juniors_3_year_stats_regression.csv",
        "train_dataset_file_path": "/usr/local/airflow/dags/datarobot-user-models/tests/testdata/juniors_3_year_stats_regression.csv",
    },
)
def create_custom_model_pipeline(
    prediction_server_id="5fbc1924ccfc5a0025c424bf", deployment_name="Demo Deployment Airflow"
):
    create_execution_environment_op = CreateExecutionEnvironmentOperator(
        task_id='create_execution_environment',
    )

    create_execution_environment_version_op = CreateExecutionEnvironmentVersionOperator(
        task_id='create_execution_environment_version',
        execution_environment_id=create_execution_environment_op.output,
    )

    create_custom_inference_model_op = CreateCustomInferenceModelOperator(
        task_id='create_custom_inference_model',
    )

    train_dataset_uploading_op = UploadDatasetOperator(
        task_id="train_dataset_uploading", file_path_param="train_dataset_file_path"
    )

    create_custom_model_version_op = CreateCustomModelVersionOperator(
        task_id='create_custom_model_version',
        custom_model_id=create_custom_inference_model_op.output,
        base_environment_id=create_execution_environment_op.output,
        training_dataset_id=train_dataset_uploading_op.output,
    )

    test_dataset_uploading_op = UploadDatasetOperator(
        task_id="test_dataset_uploading", file_path_param="test_dataset_file_path"
    )

    custom_model_test_op = CustomModelTestOperator(
        task_id='custom_model_test',
        custom_model_id=create_custom_inference_model_op.output,
        custom_model_version_id=create_custom_model_version_op.output,
        dataset_id=test_dataset_uploading_op.output,
    )

    custom_model_test_overall_status_op = GetCustomModelTestOverallStatusOperator(
        task_id='custom_model_test_overall_status',
        custom_model_test_id=custom_model_test_op.output,
    )

    def choose_branch(custom_model_test_overall_status):
        if custom_model_test_overall_status == "succeeded":
            return ['deploy_custom_model']
        return ['custom_model_test_fail']

    branching = BranchPythonOperator(
        task_id='if_tests_succeed',
        python_callable=choose_branch,
        op_args=[custom_model_test_overall_status_op.output],
    )

    deploy_custom_model_op = CreateCustomModelDeploymentOperator(
        task_id='deploy_custom_model',
        custom_model_version_id=create_custom_model_version_op.output,
        deployment_name=deployment_name,
        prediction_server_id=prediction_server_id,
        importance=DEPLOYMENT_IMPORTANCE.LOW,
    )

    custom_model_test_fail_case_op = EmptyOperator(
        task_id='custom_model_test_fail',
    )

    (
        create_execution_environment_op
        >> create_execution_environment_version_op
        >> create_custom_inference_model_op
        >> train_dataset_uploading_op
        >> create_custom_model_version_op
        >> test_dataset_uploading_op
        >> custom_model_test_op
        >> custom_model_test_overall_status_op
        >> branching
        >> [deploy_custom_model_op, custom_model_test_fail_case_op]
    )

create_custom_model_pipeline_dag = create_custom_model_pipeline()

if __name__ == "__main__":
    create_custom_model_pipeline_dag.test()


from re import A
from typing import List
from unittest import result

class Solution:
    def union_of_arr(self, arr1:List[int], arr2:List[int]):
        if len(arr1) == 0:
            result = set(arr1)
        elif len(arr2) == 0:
            result = set(arr2)
        else:
            result = set(arr1)
            result.update(set(arr2))
        return len(result)
        # result = -1
        # ans = 0
        # i, j = 0, 0
        # arr1.sort()
        # arr2.sort()
        # while i < len(arr1) and j < len(arr2):
        #     if arr1[i] == arr2[j]:
        #         if arr2[j] != result:
        #             ans += 1
        #             result = arr2[j]
        #         i += 1
        #         j += 1
        #     elif arr1[i] > arr2[j]:
        #         if arr2[j] != result:
        #             ans += 1
        #             result = arr2[j]
        #         j += 1
        #     else:
        #         if arr1[i] != result:
        #             ans += 1
        #             result = arr1[i]
        #         i += 1
        # for ele in range(i, len(arr1)):
        #     if arr1[ele] != result:
        #         ans += 1
        #         result = arr1[ele]

        # for ele in range(j, len(arr2)):
        #     if arr2[ele] != result:
        #         ans += 1
        #         result = arr2[ele]
        return ans

sol = Solution()
arr1, arr2 = [46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73], [36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60]
print(sol.union_of_arr(arr1, arr2))
# arr1 = [int(i) for i in input().split()]
# arr2 = [int(i) for i in input().split()]

# print(arr1)
# print(arr2)

DSASheet/Array/1_/6__Union of two arrays __.py


import tensorflow as tf
from tensorflow import keras

import numpy as np
import matplotlib.pyplot as plt

print(tf.__version__)

fashion_mnist = keras.datasets.fashion_mnist
(train_images, train_labels), (test_images,
                               test_labels) = fashion_mnist.load_data()

class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',
               'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']

# モデルを学習する前にデータセットのフォーマットを調べてみましょう。次の図は、トレーニングセットに6万枚の画像があり、各画像は28 x 28ピクセルで表されることを示しています。
train_images.shape

# 同様に、トレーニングセットには60,000のラベルがあります。
print(len(train_labels))
# train_labelsとtrain_imagesの要素数は同じです
print(len(train_images))

# 各ラベルは0から9までの整数です。
print(train_labels)

# テストセットには10,000枚の画像があります。繰り返しますが、各画像は28 x 28ピクセルとして表されます。
print(test_images.shape)

# テストセットには10,000の画像ラベルが含まれています。
print(len(test_labels))

# データはネットワークをトレーニングする前に前処理する必要があります。トレーニングセットの最初の画像を調べると、ピクセル値が0から255の範囲にあることがわかります。
# plt.figure()
# plt.imshow(train_images[0])
# plt.colorbar()
# plt.grid(False)
# plt.show()

# ニューラルネットワークモデルにフィードする前に、これらの値を0から1の範囲にスケーリングします。これには、画像コンポーネントのデータ型を整数からfloatにキャストし、255で割ります。これは、画像を前処理する関数です。 トレーニングセットとテストセットを同じ方法で前処理することが重要です。
train_images = train_images / 255.0
test_images = test_images / 255.0

# トレーニングセットの最初の25枚の画像を表示し、各画像の下にクラス名を表示します。データが正しい形式であることを確認してください。これでネットワークの構築とトレーニングの準備が整いました。
# plt.figure(figsize=(10,10))
# for i in range(25):
#    plt.subplot(5,5,i+1)
#    plt.xticks([])
#    plt.yticks([])
#    plt.grid(False)
#    plt.imshow(train_images[i], cmap=plt.cm.binary)
#    plt.xlabel(class_names[train_labels[i]])
# plt.show()

model = keras.Sequential([
    keras.layers.Flatten(input_shape=(28, 28)),
    keras.layers.Dense(128, activation=tf.nn.relu),
    keras.layers.Dense(10, activation=tf.nn.softmax)
])

model.compile(optimizer=tf.train.AdamOptimizer(),
              loss='sparse_categorical_crossentropy',
              metrics=['accuracy'])

model.fit(train_images, train_labels, epochs=5)

test_loss, test_acc = model.evaluate(test_images, test_labels)
print('Test accuracy:', test_acc)

predictions = model.predict(test_images)
print(predictions[0])
print(np.argmax(predictions[0]))
print(test_labels[0])

def plot_image(i, predictions_array, true_label, img):
    predictions_array, true_label, img = predictions_array[i], true_label[i], img[i]
    plt.grid(False)
    plt.xticks([])
    plt.yticks([])

    plt.imshow(img, cmap=plt.cm.binary)

    predicted_label = np.argmax(predictions_array)
    if predicted_label == true_label:
        color = 'blue'
    else:
        color = 'red'

    plt.xlabel("{} {:2.0f}% ({})".format(class_names[predicted_label],
                                         100*np.max(predictions_array),
                                         class_names[true_label]),
               color=color)

def plot_value_array(i, predictions_array, true_label):
    predictions_array, true_label = predictions_array[i], true_label[i]
    plt.grid(False)
    plt.xticks([])
    plt.yticks([])
    thisplot = plt.bar(range(10), predictions_array, color="#777777")
    plt.ylim([0, 1])
    predicted_label = np.argmax(predictions_array)

    thisplot[predicted_label].set_color('red')
    thisplot[true_label].set_color('blue')

#i = 12
# plt.figure(figsize=(6,3))
# plt.subplot(1,2,1)
#plot_image(i, predictions, test_labels, test_images)
# plt.subplot(1,2,2)
#plot_value_array(i, predictions,  test_labels)
# plt.show()

#num_rows = 5
#num_cols = 3
#num_images = num_rows*num_cols
#plt.figure(figsize=(2*2*num_cols, 2*num_rows))
# for i in range(num_images):
#    plt.subplot(num_rows, 2*num_cols, 2*i+1)
#    plot_image(i, predictions, test_labels, test_images)
#    plt.subplot(num_rows, 2*num_cols, 2*i+2)
#    plot_value_array(i, predictions, test_labels)
# plt.show()

img = test_images[0]
print(img.shape)

img = (np.expand_dims(img, 0))
print(img.shape)

predictions_single = model.predict(img)
print(predictions_single)

plot_value_array(0, predictions_single, test_labels)
_ = plt.xticks(range(10), class_names, rotation=45)

plt.show()


# coding=utf-8

import os
import sys

from dotenv import load_dotenv

BASEDIR = os.path.abspath(
    os.path.dirname(os.path.dirname(__file__))
)

dotenv_path = os.path.join(BASEDIR, '.env')
if os.path.exists(dotenv_path):
    load_dotenv(dotenv_path, override=True)

# SQLite URI compatible
WIN = sys.platform.startswith("win")
if WIN:
    prefix = 'sqlite:///'
else:
    prefix = 'sqlite:////'

def os_getenv2bool(key, default='true'):
    if not isinstance(default, str):
        default = str(default)
    return os.getenv(key, default).lower() in ('1', 'yes', 'true')

class BaseConfig(object):
    SECRET_KEY = os.getenv('SECRET_KEY') or \
        'hard to guess string and longer than 32 byte!'

    @classmethod
    def init_app(cls, app):
        pass

class TestingConfig(BaseConfig):
    TESTING = True
    WTF_CSRF_ENABLED = False
    SQLALCHEMY_DATABASE_URI = 'sqlite:///'  # in-memory database

class DevelopmentConfig(BaseConfig):
    DEBUG = True
    SQLALCHEMY_ECHO = False
    SQLALCHEMY_TRACK_MODIFICATIONS = False
    SQLALCHEMY_DATABASE_URI = os.getenv(
        'DATABASE_URL',
        prefix + os.path.join(BASEDIR, 'data-dev.db')
    )

class ProductionConfig(BaseConfig):
    SQLALCHEMY_TRACK_MODIFICATIONS = False
    SQLALCHEMY_DATABASE_URI = os.getenv(
        'DATABASE_URL',
        prefix + os.path.join(BASEDIR, 'data.db')
    )

config = {
    'development': DevelopmentConfig,
    'testing': TestingConfig,
    'production': ProductionConfig,
}


#! python3
# multiplicationTable.py - Takes a number N from the command line and
#                          creates an N×N multiplication table in an
#                          Excel spreadsheet.
# Adam Pellot

import os
import openpyxl
from openpyxl.styles import Font
import sys

os.chdir('C:\\Users\\Adam\\Documents')

# Declares number n from command line arguments.
inputedNum = sys.argv[1]
n = int(inputedNum)

# Create new workbook.
wb = openpyxl.Workbook()
sheet = wb['Sheet']

# Setup bold font.
boldFont = Font(bold=True)

# Setup the rows of the table.
rowNum = 2
columnNum = 1
for i in range(columnNum, n + 1, 1):
    sheet.cell(row=rowNum, column=columnNum).value = i
    sheet.cell(row=rowNum, column=columnNum).font = boldFont
    rowNum += 1

# Setup the columns of the table.
rowNum = 1
columnNum = 2
for i in range(rowNum, n + 1, 1):
    sheet.cell(row=rowNum, column=columnNum).value = i
    sheet.cell(row=rowNum, column=columnNum).font = boldFont
    columnNum += 1

# Fill table.
rowNum = 2
rowValue = sheet['A2'].value
for i in range(2, n + 2, 1):
    columnNum = 2
    columnValue = sheet['B1'].value
    for i in range(2, n + 2, 1):
        sheet.cell(row=rowNum, column=columnNum).value = rowValue * columnValue
        columnNum += 1
        columnValue += 1
    rowNum += 1
    rowValue += 1

wb.save('multiplicationTableFor' + inputedNum + '.xlsx')


from os import path
from logging import getLogger
from glob import glob
# from dbinteraction.configDB.fault_device_type import Fault_Device_Type as FDT
from dbinteraction.configDB.link_node_fault import Link_Node_Fault as LNF
from dbinteraction.configDB.epics_fault import Epics_Fault as EF
from dbinteraction.configDB.link_node_channel_fault import Link_Node_Channel_Fault as LNCF
from dbinteraction.configDB.link_processor_fault import Link_Processor_Fault as LPF
from dbinteraction.configDB.zlinknode import ZlinkNode as zln
from dbinteraction.mps_config import MPSConfig
from models.prepped_fault import PreppedFault

class ALLFaultsModel:
    """
    author: Evren Keskin
    ===================================================================
    ConfigDB accesses the config database to get fault data
    the fault data also allows for the related PV to be generated to access fault data
    ===================================================================
    """
    def __init__(self, accel_type: str, filename=None):
        """
        Establish logger and establish connection to mps_database.
        1) connect to the db file
        2) create dictionary of prepped faults
        """
        logger = getLogger(__name__)

        if filename and path.exists(filename):
            self.filename = filename
        else:
            if filename:
                logger.error("File does not exist. Using default .db file.")
            self.filename = self.set_filename(accel_type)

        self.configurator = MPSConfig(self.filename)

        self.set_all_node_fault_attributes()

    def set_filename(self, accel_type):
        """Finds default database filename."""
        if accel_type == 'LCLS':
            return self.set_LCLS_filename()
        else:  # FACET
            return self.set_FACET_filename()

    def set_LCLS_filename(self):
        print('redefining path to default LCLS faults file')
        phys_top = path.expandvars("$EPICS_IOC_TOP")
        versionFile = phys_top + "/MpsConfiguration/current/database/current"
        with open(versionFile) as f:
            first_line = f.readline()

        first_line = first_line.strip()
        filename = glob(phys_top + f'/MpsConfiguration/current/database/{first_line}/mpsdb.sqlite3')[0]
        return filename

    def set_FACET_filename(self):
        print('redefining path to default FACET faults file')
        phys_top = path.expandvars("$EPICS_IOC_TOP")
        versionFile = phys_top + "/MpsConfiguration-FACET/current/database/current"
        with open(versionFile) as f:
            first_line = f.readline()

        first_line = first_line.strip()
        filename = glob(phys_top + f'/MpsConfiguration-FACET/current/database/{first_line}/mpsdb.sqlite3')[0]
        return filename

    def set_all_node_fault_attributes(self):
        """
        Set attributes for all faults in one function
        Each query sets specific attributes alongside common ones, adding to the final list
        """
        query_results = []
        with self.configurator.Session() as session:
            ef_query_results = session.query(EF.area, EF.description, EF.device_name,
                                             EF.fault_name, EF.fault_number, EF.faulted_state_name,
                                             EF.is_autorecoverable, EF.ok_state_name, EF.pv_area,
                                             EF.pv_attribute, EF.pv_device_type, EF.pv_position,
                                             EF.input_pv, EF.position_x,
                                             EF.position_y, EF.position_z).all()

        for ef in ef_query_results:
            fault = PreppedFault()
            fault.device_name = ef.device_name
            fault.fault_name = ef.fault_name
            fault.fault_number = ef.fault_number
            fault.pv = (f"{ef.pv_device_type}:{ef.pv_area}:{ef.pv_position}:{ef.pv_attribute}")
            fault.ok_state_name = ef.ok_state_name
            fault.faulted_state_name = ef.faulted_state_name
            fault.is_autorecoverable = ef.is_autorecoverable
            fault.description = ef.description
            fault.area = ef.area
            fault.input_pv = ef.input_pv
            fault.position_x = ef.position_x
            fault.position_y = ef.position_y
            fault.position_z = ef.position_z
            fault.fault_type = EF

            query_results.append(fault)

        with self.configurator.Session() as session:
            lnf_query_results = session.query(LNF.area, LNF.description, LNF.device_name,
                                              LNF.fault_name, LNF.fault_number, LNF.faulted_state_name,
                                              LNF.is_autorecoverable, LNF.ok_state_name, LNF.pv_area,
                                              LNF.pv_attribute, LNF.pv_device_type, LNF.pv_position,
                                              LNF.link_node_id, zln.ZHOSTNAME).where(
                LNF.link_node_id == zln.ZLINKNODEID).all()

        for lnf in lnf_query_results:
            fault = PreppedFault()
            fault.device_name = lnf.device_name
            fault.fault_name = lnf.fault_name
            fault.fault_number = lnf.fault_number
            fault.pv = (f"{lnf.pv_device_type}:{lnf.pv_area}:{lnf.pv_position}:{lnf.pv_attribute}_LN{lnf.link_node_id}")
            fault.ok_state_name = lnf.ok_state_name
            fault.faulted_state_name = lnf.faulted_state_name
            fault.is_autorecoverable = lnf.is_autorecoverable
            fault.description = lnf.description
            fault.area = lnf.area
            fault.link_node_id = lnf.link_node_id
            fault.hostname = lnf.ZHOSTNAME
            fault.fault_type = LNF

            query_results.append(fault)

        with self.configurator.Session() as session:
            lncf_query_results = session.query(LNCF.area, LNCF.description, LNCF.device_name,
                                               LNCF.fault_name, LNCF.fault_number,
                                               LNCF.faulted_state_name,
                                               LNCF.is_autorecoverable, LNCF.ok_state_name, LNCF.pv_area,
                                               LNCF.pv_attribute, LNCF.pv_device_type, LNCF.pv_position,
                                               LNCF.cable, LNCF.card, LNCF.channel,
                                               LNCF.debounce_time, LNCF.is_deadman, LNCF.link_node_id,
                                               LNCF.position_x, LNCF.position_y, LNCF.position_z).all()

        for lncf in lncf_query_results:
            fault = PreppedFault()
            fault.device_name = lncf.device_name
            fault.fault_name = lncf.fault_name
            fault.fault_number = lncf.fault_number
            fault.pv = (f"{lncf.pv_device_type}:{lncf.pv_area}:{lncf.pv_position}:{lncf.pv_attribute}")
            fault.ok_state_name = lncf.ok_state_name
            fault.faulted_state_name = lncf.faulted_state_name
            fault.is_autorecoverable = lncf.is_autorecoverable
            fault.description = lncf.description
            fault.area = lncf.area
            fault.cable = lncf.cable
            fault.card = lncf.card
            fault.channel = lncf.channel
            fault.debounce_time = lncf.debounce_time
            fault.is_deadman = lncf.is_deadman
            fault.link_node_id = lncf.link_node_id
            fault.position_x = lncf.position_x
            fault.position_y = lncf.position_y
            fault.position_z = lncf.position_z
            fault.fault_type = LNCF

            query_results.append(fault)

        with self.configurator.Session() as session:
            lpf_query_results = session.query(LPF.area, LPF.description, LPF.device_name,
                                              LPF.fault_name, LPF.fault_number, LPF.faulted_state_name,
                                              LPF.is_autorecoverable, LPF.ok_state_name, LPF.pv_area,
                                              LPF.pv_attribute, LPF.pv_device_type, LPF.pv_position,
                                              LPF.position_x, LPF.position_y, LPF.position_z).all()

        for lpf in lpf_query_results:
            fault = PreppedFault()
            fault.device_name = lpf.device_name
            fault.fault_name = lpf.fault_name
            fault.fault_number = lpf.fault_number
            fault.pv = (f"{lpf.pv_device_type}:{lpf.pv_area}:{lpf.pv_position}:{lpf.pv_attribute}")
            fault.ok_state_name = lpf.ok_state_name
            fault.faulted_state_name = lpf.faulted_state_name
            fault.is_autorecoverable = lpf.is_autorecoverable
            fault.description = lpf.description
            fault.area = lpf.area
            fault.position_x = lpf.position_x
            fault.position_y = lpf.position_y
            fault.position_z = lpf.position_z
            fault.fault_type = LPF

            query_results.append(fault)

        self.nums_to_faults = {fault.fault_number: fault for fault in query_results}

    def get_fault_by_num(self, num=None):
        """
        Get a fault from the fault list using its fault number
        """
        if self.nums_to_faults is None:
            return None
        return self.nums_to_faults[num]

    def get_fault_by_signal(self, signalName=None):
        """
        Get a fault from the fault list using its 'signal name', a combination of the device and fault names
        """
        for fault_num in self.nums_to_faults:
            if (signalName == self.nums_to_faults[fault_num].device_name
                    + '_' + self.nums_to_faults[fault_num].fault_name):
                return self.nums_to_faults[fault_num]

        return None