from typing import Union

from django.db import transaction
from django.http import HttpResponse
from django.core.exceptions import ObjectDoesNotExist

from rest_framework import views
from rest_framework import generics, status
from rest_framework.response import Response

from app.api import serializer
from app import models
from app.repositories import avg_debt, ValidationCheck
from app.tasks import send_email

class CompanyView(generics.ListAPIView):
    """
    Информация обо все объектах сети, где есть авторизированный сотрудник
    """
    serializer_class = serializer.CompanySerializer
    queryset = models.Company.objects

    def get(self, request, *args, **kwargs) -> Response:
        queryset = self.queryset.filter(company_staff__staff=request.user.pk)
        serializers = serializer.CompanySerializer(queryset, many=True)
        return Response(serializers.data)

class CountyCompanyView(generics.ListAPIView):
    """
    Информацию об объектах определённой страны(фильтр по названию страны)
    """
    serializer_class = serializer.CompanySerializer
    queryset = models.Company.objects

    def get(self, request, country: str, *args, **kwargs) -> Response:
        queryset = self.queryset.filter(company_staff__staff=request.user.pk, country=country.lower())
        serializers = serializer.CompanySerializer(queryset, many=True)
        return Response(serializers.data)

class CreateCompanyView(generics.CreateAPIView):
    """
    Создать новую запить об объекте сети
    """
    serializer_class = serializer.CreateCompanySerializer

    def post(self, request, *args, **kwargs) -> Union[Response, HttpResponse]:
        company = models.Company()

        try:
            type_company = models.TypeCompany.objects.get(pk=request.data.get('type_company'))
        except ObjectDoesNotExist:
            return HttpResponse('Такого типа компании не существует')

        try:
            supplier = models.Company.objects.get(pk=request.data.get('supplier'))
        except ObjectDoesNotExist:
            return HttpResponse('Такого поставщика не существует')

        company.name = request.data.get('name', None)
        company.type_company = type_company
        company.email = request.data.get('email', None)
        company.country = request.data.get('country', None)
        company.city = request.data.get('city', None)
        company.street = request.data.get('street', None)
        company.house_number = request.data.get('house_number', None)
        company.supplier = supplier
        company.save()
        return Response(status=status.HTTP_201_CREATED)

class RemoveCompanyView(generics.DestroyAPIView):

    def delete(self, request, pk: int, *args, **kwargs) -> Union[Response, ObjectDoesNotExist]:
        try:
            company = models.Company.objects.get(pk=pk, company_staff__staff=request.user.pk)
            company.delete()
            return Response(status=status.HTTP_200_OK)
        except ObjectDoesNotExist:
            return Response(status=status.HTTP_404_NOT_FOUND)

class CreateProductsView(generics.CreateAPIView):
    """
    Создать новую запить об продукте
    """
    serializer_class = serializer.ProductSerializer

    def post(self, request, *args, **kwargs) -> Response:
        products = models.Products()

        products.name = request.data.get('name', None)
        products.model = request.data.get('model', None)
        products.release_data = request.data.get('release_data', None)
        products.save()
        return Response(status=status.HTTP_201_CREATED)

class RemoveProductsView(generics.DestroyAPIView):
    """
    Удалить запись продукта
    """

    def delete(self, request, pk: int, *args, **kwargs) -> Union[Response, ObjectDoesNotExist]:
        try:
            company = models.Products.objects.get(pk=pk)
            company.delete()
            return Response(status=status.HTTP_200_OK)
        except ObjectDoesNotExist:
            return Response(status=status.HTTP_404_NOT_FOUND)

class FindCompanyWithProductView(generics.ListAPIView):
    """
    Все объекты сети, где можно встретить определённый продукт (фильтр по id продукта)
    """

    serializer_class = serializer.CompanySerializer
    queryset = models.Company.objects

    def get(self, request, pk: int, *args, **kwargs) -> Response:
        queryset = self.queryset.filter(company_staff__staff=request.user.pk, products_company__product=pk)
        serializers = serializer.CompanySerializer(queryset, many=True)
        return Response(serializers.data)

class MoreAvgDebtView(generics.ListAPIView):
    """
    Выводит среднею задолженность по объектам сети и сами объекты, где задолженность превышает среднею
    """
    serializer_class = serializer.DebtMoreAvgSerializer
    queryset = models.Company.objects

    def get(self, request, *args, **kwargs) -> Response:
        static = {'avg_debt': avg_debt(), 'company': None}
        queryset = self.queryset.filter(company_staff__staff=request.user.pk, debt__gt=static['avg_debt'])
        static['company'] = serializer.CompanySerializer(queryset, many=True).data
        return Response(static, status=status.HTTP_200_OK)

class UpdateProductView(views.APIView):
    """
    Обновление информации об продукте
    """
    serializer_class = serializer.ProductSerializer

    def patch(self, request, pk: int, *args, **kwargs) -> Union[Response, HttpResponse, ObjectDoesNotExist]:
        with transaction.atomic():
            try:
                product = models.Products.objects.select_for_update().get(pk=pk)
            except ObjectDoesNotExist:
                return Response(status=status.HTTP_404_NOT_FOUND)

            name: str | None = request.data.get('name', None)
            error_name = ValidationCheck.valid_name(name, 25)
            if error_name:
                return HttpResponse(error_name)

            release_data: str | None = request.data.get('release_data', None)
            error_date = ValidationCheck.date(release_data)
            if error_date:
                return HttpResponse(error_date)

            product.name = name
            product.model = request.data.get('model', None)
            product.release_data = release_data
            product.save()
        return Response(status=status.HTTP_200_OK)

class UpdateCompanyView(views.APIView):
    """
    Обновление информации об объекте сети
    """
    serializer_class = serializer.UpdateCompanySerializer

    def patch(self, request, pk: int, *args, **kwargs) -> Union[Response, HttpResponse, ObjectDoesNotExist]:
        with transaction.atomic():
            try:
                company = models.Company.objects.select_for_update().get(pk=pk, company_staff__staff=request.user.pk)
            except ObjectDoesNotExist:
                return Response(status=status.HTTP_404_NOT_FOUND)

            try:
                type_company = models.TypeCompany.objects.get(pk=request.data.get('type_company'))
            except ObjectDoesNotExist:
                return HttpResponse('Такого типа компании не существует')

            try:
                supplier = models.Company.objects.get(pk=request.data.get('supplier'))
            except ObjectDoesNotExist:
                return HttpResponse('Такого поставщика не существует')

            name: str | None = request.data.get('name', None)
            error = ValidationCheck.valid_name(name, 50)
            if error:
                return HttpResponse(error)

            company.name = name
            company.email = request.data.get('email', None)
            company.type_company = type_company
            company.country = request.data.get('country', None)
            company.city = request.data.get('city', None)
            company.street = request.data.get('street', None)
            company.house_number = request.data.get('house_number', None)
            company.supplier = supplier
            company.save()
        return Response(status=status.HTTP_200_OK)

class QrCodeAboutCompanyView(generics.ListAPIView):
    """
    Отправка контактной информации на почту пользователя в виде QRCode
    """
    queryset = models.Company.objects

    def get(self, request, pk: int, *args, **kwargs) -> Union[Response,HttpResponse, ObjectDoesNotExist]:

        try:
            company = models.Company.objects.get(pk=pk)
        except ObjectDoesNotExist:
            return Response(status=status.HTTP_404_NOT_FOUND)

        user_email = request.user.email
        if not user_email:
            return HttpResponse('404: У пользователя отсутствует email для отправки данных')

        name_company = f'Компания {company.name}'
        info_company = f'{name_company}\nEmail: {company.email}\nСтрана:{company.country.title()}, ' \
                       f'Город: {company.city.title()}, Улица: {company.street.title()}, Дом: {company.house_number}'
        send_email.apply_async(
            args=[
                name_company,
                info_company,
                user_email,
            ],
            serializer='json',
        )
        return Response(status=status.HTTP_200_OK)


import re
import scrapy
import asyncio
from playwright.async_api import async_playwright
from mi.items import HiLabMIItem
from mi.spiders.hiaas_common import HiaasCommon
from scrapy.utils.project import get_project_settings

class CoupangCombineSpider(HiaasCommon):
    name = "coupang_combine"
    start_urls = ["data:,"]  # avoid using the default Scrapy downloader
    custom_settings = {
        'TWISTED_REACTOR' : "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
        # 'ITEM_PIPELINES' : None
    }

    marketType = "coupang"

    async def parse(self, response):

        settings = get_project_settings()
        KEYWORD_LIST = settings.get('KEYWORD_LIST')
        COUPANG_CRAWL_DELAY = settings.get('COUPANG_CRAWL_DELAY')
        COUPANG_LISTSIZE = settings.get('COUPANG_LISTSIZE')
        COUPANG_PAGE_COUNT = settings.get('COUPANG_PAGE_COUNT')
        COUPANG_SORTER = settings.get('COUPANG_SORTER')

        async with async_playwright() as pw:
            browser = await pw.firefox.launch()
            page = await browser.new_page()

            for keyword in KEYWORD_LIST:
                for pagenum in range(1, COUPANG_PAGE_COUNT):
                    search_link = f'https://www.coupang.com/np/search?rocketAll=false&q={keyword}&brand=&offerCondition=&filter=&availableDeliveryFilter=&filterType=&isPriceRange=false&priceRange=&minPrice=&maxPrice=&page={pagenum}&trcid=&traid=&filterSetByUser=true&channel=recent&backgroundColor=&searchProductCount=1857217&component=&rating=0&sorter={COUPANG_SORTER}&listSize={COUPANG_LISTSIZE}'
                    await page.goto(search_link)
                    await asyncio.sleep(COUPANG_CRAWL_DELAY)

                    locators = page.locator('ul.search-product-list> li > a.search-product-link')
                    count = await locators.count()

                    for i in range(count):
                        locator = locators.nth(i)
                        href = await locator.get_attribute('href')
                        detail_link = 'https://www.coupang.com' + href
                        if 'sourceType=srp_product_ads' in href:
                            ad = 'ad'
                        else:
                            ad = None

                        yield scrapy.Request(url=detail_link, callback=self.parse_detail, meta=dict(
                            ad = ad,
                            sk = keyword
                        ))

            await browser.close()

    def parse_detail(self, response):
        try:

            item = HiLabMIItem()

            # 마켓 타입 (ex. naver, coupang)
            item['mid'] = self.marketType

            # collection type (1:키워드, 2:카테고리)
            item['ctype'] = 1

            # 상세페이지 링크
            item['detail_link'] = response.url

            # 순위
            item['rank'] = int(response.url.split('&rank=')[1])

            # 광고 여부
            item['ad'] = response.meta.get('ad')

            ## 카테고리 -> javascript
            # item['pr1ca'] = response.css('ul#breadcrumb a::text').getall()
            #for category in response.css('#breadcrumb > li'):
                #item['pr2ca'] = category.css('a::text').getall()

            ## 정렬기준
            # if "scoreDesc" in url_page:
            #     item['sb'] = "쿠팡 랭킹순"
            # elif "salePriceAsc" in url_page:
            #     item['sb'] = "낮은가격순"
            # elif "salePriceDesc" in url_page:
            #     item['sb'] = "높은가격순"
            # elif "saleCountDesc" in url_page:
            #     item['sb'] = "판매량순"
            # elif "latestAsc" in url_page:
            #     item['sb'] = "최신순"

            ## 총 제품 수: 해당없음
            #item['prco']

            # 제품명
            pr1nm = response.css('h2.prod-buy-header__title::text').get()
            item['pr1nm'] = pr1nm

            # 가격
            pr1pr = response.css('span.total-price > strong::text').get()
            if pr1pr is not None:
                item['pr1pr'] = int(pr1pr.replace(',', ''))
            else:
                item['pr1pr'] = pr1pr

            ## 판매자
            #item['ta'] = response.xpath('//*[@id="btfTab"]/ul[2]/li[4]/div/table/tbody/tr[1]/td[1]').get()    # 판매자(상호/대표자)(?) - javascript
            #if response.xpath('//*[@id="btfTab"]/ul[2]/li[4]/div/table/tbody/tr[1]/td[1]').get() is None:
            #    item['ta'] = response.xpath('//*[@id="btfTab"]/ul[2]/li[4]/div/table/tbody/tr/td/text()')    # 판매자(쿠팡)(?) - javascript
            ##//*[@id="btfTab"]/ul[2]/li[4]/div/table/tbody/tr[1]/td[1] -> '상호/대표자'가 있을 때 xpath
            ##//*[@id="btfTab"]/ul[2]/li[4]/div/table/tbody/tr/td/text() -> '쿠팡'일 때 xpath

            # 평점
            gr = re.sub(r'[^0-9.]', '', str(response.css('span.rating-star-num::attr(style)').get()))
            item['gr'] = float(gr) / 100 * 5

            # 리뷰개수
            item['revco'] = int(re.sub(r'[^0-9]', '', str(response.css('span.count::text').get())))

            ## 할인정보
            ##item['dcinfo'] = response.css('span.discount-rate::text').get().replace("\n","").replace(" ","").replace("%","")
            ##if item['dcinfo'] != "":
            ##    item['dcinfo'] = response.css('span.discount-rate::text').get().replace("\n","").replace(" ","")
            #******************************************************************************************************************************
            # dcinfo = response.css('span.discount-rate::text').get()   # 할인정보: 할인율과 가져오는 정보가 동일하여 일단은 할인율을 가져오고 할인정보는 수집 보류
            # if dcinfo is None:
            #     item['dcinfo'] = None
            # elif dcinfo.strip() == "%":
            #     item['dcinfo'] = None
            # else:
            #     item['dcinfo'] = dcinfo.strip()

            # 무료배송 유무
            ts = response.css('div.prod-shipping-fee-message > span > em.prod-txt-bold::text').get()
            item['ts'] = ts == "무료배송"

            # 도착 예정일자
            # 로켓 아니고, 도착예정일자 전화/문자로 안내
            ardate = response.css('em.prod-txt-onyx.prod-txt-bold::text').get()
            if ardate is None:
                # 로켓 아닐 때 도착예정일자
                ardate = response.css('em.prod-txt-onyx.prod-txt-font-14::text').get()
                if ardate is None:
                    # 로켓 아닐 때 도착예정일자2
                    ardate = response.css('em.prod-txt-onyx::text').get()
                    if ardate is None:
                        # 로켓일 때 도착예정일자
                        ardate = response.css('em.prod-txt-onyx.prod-txt-green-2::text').get()
                        re_pattern = re.compile(r'\d+')
                        ardate = '/'.join(re.findall(re_pattern, str(ardate)))
                        item['ardate'] = ardate
                    else:
                        re_pattern = re.compile(r'\d+')
                        ardate = '/'.join(re.findall(re_pattern, str(ardate)))
                        item['ardate'] = ardate
                else:
                    re_pattern = re.compile(r'\d+')
                    ardate = '/'.join(re.findall(re_pattern, str(ardate)))
                    item['ardate'] = ardate
            else:
                item['ardate'] = ardate

            # 멤버십 적용유무
            ms_list = []
            # 로켓배송
            if response.css('img.delivery-badge-img::attr(src)').get() == "//image10.coupangcdn.com/image/badges/rocket/rocket_logo.png":
                ms_list.append("로켓배송")
            # 로켓 설치
            if response.css('img.delivery-badge-img.rocket-install-img::attr(src)').get() == "//image7.coupangcdn.com/image/badges/rocket-install/v3/aos_2/rocket_install_xhdpi.png":
                ms_list.append("로켓설치")
            if response.css('span.ccid-txt::text').get() is not None:
                # 카드할인
                ms_list.append(response.css('span.ccid-txt::text').get())
            if response.css('span.reward-cash-txt::text').get() is not None:
                # 캐시적립
                ms_list.append(response.css('span.reward-cash-txt::text').get().replace(' ', '').strip().replace('\n\n', ' '))
            item['ms'] = ms_list

            # 다른 구매옵션
            item['opo'] = response.css('span.prod-offer-banner-item::text').get()
            opo_list = []
            if response.css('span.prod-offer-banner-item::text').get() != None:
                tagIdx = len(response.css('span.prod-offer-banner-item'))
                for i in range(0,tagIdx):
                    opo = []
                    opo.append(response.css('span.prod-offer-banner-item::text')[i].get())
                    opo.append(response.css('span.prod-offer-banner-item__count::text')[i].get())
                    opo_str = "".join(opo)
                    opo_list.append(opo_str)
                item['opo'] = "/".join(opo_list)

            ## 구매횟수: 해당없음
            ##item['purchco']

            # 재고 현황
            if response.css('div.aos-label::text').get() is None:
                item['pr1qt'] = None
            else:
                pr1qt = response.css('div.aos-label::text').get()
                item['pr1qt'] = int(re.sub(r'[^0-9]', '', str(pr1qt)))

            ## 전체 페이지 수
            # item['pgco'] = page

    # ---------------------------------------------------------------------------------------------------------------------------------

            # 검색 키워드
            item['sk'] = response.meta.get('sk')

    # ---------------------------------------------------------------------------------------------------------------------------------

            # 브랜드(O)
            # item['pr1br'] = response.css('a.prod-brand-name::text').get()
            item['pr1br'] = pr1nm.split()[0]    # 제품명의 앞 한 단어(임시)

            # 브랜드샵link
            #item['brlk'] = response.css('a.prod-brand-name::attr(href)').get()

            # 셀러샵link
            #item['talk']

            # SKU
            #item['pr1id'] = response.css('#itemBrief > div > table > tbody > tr:nth-child(1) > td:nth-child(2)::text').get()
            #item['pr1id'] = response.xpath('//*[@id="contents"]/div[1]/div/div[3]/div[10]/div/div/div/button/table/tbody/tr/td[1]/span[2]/@text').get()  # SKU
            #contents > div.prod-atf > div > div.prod-buy.new-oos-style.not-loyalty-member.eligible-address.without-subscribe-buy-type.DISPLAY_0.has-loyalty-exclusive-price > div.prod-option > div:nth-child(2) > div > div > button > table > tbody > tr > td:nth-child(1) > span.value
            #//*[@id="contents"]/div[1]/div/div[3]/div[10]/div[2]/div/div/button/table/tbody/tr/td[1]/span[2]

            # 할인율
            dcrate = response.css('span.discount-rate::text').get()
            if dcrate is None:
                item['dcrate'] = None
            elif dcrate.strip() == "%":
                item['dcrate'] = None
            else:
                item['dcrate'] = int(dcrate.strip().replace('%', ''))

            # 정가
            ##item['fullpr'] = response.css('span.origin-price::text').get().replace("\n","").replace(" ","")    -> 값 안 나오는 거 있음
            #********************************************************************************************
            fullpr = response.css('span.origin-price::text').get()   # 정가(O)  # 가격 관련 아이템들은 다시 한번 살펴봐야 할 것
            if fullpr == "원":
                item['fullpr'] = int(response.css('span.total-price > strong::text').get().replace(',', ''))
            else:
                item['fullpr'] = int(fullpr.replace("원", "").replace(',', ''))

            # # 할인가: 와우할인가로 가져오기
            # # 쿠팡판매가, 와우할인가 나눠져 있을 때
            # ##item['dcpr'] = response.css('div.prod-sale-price.instant-discount > span.total-price > strong::text').get()   # 쿠팡판매가
            # item['dcpr'] = response.css('div.prod-coupon-price.prod-major-price > span.total-price > strong::text').get()   # 와우할인가
            # # item['dcpr'] = response.css('span.total-price > strong::text').getall() ## 쿠팡판매가, 와우판매가 2개 다 cawling
            # # 쿠팡판매가, 와우할인가 안 나눠져 있을 때
            # item['dcpr'] = response.css('prod-sale-price.prod-major-price > span.total-price > strong::text').get()
            # #위에 '가격' 크롤링 한 css로 똑같이 가져오면 쿠팡판매가, 와우할인가 나뉘어진 상품은 쿠팡판매가가 나옴.(75)
            # ##item['dcpr'] = response.css('span.total-price > strong::text').get()

            # 품절 유무
            soldout = response.css('div.oos-label::text').get()
            if soldout is not None:
                item['soldout'] = soldout.strip() == "일시품절"
            else:
                soldout = response.css('div.prod-not-find-known__buy__button').get()
                item['soldout'] = soldout == "품절"

            # 제품 구매 옵션
            #item['pr1va'] = response.css('div.prod-option__dropdown-item-title > strong::text').get()

            # 멤버십 혜택(ex. 카드혜택, 캐시적립혜택)
            item['msbf'] = response.css('strong.tit-txt::text').getall()

            # 제품 상세
            item['prdetail'] = response.css('li.prod-attr-item::text').getall()
            ##item['prdetail'] = response.css('ul.prod-description-attribute > li.prod-attr-item::text').get()
            ##prdetail_list=[]
            ###if response.css('ul.prod-description-attribute > li.prod-attr-item::text').get() != None:
            ##    tagIdx = len(response.css('ul.prod-description-attribute > li.prod-attr-item'))
            ##    for i in range(0,tagIdx):
            ##        prdetail_list.append(response.css('ul.prod-description-attribute > li.prod-attr-item::text')[i].get())
            ##    item['prdetail'] = "\n".join(prdetail_list)
            #***********************************************************************

    # ---------------------------------------------------------------------------------------------------------------------------------

            ## 리뷰 특징 요약
            #item['revsum']

            ##  정렬기준
            #item['revsb'] = response.css('button.sdp-review__article__order__sort__best-btn sdp-review__article__order__sort__btn--active js_reviewArticleHelpfulListBtn js_reviewArticleSortBtn::text').get()

            ## 작성자
            #item['reviewer'] = response.css('span.sdp-review__article__list__info__user__name js_reviewUserProfileImage::text').get()

            ## 평점
            #item['ingrade'] = response.css('div.sdp-review__article__list__info__product-info__star-orange js_reviewArticleRatingValue')

            ## 작성 일자
            #item['revdate'] = response.css('div.sdp-review__article__list__info__product-info__reg-date::text').get()

            ## 구매품목 디테일
            #item['purchdetail'] = response.css('div.sdp-review__article__list__info__product-info__name::text').get()

            ## 리뷰 디테일
            #item['revdetail'] = response.css('div.sdp-review__article__list__headline').get() + response.css('div.sdp-review__article__list__review__content js_reviewArticleContent').get()

            ## 블로그 리뷰: 해당없음
            #item['blogrev']

            ## 리뷰 조회수: 해당없음
            #item['revviews']
            yield item

        except Exception as e:
            print('e: ', e)
            # logger.error('Error [parse_detail]: %s', e)

    # def errback_httpbin(self, failure):
    #     # log all failures
    #     logger.error(repr(failure))

    #     # in case you want to do something special for some errors,
    #     # you may need the failure's type:

    #     if failure.check(HttpError):
    #         # these exceptions come from HttpError spider middleware
    #         # you can get the non-200 response
    #         response = failure.value.response
    #         logger.error('HttpError on %s', response.url)

    #     elif failure.check(DNSLookupError):
    #         # this is the original request
    #         request = failure.request
    #         logger.error('DNSLookupError on %s', request.url)

    #     elif failure.check(TimeoutError, TCPTimedOutError):
    #         request = failure.request
    #         logger.error('TimeoutError on %s', request.url)

    #         logger.error('TimeoutError on %s', request.url)

# ----------------------scrapy-playwright---------------------------
# import asyncio
# import scrapy

# from mi.spiders.hiaas_common import HiaasCommon
# from scrapy.utils.project import get_project_settings
# from scrapy_playwright.page import PageMethod

# class CoupangCombineSpider(HiaasCommon):
#     name = "coupang_combine"
#     # start_urls = ["data:,"]  # avoid using the default Scrapy downloader
#     custom_settings = {
#         'TWISTED_REACTOR' : "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
#         # 'ITEM_PIPELINES' : None,
#         'DOWNLOAD_HANDLERS' : {
#             "http": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler",
#             "https": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler",
#         }
#     }

#     def start_requests(self):
#         settings = get_project_settings()
#         KEYWORD_LIST = settings.get('KEYWORD_LIST')
#         COUPANG_PAGE_COUNT = settings.get('COUPANG_PAGE_COUNT')
#         COUPANG_SORTER = settings.get('COUPANG_SORTER')
#         COUPANG_LISTSIZE = settings.get('COUPANG_LISTSIZE')

#         for keyword in KEYWORD_LIST:
#             for pagenum in range(1, COUPANG_PAGE_COUNT):
#                 search_link = f'https://www.coupang.com/np/search?rocketAll=false&q={keyword}&brand=&offerCondition=&filter=&availableDeliveryFilter=&filterType=&isPriceRange=false&priceRange=&minPrice=&maxPrice=&page={pagenum}&trcid=&traid=&filterSetByUser=true&channel=recent&backgroundColor=&searchProductCount=1857217&component=&rating=0&sorter={COUPANG_SORTER}&listSize={COUPANG_LISTSIZE}'

#                 yield scrapy.Request(url=search_link, callback=self.parse_pagelink, meta=dict(
#                     playwright = True,
#                     playwright_include_page = True,
#                     errback=self.errback,
#                 ))

#     async def parse_pagelink(self, response):
#         settings = get_project_settings()
#         COUPANG_CRAWL_DELAY = settings.get('COUPANG_CRAWL_DELAY')

#         page = response.meta["playwright_page"]
#         # await asyncio.sleep(COUPANG_CRAWL_DELAY)
#         await page.close()

#         hrefs = response.css('ul.search-product-list > li > a.search-product-link::attr(href)').getall()

#         for href in hrefs:
#             detail_link = 'https://www.coupang.com' + href
#             yield {"detail_link": detail_link}

#     async def errback(self, failure):
#         page = failure.request.meta["playwright_page"]
#         await page.close()


import pandas as pd
from typing import Any, Optional
from typing import Union, List

class Melter:
    def __init__(
        self,
        id_vars: Union[str, List[str]],
        created_cell_col: str = "cell_id",
        created_value_col: str = "value",
        cast_cell_col_type: Optional[str] = None,
        drop_cols: Optional[List[str]] = None,
    ):
        self.id_vars = id_vars
        self.created_cell_col = created_cell_col
        self.created_value_col = created_value_col
        self.cast_cell_col_type = cast_cell_col_type
        self.drop_cols = drop_cols

    def _cast_cell_col(self, df: pd.DataFrame) -> pd.DataFrame:
        if self.cast_cell_col_type is not None:
            df[self.created_cell_col] = df[self.created_cell_col].astype(
                self.cast_cell_col_type
            )
        return df

    def __call__(self, df_traces: pd.DataFrame) -> pd.DataFrame:
        if self.drop_cols is not None:
            df_traces = df_traces.drop(columns=self.drop_cols)
        df_traces = pd.melt(
            df_traces,
            id_vars=self.id_vars,
            var_name=self.created_cell_col,
            value_name=self.created_value_col,
        )
        df_traces = self._cast_cell_col(df_traces)
        return df_traces


from django.conf import settings
from django.contrib import admin
from django.views import defaults as default_views
from django.urls import include, path, re_path
from django.conf.urls.static import static
from invoices.views import handler403, handler404, handler500
from authentication.views import (
    update_vertrieblers,
    update_elektrikers,
    protected_schema_view,
)

urlpatterns = [
    path("admin/", admin.site.urls),
    path(
        "adminfeautures/",
        include(("adminfeautures.urls", "adminfeautures"), namespace="adminfeautures"),
    ),
    path("update_vertrieblers/", update_vertrieblers, name="update_vertrieblers"),
    path("update_elektrikers/", update_elektrikers, name="update_elektrikers"),
    path("admin/schema/", protected_schema_view, name="schema_view"),
    path("/", include("djoser.urls")),
    path("", include("djoser.urls.jwt")),
    path("", include("authentication.urls", namespace="authentication")),
    path("", include("vertrieb_interface.urls", namespace="vertrieb_interface")),
    path(
        "proj/", include("projektant_interface.urls", namespace="projektant_interface")
    ),
    path("elektriker_interface/", include("invoices.urls", namespace="invoices")),
    path(
        "elektriker_kalender",
        include("elektriker_kalender.urls", namespace="elektriker_kalender"),
    ),
    re_path(
        r"^400/$",
        default_views.bad_request,
        kwargs={"exception": Exception("Bad Request!")},
    ),
    re_path(
        r"^403/$",
        default_views.permission_denied,
        kwargs={"exception": Exception("Permission Denied")},
    ),
    re_path(
        r"^404/$",
        default_views.page_not_found,
        kwargs={"exception": Exception("Page not Found")},
    ),
    re_path(r"^500/$", default_views.server_error),
]

if settings.DEBUG:
    urlpatterns += [
        re_path(r"^403/$", handler403),
        re_path(r"^404/$", handler404),
        re_path(r"^500/$", handler500),
    ]

urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)
urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)


# intercept logging msg to loguru
##################################
import logging
import sys

from loguru import logger

class InterceptHandler(logging.Handler):
    def emit(self, record):
        # Get corresponding Loguru level if it exists.
        try:
            level = logger.level(record.levelname).name
        except ValueError:
            level = record.levelno

        # Find caller from where originated the logged message.
        frame, depth = sys._getframe(6), 6
        while frame and frame.f_code.co_filename == logging.__file__:
            frame = frame.f_back
            depth += 1

        logger.opt(depth=depth, exception=record.exc_info).log(
            level, record.getMessage()
        )

logging.basicConfig(handlers=[InterceptHandler()], level=0, force=True)

# set log level for logging logger
logging_logger = logging.getLogger(__name__)
logging_logger.setLevel(logging.INFO)

logging_logger.info("This is a test message")
logging_logger.debug("This is a test message")

# alternatively set log level for loguru logger
logger.remove()
logger.add(sys.stderr, level="INFO")

logger.info("This is a test message")
logger.debug("This is a test message")

logging_logger.info("This is a test message")
logging_logger.debug("This is a test message")


import clean
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from datetime import date, datetime, timedelta
from scipy.stats import gamma

pd.set_option('display.max_rows', 100)

combined_deaths = 'combined_deaths'
date_col = 'Date'

def plot_posterior(a, b, week):
    # Take arbitrary X-axis range of (5, 20)
    x_points = np.linspace(5, 20, 500)
    # Generate PDF for gamma(a,b) for each x
    y_pdf = gamma.pdf(x_points, a=a, scale=b)

    map_val = round(x_points[y_pdf.argmax()], 3)
    print("MAP for week {0} Posterior Distribution of λ = {1}".format(week, map_val))
    plt.plot(x_points, y_pdf, label = 'Posterior: Week {0}, MAP = {1}'.format(week, map_val))

def calculate_posterior(daily_data):
    # Take a subset of only first 8 weeks of June-July 2020 data
    start_date, date_format = '2020-06-01', '%Y-%m-%d'
    end_date = (datetime.strptime(start_date, date_format) + timedelta(weeks=8)).strftime(date_format)
    bi_data = daily_data[(daily_data[date_col] >= start_date) & (daily_data[date_col] < end_date)].copy()
    # Compute combined deaths for the months of June-July
    bi_data[combined_deaths] = bi_data.loc[:, ['CT deaths', 'DC deaths']].sum(axis=1)

    # Given, combined deaths are assumed to be Poisson(lambda) distributed. MME for the Poisson parameter, lambda_mme, is the same as
    # the sample mean of the first 4 weeks' data
    training_days = 4 * 7
    training_end_date = (datetime.strptime(start_date, date_format) + timedelta(days=training_days)).strftime(date_format)
    lambda_mme = bi_data[bi_data[date_col] < training_end_date][combined_deaths].mean()
    # Given, mean of prior exponential = beta = lambda_mme
    beta = lambda_mme

    plt.figure('Posterior Distribution', figsize=(15,8))
    print("{0} 2d) Posterior Distributions of λ {0}".format(20*"-"))
    # Find posterior from weeks 5-8
    for week in range(5, 9):
        week_end_date = (datetime.strptime(start_date, date_format) + timedelta(weeks=week)).strftime(date_format)
        week_i_deaths = bi_data[bi_data[date_col] < week_end_date][combined_deaths]

        # As derived in the report, since both Poisson and Exponential distributions have Gamma distribution as their conjugate priors,
        # the posteriors too will be gamma distributed in the form of Gamma(a, b), where:
        # shape parameter a = sample sum sigma_x_i + 1, scale parameter b = 1 / (sample data size + (1/beta))
        sample_size = week_i_deaths.size
        a = week_i_deaths.sum() + 1
        b = 1 / (sample_size + (1/beta))
        plot_posterior(a, b, week)

    # Plot posterior distribution graph for weeks 5-8
    plt.title('Posterior Distribution of λ')
    plt.xlabel('X')
    plt.ylabel('λ ~ Gamma(a, b) PDF')
    plt.legend(loc='best')
    plt.show()


import tensorflow as tf

import tensorflow_datasets as tfds

from IPython.display import clear_output
import matplotlib.pyplot as plt

import os
import sys
import random

import numpy as np
import cv2

from tensorflow import keras

dataset, info = tfds.load('oxford_iiit_pet:3.*.*', with_info=True)

image_size = 128

def normalize(input_image, input_mask):
  input_image = tf.cast(input_image, tf.float32) / 255.0
  input_mask -= 1
  return input_image, input_mask

@tf.function
def load_image_train(datapoint):
  input_image = tf.image.resize(datapoint['image'], (image_size, image_size))
  input_mask = tf.image.resize(datapoint['segmentation_mask'], (image_size, image_size))

  if tf.random.uniform(()) > 0.5:
    input_image = tf.image.flip_left_right(input_image)
    input_mask = tf.image.flip_left_right(input_mask)

  input_image, input_mask = normalize(input_image, input_mask)

  return input_image, input_mask

def load_image_test(datapoint):
  input_image = tf.image.resize(datapoint['image'], (image_size, image_size))
  input_mask = tf.image.resize(datapoint['segmentation_mask'], (image_size, image_size))

  input_image, input_mask = normalize(input_image, input_mask)

  return input_image, input_mask

TRAIN_LENGTH = info.splits['train'].num_examples
BATCH_SIZE = 64
BUFFER_SIZE = 1000
STEPS_PER_EPOCH = TRAIN_LENGTH // BATCH_SIZE

print(TRAIN_LENGTH)
print(STEPS_PER_EPOCH)

train = dataset['train'].map(load_image_train, num_parallel_calls=tf.data.experimental.AUTOTUNE)
test = dataset['test'].map(load_image_test)

train_dataset = train.cache().shuffle(BUFFER_SIZE).batch(BATCH_SIZE).repeat()
train_dataset = train_dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)
test_dataset = test.batch(BATCH_SIZE)

def display(display_list):
  plt.figure(figsize=(15, 15))

  title = ['Input Image', 'True Mask', 'Predicted Mask']

  for i in range(len(display_list)):
    plt.subplot(1, len(display_list), i+1)
    plt.title(title[i])
    plt.imshow(tf.keras.preprocessing.image.array_to_img(display_list[i]))
    plt.axis('off')
  plt.show()

for image, mask in train.take(3):
  sample_image, sample_mask = image, mask
display([sample_image, sample_mask])

def down_block(x, filters, kernel_size=(3, 3), padding="same", strides=1):
    c = keras.layers.Conv2D(filters, kernel_size, padding=padding, strides=strides, activation="relu")(x)
    c = keras.layers.Conv2D(filters, kernel_size, padding=padding, strides=strides, activation="relu")(c)
    p = keras.layers.MaxPool2D((2, 2), (2, 2))(c)
    return c, p

def up_block(x, skip, filters, kernel_size=(3, 3), padding="same", strides=1):
    us = keras.layers.UpSampling2D((2, 2))(x)
    concat = keras.layers.Concatenate()([us, skip])
    c = keras.layers.Conv2D(filters, kernel_size, padding=padding, strides=strides, activation="relu")(concat)
    c = keras.layers.Conv2D(filters, kernel_size, padding=padding, strides=strides, activation="relu")(c)
    return c

def bottleneck(x, filters, kernel_size=(3, 3), padding="same", strides=1):
    c = keras.layers.Conv2D(filters, kernel_size, padding=padding, strides=strides, activation="relu")(x)
    c = keras.layers.Conv2D(filters, kernel_size, padding=padding, strides=strides, activation="relu")(c)
    return c

def UNet():
    f = [16, 32, 64, 128, 256]   #Number of conv. filters in each block
    inputs = keras.layers.Input((image_size, image_size, 3))

    p0 = inputs
    #p1 will go to the next down block and c1 will go to the matching up block
    c1, p1 = down_block(p0, f[0]) #128 -> 64

    ########################################################
    #************* Complete the model here *****************
    #Your code should be only in this block. Add the corresponding
    #down blocks, followed by the bottleneck block, followed by matching
    #up blocks.

    c2, p2 = down_block(p1, f[1]) #64 -> 32
    c3, p3 = down_block(p2, f[2]) #32 -> 16
    c4, p4 = down_block(p3, f[3]) #16 -> 8
    c5 = bottleneck(p4, f[4]) #8 -> 8
    u1 = up_block(c5, c4, f[3]) #8 -> 16
    u2 = up_block(u1, c3, f[2]) #16 -> 32
    u3 = up_block(u2, c2, f[1]) #32 -> 64

    ########################################################

    u4 = up_block(u3, c1, f[0]) #64 -> 128

    #outputs = keras.layers.Conv2D(1, (1, 1), padding="same", activation="sigmoid")(u4)
    outputs = keras.layers.Conv2D(4, (1, 1), padding="same", activation="softmax")(u4)
    model = keras.models.Model(inputs, outputs)
    return model

model = UNet()
model.compile(optimizer='adam',
              loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
              metrics=['accuracy'])
model.summary()

def create_mask(pred_mask):
  pred_mask = tf.argmax(pred_mask, axis=-1)
  pred_mask = pred_mask[..., tf.newaxis]
  return pred_mask[0]

def show_predictions(dataset=None, num=1):
  if dataset:
    for image, mask in dataset.take(num):
      pred_mask = model.predict(image)
      display([image[0], mask[0], create_mask(pred_mask)])
  else:
    display([sample_image, sample_mask,
             create_mask(model.predict(sample_image[tf.newaxis, ...]))])

class DisplayCallback(tf.keras.callbacks.Callback):
  def on_epoch_end(self, epoch, logs=None):
    clear_output(wait=True)
    show_predictions()
    print ('\nSample Prediction after epoch {}\n'.format(epoch+1))

EPOCHS = 15
VAL_SUBSPLITS = 5
VALIDATION_STEPS = info.splits['test'].num_examples//BATCH_SIZE//VAL_SUBSPLITS

model_history = model.fit(train_dataset, epochs=EPOCHS,
                          steps_per_epoch=STEPS_PER_EPOCH,
                          validation_steps=VALIDATION_STEPS,
                          validation_data=test_dataset,
                          callbacks=[DisplayCallback()])

show_predictions(test_dataset, 3)


import numpy as np
import xarray as xr
import netCDF4
import time
import os
import glob

## global, fixed parameters

# grid parameters
xmax = 1010
dx = 10 # grid spacing
nx = 101
nz = 10
dt = 0.1

# physical parameters
g = 9.81 # acceleration due to gravity
hmin = 0.01 # minimum layer thickness

eps = 0.05

x_eta = np.empty(nx+2) # horizontal grid for depths and displacements
x_u = np.empty(nx+2) # horizontal grid for horizontal velocities
z = np.empty(nz+2) # vertical grid

htotal = np.empty(nx+2) # initial bathymetry
hzero = np.empty((nz+2, nx+2)) # initial layer thicknesses
h = np.empty((nz+2, nx+2)) # actual layer thicnesses
dp = np.empty((nz+2, nx+2)) # dynamic pressure

eta = np.empty((nz+2, nx+2)) # actual interface displacements
etan = np.empty((nz+2, nx+2)) # interface displacements at time level n+1
eta0 = np.empty((nz+2, nx+2)) # initial interface displacements
dhdt = np.empty((nz+2, nx+2)) # actual layer-thickness change
u = np.empty((nz+2, nx+2)) # actual lateral velocity
un = np.empty((nz+2, nx+2)) # lateral velocity at time level n+1
rho = np.empty(nz+2) # layer densities

wet = np.empty((nz+2, nx+2), dtype=int)

# In[383]:

def init_output(basename='', **attrs):

    # ensure we are writing to a new file each time
    previous_files = sorted(glob.glob(f'OUTPUT/{basename}_*.nc'))
    if len(previous_files) == 0:
        counter = 1
    else:
        counter = int(previous_files[-1][-7:-3]) + 1
    filename = f'OUTPUT/{basename}_{counter:04d}.nc'

    nc = netCDF4.Dataset(filename, "w")

    # NetCDF files have 'dimensions'
    nc.createDimension("time")
    nc.createDimension("x", nx+2)
    nc.createDimension("z", nz+2)

    # NetCDF files have 'variables'
    nc.createVariable("time", "f8", ("time",))
    # NetCDF variables can have 'attributes'
    nc.variables["time"].units = "seconds since 2000-01-01"
    nc.variables["time"].calendar = "gregorian"

    nc.createVariable("x_eta", "f8", ("x"))
    nc.variables["x_eta"].units = "m"
    nc.variables["x_eta"][:] = x_eta

    nc.createVariable("x_u", "f8", ("x"))
    nc.variables["x_u"].units = "m"
    nc.variables["x_u"][:] = x_u

    nc.createVariable("htotal", "f8", ("x"))
    nc.variables["htotal"].units = "m"
    nc.variables["htotal"][:] = htotal

    nc.createVariable("eta", "f8", ("time", "z", "x"))
    nc.variables["eta"].units = "m"
    nc.createVariable("u", "f8", ("time", "z", "x"))
    nc.variables["u"].units = "m s-1"
    nc.createVariable("h", "f8", ("time", "z", "x"))
    nc.variables["h"].units = "m"

    # NetCDF files have also have global attributes
    nc.setncatts(attrs)
    nc.history = "Created " + time.ctime(time.time())
    nc.source = "OMB Exercise 7"

    # It is important to close a NetCDF file
    nc.close()

    return filename

def write_output(filename, t, n=None):
    # open a NetCDF file in 'append' mode
    nc = netCDF4.Dataset(filename, mode='a')

    # if n is not provided, place values in the last position
    if n is None:
        n = len(nc.variables['time'])

    nc.variables["time"][n] = t
    nc.variables["eta"][n, :, :] = eta
    nc.variables["h"][n, :, :] = h
    nc.variables["u"][n, :, :] = u

    # It is important to close a NetCDF file
    nc.close()

# ### Subroutines

# In[384]:

def init():

    # calculate horizontal grids
    x_eta[:] = np.arange(-0.75 * dx, xmax+dx, dx)
    x_u[:] = x_eta + 0.5 * dx

    # bathymetry
    for k in range(1, nx+1):
        htotal[k] = 100

    # triangle-shaped island
    for k in range(31, 52):
        htotal[k] = 100 - 95*(k-30)/21
    for k in range(52, 72):
        htotal[k] = 100 - 95*(71 - k + 1)/20

    htotal[0] = -10
    htotal[nx+1] = -10

    # undisturbed layer thicknesses & interface displacements
    hini = np.ones(nz+2)*10

    for k in range(0, nx+2):
        htot = htotal[k]
        for i in range(1, nz+1):
            hzero[i, k] = max( min( hini[i], htot), 0)
            eta[i, k] = max(0, -htot)
            htot = htot - hini[1]

    # layer densities
    rho[0] = 0 # air density ignored
    rho[1] = 1025
    for i in range(2, nz+1):
        rho[i] = 1026 + (i-2)/(nz-2)*0.5

    # boundary values for dp and eta
    for k in range(0, nx+2):
        dp[0, k] = 0 # air pressure ignored
        eta[nz+1, k] = 0 # sea floor is rigid

    # store initial interface displacements
    for k in range(0, nx+2):
        for i in range(1, nz+2):
            eta0[i, k] = eta[i, k]

    # layer thicknesses, wet\dry pointers and velocities
    for i in range(1, nz+1):
        for k in range(0, nx+2):
            h[i, k] = hzero[i, k]
            wet[i, k] = 1
            if h[i, k] < hmin:
                wet[i, k] = 0
            u[i, k] = 0
            un[i, k] = 0

def dyn():

    # calculate dynamic pressure
    for k in range(0, nx+2):
        for i in range(1, nz+1):
            dp[i, k] = dp[i-1, k] + (rho[i] - rho[i-1])*g*eta[i,k]

    for k in range(1, nx+1):
        for i in range(1, nz+1):

            # velocity predictor for wet grid cells
            pgradx = -(dp[i, k+1]- dp[i, k])/rho[i]/dx
            un[i, k] = 0

            if wet[i, k]:
                if wet[i, k+1] or (pgradx>0):
                    un[i, k] = u[i, k] + dt*pgradx
            else:
                if wet[i, k+1] and (pgradx<0):
                    un[i, k] = u[i, k] + dt*pgradx

    # layer-thickness change predictor
    for k in range(1, nx+1):
        for i in range(1, nz+1):
            hep = 0.5*(un[i,k]+abs(un[i,k]))*h[i,k]
            hen = 0.5*(un[i,k]-abs(un[i,k]))*h[i,k+1]
            hue = hep+hen
            hwp = 0.5*(un[i,k-1]+abs(un[i, k-1]))*h[i,k-1]
            hwn = 0.5*(un[i,k-1]-abs(un[i, k-1]))*h[i,k]
            huw = hwp+hwn

            dhdt[i,k] = -(hue-huw)/dx

    # update interface displacements
    for k in range(1, nx+1):
        deta = 0
        for i in range(nz, 0, -1):
            deta = deta + dhdt[i, k]
            etan[i, k] = eta[i, k] + dt*deta

    # apply Shapiro filter
    shapiro()

    # update layer thicknesses, lateral velocities and wet/dry pointers
    for k in range(1, nx+1):
        for i in range(1, nz+1):
            h[i, k] = hzero[i, k] + eta[i, k] - eta[i+1, k] - eta0[i, k] + eta0[i+1, k]
            u[i, k] = un[i, k]
            wet[i, k] = 1
            if h[i, k] < hmin:
                wet[i, k] = 0

def shapiro():
    for i in range(1, nz+1):
        for k in range(1, nx+1):
            if wet[i, k]:
                term1 = (1.0-0.5*eps*(wet[i, k+1]+wet[i, k-1]))*etan[i, k]
                term2 = 0.5*eps*(wet[i, k+1]*etan[i, k+1]+wet[i, k-1]*etan[i, k-1])
                eta[i, k] = term1 + term2
            else:
                eta[i, k] = etan[i, k]

# In[385]:

def multi():

    # initialize arrays to the initial values
    init()

    # determine maximum water depth
    # automatic setting of time step (10% below CFL threshold)
    hmax = 100 # total water depth
    dt = 0.9 * dx / np.sqrt(g*hmax)
    print(f"Time step = {dt:.1f} seconds")

    # set epsilon for Shapiro filter
    eps = 0.05

    # parameters for wave paddle
    Apaddle = 1 # amplitude in metres
    Tpaddle = 10 # period in seconds CASE 1
    #Tpaddle = 2*3600 # period in seconds CASE 2

    # runtime parameters
    tmax = 10 * Tpaddle
    ntot = int(tmax/dt)

    # output parameter
    tout = Tpaddle/10
    nout = int(tout / dt)

    filename = init_output('multi')

    for n in range(ntot):
        t = n*dt

        for i in range(1, nz+1):
            eta[i, 1] = Apaddle*np.sin(2*np.pi*t/Tpaddle)

        #---- prognostic equations ----
        dyn()
        #------------------------------

        if n % nout == 0:
            write_output(filename, t)

    return filename

filename = multi()
print(filename)


# you can write to stdout for debugging purposes, e.g.
# print("this is a debug message")
import collections
def solution(A):
    # write your code in Python 3.6
    dic = dict(collections.Counter(A))
    leader, times = max(dic.items(), key = lambda x: x[1])

    left = 0
    equiCnt = 0
    for i, v in enumerate(A):
        if v == leader:
            left += 1
        if left > (i+1) / 2 and (times-left) > (len(A)-i-1) / 2:
            equiCnt += 1
    return equiCnt


import logging
from decimal import Decimal
from django.conf import settings

from exchange.models import Currency, ExchangeRate

from exchange.utils import update_many, insert_many

logger = logging.getLogger(__name__)

class BaseAdapter(object):
    """Base adapter class provides an interface for updating currency and
    exchange rate models

    """
    def update(self):
        """Actual update process goes here using auxialary ``get_currencies``
        and ``get_exchangerates`` methods. This method creates or updates
        corresponding ``Currency`` and ``ExchangeRate`` models

        """
        currencies = self.get_currencies()
        currency_objects = {}
        for code, name in currencies:
            currency_objects[code], created = Currency.objects.get_or_create(
                code=code, defaults={'name': name})
            if created:
                logger.info('currency: %s created', code)
        existing = ExchangeRate.objects.values('source__code',
                                               'target__code',
                                               'id')
        existing = {(d['source__code'], d['target__code']): d['id']
                    for d in existing}
        usd_exchange_rates = dict(self.get_exchangerates('USD'))
        updates = []
        inserts = []
        for source in currencies:
            for target in currencies:
                rate = self._get_rate_through_usd(source.code,
                                                  target.code,
                                                  usd_exchange_rates)

                exchange_rate = ExchangeRate(source=currency_objects[source.code],
                                             target=currency_objects[target.code],
                                             rate=rate)

                if (source.code, target.code) in existing:
                    exchange_rate.id = existing[(source.code, target.code)]
                    updates.append(exchange_rate)
                    logger.debug('exchange rate updated %s/%s=%s'
                                 % (source, target, rate))
                else:
                    inserts.append(exchange_rate)
                    logger.debug('exchange rate created %s/%s=%s'
                                 % (source, target, rate))

            logger.info('exchange rates updated for %s' % source.code)
        logger.info("Updating %s rows" % len(updates))
        update_many(updates)
        logger.info("Inserting %s rows" % len(inserts))
        insert_many(inserts)
        logger.info('saved rates to db')

    def _get_rate_through_usd(self, source, target, usd_rates):
        # from: https://openexchangerates.org/documentation#how-to-use
        # gbp_hkd = usd_hkd * (1 / usd_gbp)
        usd_source = usd_rates[source]
        usd_target = usd_rates[target]
        rate = usd_target * (Decimal(1.0) / usd_source)
        rate = rate.quantize(Decimal('0.123456'))  # round to 6 decimal places
        return rate

    def get_currencies(self):
        """Subclasses must implement this to provide all currency data

        :returns: currency tuples ``[(currency_code, currency_name),]``
        :rtype: list

        """
        raise NotImplementedError()

    def get_exchangerates(self, base):
        """Subclasses must implement this to provide corresponding exchange
        rates for given base currency

        :returns: exchange rate tuples ``[(currency_code, rate),]``
        :rtype: list

        """
        raise NotImplementedError()


import math
import torch

class2id = {'None': 0, 'A': 1, 'O': 2, 'Mixed': 3, 'Negative': 4, 'Neutral': 5, 'Positive': 6}

class Instance(object):
    def __init__(self, sentence_pack, word2index, category2index, args):
        self.id = sentence_pack['id']
        self.sentence = sentence_pack['sentence']

        '''generate sentence tokens'''
        self.sentence_tokens = torch.zeros(args.max_sequence_len).long()
        words = self.sentence.split()
        self.length = len(words)
        for i, w in enumerate(words):
            # word = w.lower()
            word = w
            if word in word2index:
                self.sentence_tokens[i] = word2index[word]
            else:
                self.sentence_tokens[i] = word2index['<unk>']

        self.tags = torch.zeros(1+args.max_sequence_len, 1+args.max_sequence_len).long()
        self.tags_category = torch.zeros(1+args.max_sequence_len, 1+args.max_sequence_len).long()
        self.tags[:, :] = -1
        self.tags_category[:, :] = -1

        for i in range(self.length+1):
            for j in range(i, self.length+1):
                self.tags[i][j] = 0
                self.tags_category[i][j] = 0

        self.quads = sentence_pack['quads']
        for quad in self.quads:
            a_l_span, a_r_span = quad[0]+1, quad[1]+1
            o_l_span, o_r_span = quad[4]+1, quad[5]+1
            category, sentiment = category2index[quad[2]], quad[3] + 3

            # EA & EO
            if a_l_span != 0 and a_r_span != 0 and o_l_span != 0 and o_r_span != 0:
                for i in range(a_l_span, a_r_span+1):
                    self.tags[i][i] = class2id['A']
                    if i > a_l_span:
                        self.tags[i-1][i] = class2id['A']
                    for j in range(i, a_r_span+1):
                        self.tags[i][j] = class2id['A']

                for i in range(o_l_span, o_r_span+1):
                    self.tags[i][i] = class2id['O']
                    if i > o_l_span:
                        self.tags[i-1][i] = class2id['O']
                    for j in range(i, o_r_span+1):
                        self.tags[i][j] = class2id['O']

                for i in range(a_l_span, a_r_span+1):
                    for j in range(o_l_span, o_r_span+1):
                        if i > j:
                            self.tags[j][i] = sentiment
                            self.tags_category[j][i] = category
                        else:
                            self.tags[i][j] = sentiment
                            self.tags_category[i][j] = category

            # IA & EO
            elif a_l_span == 0 and a_r_span == 0 and o_l_span != 0 and o_r_span != 0:
                if self.tags[0][0] < 4:
                    if self.tags[0][0] == class2id['None']:
                        self.tags[0][0] = class2id['A']
                    elif self.tags[0][0] == class2id['O']:
                        self.tags[0][0] = class2id['Mixed']

                for i in range(o_l_span, o_r_span+1):
                    self.tags[i][i] = class2id['O']
                    if i > o_l_span:
                        self.tags[i-1][i] = class2id['O']
                    for j in range(i, o_r_span+1):
                        self.tags[i][j] = class2id['O']

                for i in range(a_l_span, a_r_span+1):
                    for j in range(o_l_span, o_r_span+1):
                        if i > j:
                            self.tags[j][i] = sentiment
                            self.tags_category[j][i] = category
                        else:
                            self.tags[i][j] = sentiment
                            self.tags_category[i][j] = category

            # EA & IO
            elif a_l_span != 0 and a_r_span != 0 and o_l_span == 0 and o_r_span == 0:
                for i in range(a_l_span, a_r_span+1):
                    self.tags[i][i] = class2id['A']
                    if i > a_l_span:
                        self.tags[i-1][i] = class2id['A']
                    for j in range(i, a_r_span+1):
                        self.tags[i][j] = class2id['A']

                if self.tags[0][0] < 4:
                    if self.tags[0][0] == class2id['None']:
                        self.tags[0][0] = class2id['O']
                    elif self.tags[0][0] == class2id['A']:
                        self.tags[0][0] = class2id['Mixed']

                for i in range(a_l_span, a_r_span+1):
                    for j in range(o_l_span, o_r_span+1):
                        if i > j:
                            self.tags[j][i] = sentiment
                            self.tags_category[j][i] = category
                        else:
                            self.tags[i][j] = sentiment
                            self.tags_category[i][j] = category

            # IA & IO
            elif a_l_span == 0 and a_r_span == 0 and o_l_span == 0 and o_r_span == 0:
                self.tags[0][0] = sentiment
                self.tags_category[0][0] = category

        '''generate mask of the sentence'''
        self.sentence_mask = torch.zeros(args.max_sequence_len)
        self.sentence_mask[:self.length] = 1

        '''generate mask of the overall sentence'''
        self.mask = torch.zeros(args.max_sequence_len+1)
        self.mask[:self.length+1] = 1

def load_data_instances(sentence_packs, word2index, category2index, args):
    instances = list()
    for sentence_pack in sentence_packs:
        instances.append(Instance(sentence_pack, word2index, category2index, args))
    return instances

class DataIterator(object):
    def __init__(self, instances, args):
        self.instances = instances
        self.args = args
        self.batch_count = math.ceil(len(instances)/args.batch_size)

    def get_batch(self, index):
        sentence_ids = []
        sentence_tokens = []
        lengths = []
        sentence_masks = []
        masks = []
        tags = []
        tags_category = []

        for i in range(index * self.args.batch_size,
                       min((index + 1) * self.args.batch_size, len(self.instances))):
            sentence_ids.append(self.instances[i].id)
            sentence_tokens.append(self.instances[i].sentence_tokens)
            lengths.append(self.instances[i].length)
            sentence_masks.append(self.instances[i].sentence_mask)
            masks.append(self.instances[i].mask)
            tags.append(self.instances[i].tags)
            tags_category.append(self.instances[i].tags_category)

        indexes = list(range(len(sentence_tokens)))
        indexes = sorted(indexes, key=lambda x: lengths[x], reverse=True)

        sentence_ids = [sentence_ids[i] for i in indexes]
        sentence_tokens = torch.stack(sentence_tokens).to(self.args.device)[indexes]
        lengths = torch.tensor(lengths).to(self.args.device)[indexes]
        sentence_masks = torch.stack(sentence_masks).to(self.args.device)[indexes]
        masks = torch.stack(masks).to(self.args.device)[indexes]
        tags = torch.stack(tags).to(self.args.device)[indexes]
        tags_category = torch.stack(tags_category).to(self.args.device)[indexes]

        return sentence_ids, sentence_tokens, lengths, sentence_masks, masks, tags, tags_category


# coding:utf-8
# from keras.backend.tensorflow_backend import set_session
import pandas as pd
import numpy as np
import time
import csv
import tensorflow as tf
import matplotlib
matplotlib.use('agg')
import matplotlib.pyplot as plt
import math
from math import log, exp
from matplotlib import dates as mdates
from matplotlib import ticker as mticker
from matplotlib.dates import DateFormatter, WeekdayLocator, DayLocator, MONDAY, YEARLY
from matplotlib.dates import MonthLocator, MONTHLY
import datetime as dt

import h5py
from PIL import Image
import os
import threading
import multiprocessing as mp
import multiprocessing
from multiprocessing import Pool
import random

from numpy.random import randint
import imageio
import glob
from random import shuffle
import keras
from keras.utils import np_utils
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPooling2D, Activation, BatchNormalization
from keras.callbacks import ModelCheckpoint, EarlyStopping
from keras.optimizers import Adam
from keras.applications.resnet50 import ResNet50
from keras.applications.inception_resnet_v2 import InceptionResNetV2
from keras.applications.xception import Xception
from keras.utils import Sequence
from keras.utils import multi_gpu_model
from keras.preprocessing.image import img_to_array
from numpy import float
# from multi_gpu_model_fixed import multi_gpu_model

stocknum = {
    '0051', '1102', '1216', '1227', '1314', '1319', '1434', '1451', '1476',
    '1477', '1504', '1536', '1560', '1590', '1605', '1704', '1717', '1718',
    '1722', '1723', '1789', '1802', '1909', '2015', '2049', '2059', '2106',
    '2201', '2204', '2207', '2227', '2231', '2312', '2313', '2324', '2327',
    '2337', '2344', '2347', '2352', '2353', '2356', '2360', '2371', '2376',
    '2377', '2379', '2385', '2439', '2448', '2449', '2451', '2478', '2492',
    '2498', '2542', '2603', '2606', '2610', '2615', '2618', '2723', '2809',
    '2812', '2834', '2845', '2867', '2888', '2912', '2915', '3019', '3034',
    '3044', '3051', '3189', '3231', '3406', '3443', '3532', '3673', '3682',
    '3702', '3706', '4137', '4915', '4943', '4958', '5264', '5522', '5871',
    '6005', '6116', '6176', '6239', '6269', '6285', '6409', '6414', '6415',
    '6452', '6456', '8454', '8464', '9910', '9914', '9917', '9921', '9933',
    '9938', '9941', '9945'
}

def generate_from_file(x, y, batch_size):

    ylen = len(y)

    while (True):
        #X_output = np.zeros((batch_size,) + (224, 224, 3), dtype=np.float)
        Data = []

        randpick = randint(0, ylen, size=batch_size)
        labels = np.zeros((batch_size, 3), dtype=np.uint8)

        for i, value in enumerate(randpick):
            labels[i] = y[value]

            pic_addr = x[value]
            pic = imageio.imread(pic_addr)
            #X_output[i] = pic
            Data.append(pic)

        Data = np.array(
            Data, dtype=np.uint8).reshape((batch_size, ) + (224, 224, 3))
        # print(Data.shape)

        yield Data, labels

def main():
    #將單一一支股票當作validation
    pic_train = []
    table_train = []
    pic_validation = []
    table_validation = []

    for stockid in stocknum:
        df = pd.read_hdf('h5_data/' + stockid + '_table_sumchange.h5',
                         'stock_data_table')
        pic_addrs = []
        for i in range(len(df)):
            if (os.path.exists('stock_pic/' + stockid + 'pic/' + str(i)
                               .zfill(4) + '_' + stockid + '.jpg')) == False:
                print('FileNotFoundError error 圖片不足\n')
                raise FileNotFoundError
            pic_addrs.append('stock_pic/' + stockid + 'pic/' +
                             str(i).zfill(4) + '_' + stockid + '.jpg')

        randpickstock = randint(0, 111, size=15)
        valflag = 0
        for num in randpickstock:
            if stockid == '0051':
                break
            if stockid == list(stocknum)[num]:
                for item in pic_addrs:
                    item = item.replace('\\', '/')
                    pic_validation.append(item)
                for table in df.values:
                    table_validation.append(table)
                valflag = 1

        if valflag == 0:
            for item in pic_addrs:
                item = item.replace('\\', '/')
                pic_train.append(item)
            for table in df.values:
                table_train.append(table)

    # for i,labels in enumerate(table_train) :
    #     if labels.shape !=(3,):
    #         raise ValueError
    #     print(i)
    # for i,labels in enumerate(table_validation) :
    #     if labels.shape !=(3,):
    #         raise ValueError
    #     print(i)

    # for i,picaddr in enumerate(pic_train) :
    #     pic = imageio.imread(picaddr,format='jpg')
    #     if pic.shape !=(224,224,3):
    #         raise ValueError
    #     print(i)
    # for i,picaddr in enumerate(pic_validation) :
    #     pic = imageio.imread(picaddr,format='jpg')
    #     if pic.shape !=(224,224,3):
    #         raise ValueError
    #     print(i)
    print('pic_train shape = ' + str(len(pic_train)))
    print('table_train shape = ' + str(len(table_train)))
    print('pic_validation shape = ' + str(len(pic_validation)))
    print('table_validation shape = ' + str(len(table_validation)))

    c = list(zip(pic_train, table_train))
    shuffle(c)
    addrs, labels = zip(*c)
    train_addrs = list(addrs)
    train_labels = list(labels)

    vc = list(zip(pic_validation, table_validation))
    shuffle(vc)
    addrs, labels = zip(*vc)
    val_addrs = list(addrs)
    val_labels = list(labels)

    print('reading model......\n')
    parallel_model = keras.models.load_model(
        'best_acc.h5', custom_objects={"tf": tf})
    print('read model OK')
    # with tf.device('/cpu:0'):
        # model = ResNet50(
        #     input_shape=(224, 224, 3),
        #     classes=3,
        #     pooling='max',
        #     include_top=True,
        #     weights=None)
        # model = InceptionResNetV2(
        #     input_shape=(224, 224, 3), classes=3, include_top=True,weights=None)
    # parallel_model = Xception(
    #     input_shape=(224, 224, 3),
    #     classes=3,
    #     include_top=True,
    #     weights=None)

    # # parallel_model = multi_gpu_model(model, gpus=4)
    # parallel_model.compile(
    #     loss='categorical_crossentropy',
    #     optimizer='adam',
    #     metrics=['accuracy'])

    parallel_model.summary()
    print('\n\n\n\n')

    # if not os.path.exists('sum_pic.h5'):
    #     sum_pic = h5py.File('sum_pic.h5', mode='w')

    #     sum_pic.create_dataset(
    #         'pic', shape=(len(addrs), 224, 224, 3), dtype=np.uint8)

    #     for i, addr in enumerate(addrs):
    #         pic = imageio.imread(addr)
    #         sum_pic['pic'][i] = pic
    #         if i % 100 == 0:
    #             print(str(i) + ' / ' + str(len(addrs)) + '  pic done')

    #     sum_pic.create_dataset('table', data=labels, dtype=np.uint8)

    # else:
    #     sum_pic = h5py.File('sum_pic.h5', mode='r')

    batch_size = 10

    model_checkpoint_save = ModelCheckpoint(
        "best_acc.h5",
        monitor='val_acc',
        verbose=1,
        save_best_only=True,
        mode='max')
    model_checkpoint = EarlyStopping(
        monitor='val_acc', verbose=1, mode='max', min_delta=0.0001, patience=8)

    # train_history = parallel_model.fit(x=sum_pic['pic'][0:int(len(sum_pic['table'])*0.9)],
    #                                    y=sum_pic['table'][0:int(
    #                                        len(sum_pic['table'])*0.9)],
    #                                    #steps_per_epoch=None,
    #                                    steps_per_epoch=50000,
    #                                    epochs=1,
    #                                    #batch_size=batch_size,
    #                                    verbose=1, callbacks=[model_checkpoint],
    #                                    validation_split=0.2, shuffle=True)

    train_history = parallel_model.fit_generator(
        generate_from_file(train_addrs, train_labels, batch_size),
        steps_per_epoch=len(train_addrs) // batch_size // 3,
        # steps_per_epoch=1000,
        epochs=50,
        verbose=1,
        validation_data=generate_from_file(
            val_addrs[0:int(len(val_addrs) * 0.7)],
            val_labels[0:int(len(val_addrs) * 0.7)], batch_size),
        validation_steps=len(val_addrs[0:int(len(val_addrs) * 0.7)]) //
        batch_size//3,
        #validation_steps=100,
        # workers=mp.cpu_count(),
        workers=7,
        max_queue_size=1000,
        shuffle=True,
        callbacks=[model_checkpoint,model_checkpoint_save],
        use_multiprocessing=True)

    loss, accuracy = parallel_model.evaluate_generator(
        generate_from_file(val_addrs[int(len(val_addrs) * 0.7):],
                           val_labels[int(len(val_addrs) * 0.7):], batch_size),
        steps=len(val_addrs[int(len(val_addrs) * 0.7):]) // batch_size,
        # workers=mp.cpu_count(),
        workers=7,
        use_multiprocessing=True,
        max_queue_size=1000)

    print(train_history)
    plt.figure()
    plt.plot(train_history.history['acc'])
    plt.savefig(
        'model_accuracy' + str(round(accuracy, 2)) + '.jpg',
        dpi=200,
        bbox_inches='tight',
        mode='w')
    plt.figure()
    plt.plot(train_history.history['loss'])
    plt.savefig(
        'model_loss' + str(round(loss, 2)) + '.jpg',
        dpi=200,
        bbox_inches='tight',
        mode='w')

    print('\ntest loss: ', loss)
    print('\ntest accuracy: ', accuracy)

    parallel_model.save('my_model_' + str(round(accuracy, 3)) + '.h5')

main()


import time
import digitalio
import board
import adafruit_matrixkeypad
import pyaudiotest2
import pyaudio

# Membrane 3x4 matrix keypad on Raspberry Pi -
# https://www.adafruit.com/product/419
cols = [digitalio.DigitalInOut(x) for x in (board.D16, board.D20, board.D21)]
rows = [digitalio.DigitalInOut(x) for x in (board.D5, board.D6, board.D13, board.D19)]
hook = digitalio.DigitalInOut(board.D12)
hook.pull = digitalio.Pull.UP
print(hook.value);

keys = ((1, 2, 3),
        (4, 5, 6),
        (7, 8, 9),
        ('*', 0, '#'))

keypad = adafruit_matrixkeypad.Matrix_Keypad(rows, cols, keys)
lastKeysPressed = []
offHook = True

p = pyaudio.PyAudio()
stream = p.open(format=pyaudio.paFloat32, channels=1, rate=44100, output = 1)
pyaudiotest2.play_dial_tone(stream,2);
stream.close()
p.terminate()

def areEqual(arr1, arr2):
	#print("array 1");
	#print("array 2");
	#print(arr1);
	#print(arr2);
	if(len(arr1) != len(arr2)):
		return False;

	arr1.sort();
	arr2.sort();

	for i in range(0, len(arr1)-1):
		if(arr1[i] != arr2[i]):
			return False;

	return True;

while offHook:
	keys = keypad.pressed_keys
	if (not areEqual(keys, lastKeysPressed) and len(keys) > 0):
		print("Pressed: ", keys)
		lastKeysPressed = keys
	elif not hook.value:
		offHook = False;
	elif (len(keys) == 0):
		lastKeysPressed = []
	time.sleep(0.1)


from django.shortcuts import render, reverse, redirect
from django.views.decorators.http import require_POST, require_GET
from django.template.defaultfilters import escape
from django.core.exceptions import ObjectDoesNotExist

from .restfulAPI import restful
from .decorators import authenticate_user
from .models import Comment
from .forms import CommentForm
from apps.blogs.models import Blog

# 保存评论内容
@require_POST
def comment_viwe(request):
    user = request.user
    if user.is_authenticated:
        form = CommentForm(request.POST)
        if form.is_valid():
            content = form.cleaned_data.get('content')
            blog_pk = request.POST.get('blog_pk')
            excape_content = escape(content)
            username = user.username
            user_pk = user.pk
            try:
                blog = Blog.objects.get(pk = blog_pk)
                Comment.objects.create(auth=user, content=excape_content, blog=blog)
                return restful.result(data={'content': excape_content,'username': username,'user_pk': user_pk})
            except Exception as e:
                print(e)
                return restful.do_nothing(message='未知错误！')
        else:
            return restful.params_error(message='内容不能为空')
    else:
        return restful.un_signup(message='请先登录！')

@require_GET
def get_comment_reply(request):
    root_comment_pk = request.GET.get('root_comment_pk')
    try:
        comment = Comment.objects.get(pk = root_comment_pk)
        son_comments = comment.root_comment.all()
        if son_comments:
            context = {}
            for each_comment in son_comments:
                context[each_comment.auth.username] = [each_comment.content, each_comment.date_time]
            # context['son_comments'] =serializers.serialize('json',son_comments)
            print(context)
            return restful.result(data=context)
        else:
            return restful.do_nothing(message='该评论暂无回复！')

    except ObjectDoesNotExist:
        return restful.do_nothing(message='该评论不存在或已经删除！')

@require_POST
@authenticate_user
def reply_comment(request):
    # 回复的comment_id 回复的user_id 回复内容
    comment_id = request.POST.get('comment_id')
    user = request.user
    content = request.POST.get('content')
    try:
        parent_comment = Comment.objects.get(id=comment_id)
        root_comemnt = parent_comment.root or parent_comment
        try:
            Comment.objects.create(parent=parent_comment, root=root_comemnt, auth=user, reply_auth=parent_comment.auth, content=content, blog=parent_comment.blog)
            return restful.ok()
        except Exception as e:
            print(e)
            print(type(e))
            return restful.internal_error(message=e)
    except ObjectDoesNotExist:
        return restful.do_nothing(messag='没有找到该篇博客!')


import contextlib
import logging
import os
import sys

logging.basicConfig(level=logging.ERROR)

self_dir = os.path.abspath(os.path.dirname(__file__))
top_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                       os.pardir,
                                       os.pardir))
sys.path.insert(0, top_dir)
sys.path.insert(0, self_dir)

from oslo_utils import uuidutils

import taskflow.engines
from taskflow.patterns import linear_flow as lf
from taskflow.persistence import models
from taskflow import task

import example_utils as eu  # noqa

# INTRO: In this example linear_flow is used to group three tasks, one which
# will suspend the future work the engine may do. This suspend engine is then
# discarded and the workflow is reloaded from the persisted data and then the
# workflow is resumed from where it was suspended. This allows you to see how
# to start an engine, have a task stop the engine from doing future work (if
# a multi-threaded engine is being used, then the currently active work is not
# preempted) and then resume the work later.
#
# Usage:
#
#   With a filesystem directory as backend
#
#     python taskflow/examples/resume_from_backend.py
#
#   With ZooKeeper as backend
#
#     python taskflow/examples/resume_from_backend.py \
#       zookeeper://127.0.0.1:2181/taskflow/resume_from_backend/

# UTILITY FUNCTIONS #########################################

def print_task_states(flowdetail, msg):
    eu.print_wrapped(msg)
    print("Flow '%s' state: %s" % (flowdetail.name, flowdetail.state))
    # Sort by these so that our test validation doesn't get confused by the
    # order in which the items in the flow detail can be in.
    items = sorted((td.name, td.version, td.state, td.results)
                   for td in flowdetail)
    for item in items:
        print(" %s==%s: %s, result=%s" % item)

def find_flow_detail(backend, lb_id, fd_id):
    conn = backend.get_connection()
    lb = conn.get_logbook(lb_id)
    return lb.find(fd_id)

# CREATE FLOW ###############################################

class InterruptTask(task.Task):
    def execute(self):
        # DO NOT TRY THIS AT HOME
        engine.suspend()

class TestTask(task.Task):
    def execute(self):
        print('executing %s' % self)
        return 'ok'

def flow_factory():
    return lf.Flow('resume from backend example').add(
        TestTask(name='first'),
        InterruptTask(name='boom'),
        TestTask(name='second'))

# INITIALIZE PERSISTENCE ####################################

with eu.get_backend() as backend:

    # Create a place where the persistence information will be stored.
    book = models.LogBook("example")
    flow_detail = models.FlowDetail("resume from backend example",
                                    uuid=uuidutils.generate_uuid())
    book.add(flow_detail)
    with contextlib.closing(backend.get_connection()) as conn:
        conn.save_logbook(book)

    # CREATE AND RUN THE FLOW: FIRST ATTEMPT ####################

    flow = flow_factory()
    engine = taskflow.engines.load(flow, flow_detail=flow_detail,
                                   book=book, backend=backend)

    print_task_states(flow_detail, "At the beginning, there is no state")
    eu.print_wrapped("Running")
    engine.run()
    print_task_states(flow_detail, "After running")

    # RE-CREATE, RESUME, RUN ####################################

    eu.print_wrapped("Resuming and running again")

    # NOTE(harlowja): reload the flow detail from backend, this will allow us
    # to resume the flow from its suspended state, but first we need to search
    # for the right flow details in the correct logbook where things are
    # stored.
    #
    # We could avoid re-loading the engine and just do engine.run() again, but
    # this example shows how another process may unsuspend a given flow and
    # start it again for situations where this is useful to-do (say the process
    # running the above flow crashes).
    flow2 = flow_factory()
    flow_detail_2 = find_flow_detail(backend, book.uuid, flow_detail.uuid)
    engine2 = taskflow.engines.load(flow2,
                                    flow_detail=flow_detail_2,
                                    backend=backend, book=book)
    engine2.run()
    print_task_states(flow_detail_2, "At the end")


from pyspark import SparkContext
import pandas as pd
import numpy as np
from functools import reduce
from operator import add
import matplotlib.pyplot as plt
from tqdm import tqdm
from time import time

# preprocess the csv file to get data matrix
def preprocessFile(filename):
    frame = pd.read_csv(filename)
    label = 'TenYearCHD'
    variables = list(frame)
    variables.remove(label)
    train = frame[variables]
    train = (train - train.mean()) / train.std()
    train['biascol'] = 1
    X = np.array(train)
    y = np.array(frame[label]).reshape((-1, 1))
    xy = np.concatenate([X, y], axis=-1).tolist()
    return xy

def gradient (line, w):
    y = np.array(line[-1])
    x = np.array(line[:-1])
    z = x.dot(w)
    s = 1. / (1. + np.exp(-z) )
    return x * (s - y)

def forward(line, w):
    x = np.array(line[:-1])
    z = x.dot(w)
    return int(z > 0)

sc = SparkContext()
sc.setLogLevel('ERROR')

xy = preprocessFile('heart_disease.csv')
data = sc.parallelize( xy ).cache()
numWeights = len(xy[0])-1
m = len(xy)
w = 2 * np.random.ranf(numWeights) - 1

start = time()
iter_times = []
for i in tqdm(range(100)):
    w -= data.map( lambda m : gradient (m , w ) ).reduce( add ) / m
    iter_times.append(time() - start)

print("Getting accuracy")
preds = np.array(list(map(lambda f: forward(f, w), xy)))
actual = np.array(list(map(lambda x: x[-1], xy)))
print((preds == actual).mean())
sc.stop()

print("Logging to sparktimes.txt")
with open('sparktimes.txt', 'w') as file:
    for num in iter_times:
        file.write(str(num) + '\n')


from django.urls import path
from book import views

urlpatterns = [
    path('add/',views.add),
    path('register/',views.register),
    path('login/',views.userlogin),
    path('delete/<int:tid>',views.delete),
    path('update/<str:tid>',views.update),
    path('refresh/',views.refersh),
    path('ltoh/',views.priceLtoH),
    path('htol/',views.priceHtoL),
    path('asc/',views.nameasc),
    path('dsc/',views.namedsc),
    path('reference/',views.reference),
    path('fiction/',views.fiction),
    path('nfiction/',views.nfiction),
    path('edited/',views.edited)

]


from abc import ABC
from typing import Generic
from uuid import UUID

from app.services.service_protocols.database_service_protocol import (
    BMT,
    DatabaseServiceProtocol,
)
from app.services.service_protocols.io_service_protocol import (
    IOServiceProtocol,
)
from app.utils import json_utils

class JsonDatabaseService(DatabaseServiceProtocol[BMT], Generic[BMT], ABC):
    def __init__(
        self, io_service: IOServiceProtocol, model_type: type[BMT]
    ) -> None:
        self.model_type = model_type
        self._io_service = io_service

    async def get_all(self) -> list[BMT]:
        return await self._get_data()

    async def get(self, id_: UUID) -> BMT | None:
        data = await self._get_data()
        return next((item for item in data if item.id == id_), None)

    async def create(self, new: BMT) -> None:
        data = await self._get_data()

        data.append(new)

        await self._save_file(data)

    async def delete(self, id_: UUID) -> None:
        data = await self._get_data()

        data = [item for item in data if item.id != id_]

        await self._save_file(data)

    async def put(self, id_: UUID, new: BMT) -> None:
        data = await self._get_data()

        for i, item in enumerate(data):
            if item.id == id_:
                data[i] = new
                break

        await self._save_file(data)

    async def _get_data(self) -> list[BMT]:
        files_contents = await self._io_service.read()

        data = json_utils.loads(files_contents)

        return [self.model_type(**item) for item in data]

    async def _save_file(self, data: list) -> None:
        file_contents = json_utils.dumps(data, indent=4)

        await self._io_service.write(file_contents)


import os

import fire
import numpy as np
import scipy.stats
import pandas
import matplotlib
import matplotlib.pyplot as plt
import itermplot

from matplotlib.ticker import FuncFormatter

OUTPUT = 'generated'
RNG = np.random.RandomState(0)

# reverse plot color is output is in the terminal
itermplot.THEME = 'rv' if OUTPUT is None else ''
matplotlib.use('module://itermplot')

matplotlib.rcParams['font.sans-serif'] = 'Source Sans Pro'
matplotlib.rcParams['font.family'] = 'sans-serif'
matplotlib.rcParams['font.weight'] = 'semibold'
matplotlib.rcParams['font.size'] = '10'
matplotlib.rcParams['pdf.fonttype'] = 42

def formatter(x, pos):
    if x != 0:
        if abs(x) > 1:  # hacky
            x = f'{int(x)}'
        else:
            x = ('%.2f' % x).lstrip('0').rstrip('0')
    else:
        x = '0'
    return x

def corr(df, x, y, report_p=False):

    sel = df[x].notnull() & df[y].notnull()
    try:
        r, p = scipy.stats.pearsonr(df.loc[sel, x], df.loc[sel, y])
    except:
        breakpoint()
    if p > .05:
        r = 'r: n.s.'
    else:
        r = 'r = ' + f'{r:.2f}'.lstrip('0')

    if report_p:
        if p < .001:
            p_level = f'{p:.0e}'
        elif p < .01:
            p_level = '.01'
        elif p < .05:
            p_level = '.05'
        else:
            p_level = ''

        if p_level != '':
            r += f' (p < {p_level})'

    return r

def output_paper_quality(ax, title=None, xlabel=None, ylabel=None):
    ax.set_title(title)
    ax.set_xlabel(xlabel, weight='semibold', size=12)
    ax.set_ylabel(ylabel, weight='semibold', size=12)

    ax.xaxis.set_major_formatter(FuncFormatter(formatter))
    ax.yaxis.set_major_formatter(FuncFormatter(formatter))
    ax.spines['right'].set_visible(False)
    ax.spines['top'].set_visible(False)
    ax.tick_params(bottom=False, left=False)

def output(filename):
    plt.tight_layout()
    if OUTPUT is not None:
        plt.savefig(os.path.join(OUTPUT, f'{filename}.png'), bbox_inches='tight', dpi=300)
        plt.savefig(os.path.join(OUTPUT, f'{filename}.pdf'), bbox_inches='tight', transparent=True)
    else:
        plt.show()

def scatterplot(ax, df, x, y, scale=30, annotate=False):
    dff = df[df.Model != 'CORnet-S']
    jitter = dff.jitter if 'jitter' in dff else 0
    ax.scatter(dff[x] + jitter, dff[y], s=scale, color=dff.color, alpha=.7, edgecolors='none')#linewidths=.1)

    dff = df[df.Model == 'CORnet-S']
    jitter = dff.jitter if 'jitter' in dff else 0
    ax.scatter(dff[x] + jitter, dff[y], s=2*scale, color=dff.color, alpha=.7, edgecolors='none')#, linewidths=.1)

    if annotate:
        for idx, row in df.iterrows():
            ax.text(row[x], row[y], row.Model)
    return ax

def read_common_data():
    df = pandas.read_csv('data/data.csv')

    for idx, row in df.iterrows():
        if row.Model.lower().startswith('basenet'):
            color = 'gray'
        elif row.Model.lower().startswith('cornet'):
            color = 'crimson'
        else:
            color = '#078930'
        df.loc[idx, 'color'] = color

    return df

def _fig1(df, scale=15, inset=False):
    ax = plt.subplot(111)
    scatterplot(ax, df, x='ImageNet', y='Brain-Score', scale=scale)
    if not inset:
        df = df[df.ImageNet < .7]
    r = corr(df, x='ImageNet', y='Brain-Score')

    ax.annotate(r, xy=(.75, .1),
                xycoords='axes fraction',
                # horizontalalignment='left', verticalalignment='top',
                fontsize=10)
    output_paper_quality(ax,
            xlabel='ImageNet top-1 performance',
            ylabel='Brain-Score')

def fig1():
    df = read_common_data()
    plt.figure(figsize=(4, 4))
    _fig1(df, scale=15)
    output('fig1')
    plt.figure(figsize=(3, 3))
    _fig1(df[(df['ImageNet'] >= .7) & (df.Model != 'CORnet-S')], scale=45, inset=True)
    output('fig1_inset')

def _fig2(ax, df, x, y, title=None, xlabel=None, ylabel=None, index=0, r=None):
    scatterplot(ax, df, x, y)
    r = corr(df[df.Model != 'CORnet-S'], x, y)
    ax.annotate(r, xy=(.75, .1), xycoords='axes fraction', fontsize=10)
    ax.annotate(f"({'abcd'[index]})", xy=(.05 + .24 * index, .9),
                xycoords='figure fraction', fontsize=20)
    output_paper_quality(ax, title=title,
        xlabel=xlabel, ylabel=ylabel)
    ax.set_title(title, va='top', pad=20)

def fig2():
    df = read_common_data()
    dff = df[~df.Model.str.startswith('BaseNet')]

    fig, axes = plt.subplots(ncols=4, figsize=(12,3))
    _fig2(axes[0], dff, x='IT', y='IT (new data)',
        xlabel='IT score (original neurons)', ylabel='IT score (new neurons)',
        title='New neural recordings,\nsame images', index=0, r=.93)
    _fig2(axes[1], dff, x='IT', y='IT (new images)',
        xlabel='IT score (original neurons)', ylabel='IT score (new neurons)',
        title='New neural recordings,\nnew images', index=1, r=.76)
    _fig2(axes[2], dff, x='Behavior', y='Behavior (new data)',
        xlabel='Behavioral score (original)', ylabel='Behavioral score (new)',
        title='New behavioral recordings,\nnew images', index=2, r=.83)
    _fig2(axes[3], dff, x='Brain-Score', y='CIFAR-100',
        xlabel='Brain-Score', ylabel='CIFAR-100 transfer',
        title='CIFAR-100 transfer', index=3, r=.69)
    output('fig2')

def _fig3(ax, df, y, ylabel=None, title=None):
    scatterplot(ax, df, x='Depth', y=y)
    ax.set_xscale('log')
    ax.set_xticks([10, 25, 50, 100, 200])
    output_paper_quality(ax, title=title,
        xlabel='Model Depth (log scale)', ylabel=ylabel)

def fig3():
    df = read_common_data()
    dff = df[~df.Model.str.startswith('BaseNet')].copy()

    fig, axes = plt.subplots(ncols=3, figsize=(9,3), sharex=True)
    rangex = .05 * (dff['Depth'].max() - dff['Depth'].min())

    jitter = []
    for idx, row in dff.iterrows():
        if row.Model.startswith('MobileNet'):
            j = RNG.uniform(-rangex, rangex)
        else:
            j = 0
        jitter.append(j)
    dff['jitter'] = jitter

    _fig3(axes[0], dff, y='Brain-Score', ylabel='Brain-Score')
    _fig3(axes[1], dff, y='ImageNet', ylabel='ImageNet top-1')
    _fig3(axes[2], dff, y='CIFAR-100', ylabel='CIFAR-100 transfer')
    output('fig3')

def _fig_a1(ax, df, region):
    x = f'{region} number of features'
    scatterplot(ax, df, x=x, y=region)
    r = corr(df[df[x] >= 1000], x, region, report_p=True)
    print(f'Fig A1: {r}')
    # ax.annotate(r, xy=(.5, .1), xycoords='axes fraction', fontsize=10)
    output_paper_quality(ax, xlabel='Number of features', ylabel=f'{region} neural score')

def fig_a1():
    df = read_common_data()
    dff = df[(df['V4 number of features'] < 20000) & (df['IT number of features'] < 20000)]

    fig, axes = plt.subplots(ncols=2, figsize=(6,3))
    _fig_a1(axes[0], dff, region='V4')
    _fig_a1(axes[1], dff, region='IT')
    output('fig_a1')

def fig_a2():
    df = pandas.read_csv('data/cornet_search.csv')

    plt.figure(figsize=(6, 4))
    ax = plt.subplot(111)
    ax.scatter(df['ImageNet'], df['Behavior'], s=10, color='gray', alpha=.7, edgecolors='none')
    r = corr(df, x='ImageNet', y='Behavior')
    ax.annotate(r, xy=(.75, .1), xycoords='axes fraction', fontsize=10)
    output_paper_quality(ax, xlabel='ImageNet top-1', ylabel='Behavioral score')
    output('fig_a2')

def _fig_a3(ax, df, region, time):
    y = f'{region} ({time})'
    ms = 100 if time == 'early' else 200
    scatterplot(ax, df, x='ImageNet', y=y)
    r = corr(df, x='ImageNet', y=y, report_p=True)
    ax.annotate(r, xy=(.5, .1), xycoords='axes fraction', fontsize=10)
    output_paper_quality(ax, xlabel='ImageNet top-1',
                         ylabel=f'{region} neural score (at {ms} ms)',
                         title=f'{region} {time}')

def fig_a3():
    df = read_common_data()

    fig, axes = plt.subplots(nrows=2, ncols=2, figsize=(6,6))
    _fig_a3(axes[0,0], df, region='V4', time='early')
    _fig_a3(axes[0,1], df, region='V4', time='late')
    _fig_a3(axes[1,0], df, region='IT', time='early')
    _fig_a3(axes[1,1], df, region='IT', time='late')
    output('fig_a3')

def _highlight_max(data):
    if data.dtype.name == 'object':  # got a string (model)
        return data

    max_idx = data.idxmax()
    formatted_data = []
    for idx, value in data.iteritems():
        if value != 0:
            value = f'{value:.3f}'.lstrip('0').rstrip('0')
        else:
            value = '0'

        if idx == max_idx:
            value = '\\textbf{' + value + '}'
        formatted_data.append(value)

    return formatted_data

def table_a1():
    df = read_common_data()
    dff = df[df['Brain-Score'].notnull() & ~df.Model.str.startswith ('BaseNet') & ~df.Model.str.startswith('MobileNet')]

    idx = df.loc[df.Model.str.startswith ('BaseNet'), 'Brain-Score'].idxmax()
    best_basenet = df.loc[idx].copy()
    best_basenet['Model'] = 'Best BaseNet'
    dff = dff.append(best_basenet)

    idx = df.loc[df.Model.str.startswith ('MobileNet'), 'Brain-Score'].idxmax()
    best_mobilenet = df.loc[idx].copy()
    best_mobilenet['Model'] = 'Best MobileNet'
    dff = dff.append(best_mobilenet)

    dff = dff.sort_values(by='Brain-Score', ascending=False)
    dff = dff[['Model', 'Brain-Score', 'V4', 'IT', 'OST', 'Behavior']].apply(_highlight_max)

    dff.to_latex(os.path.join(OUTPUT, 'table_a1.tex'), escape=False, index=False)

def gen_all():
    fig1()
    fig2()
    fig3()
    # data for fig4 is not provided
    # fig5 is generated by fig5.py from scratch
    table_a1()
    fig_a1()
    fig_a2()
    fig_a3()

if __name__ == '__main__':
    fire.Fire()


import argparse
import requests

from oauth2client.client import GoogleCredentials

# function to get authorization bearer token for requests
def get_access_token():
    """Get access token."""

    scopes = ["https://www.googleapis.com/auth/userinfo.profile", "https://www.googleapis.com/auth/userinfo.email"]
    credentials = GoogleCredentials.get_application_default()
    credentials = credentials.create_scoped(scopes)

    return credentials.get_access_token().access_token

def call_rawls_batch_upsert(workspace_name, project, request):
    """Post entities to Terra workspace using batchUpsert."""

    # rawls request URL for batchUpsert
    uri = f"https://rawls.dsde-prod.broadinstitute.org/api/workspaces/{project}/{workspace_name}/entities/batchUpsert"

    # Get access token and and add to headers for requests.
    # -H  "accept: */*" -H  "Authorization: Bearer [token] -H "Content-Type: application/json"
    headers = {"Authorization": "Bearer " + get_access_token(), "accept": "*/*", "Content-Type": "application/json"}

    # capture response from API and parse out status code
    response = requests.post(uri, headers=headers, data=request)
    status_code = response.status_code

    if status_code != 204:  # entities upsert fail
        print(f"WARNING: Failed to upload entities.")
        print(response.text)
        return

    # entities upsert success
    print(f"Successfully uploaded entities." + "\n")

def write_request_json(request, filename_prefix):
    """Create output file with json request."""

    save_name = f"{filename_prefix}_batch_upsert_request.json"
    with open(save_name, "w") as f:
        f.write(request)

def convert_string_to_list(input_string):
    """Convert a given string into an array compatible with data model tsvs for array attributes."""

    # remove single & double quotes, remove spaces, remove [ ], separate remaining string on commas (resulting in a list)
    output_list = str(input_string).replace("'", '').replace('"', '').replace(" ", "").strip('[]').split(",")

    return output_list

def create_list_attr_operation(var_attribute_list_name):
    """Return request string for a single operation to create an attribute of type array/list."""

    return '{"op":"CreateAttributeValueList","attributeName":"' + var_attribute_list_name + '"},'

def add_list_member_operation(var_attribute_list_name, var_attribute_list_member):
    """Return request string for a single operation to add a list member to an attribute of type array/list."""

    return '{"op":"AddListMember","attributeListName":"' + var_attribute_list_name + '", "newMember":"' + var_attribute_list_member + '"},'

def create_non_array_attr_operation(var_attribute_name, var_attribute_value):
    """Return request string for a single operation to create a non-array attribute."""

    return '{"op":"AddUpdateAttribute","attributeName":"' + var_attribute_name + '", "addUpdateAttribute":"' + var_attribute_value + '"},'

def create_single_entity_request(var_entity_id, var_entity_type, single_entity_operations):
    """Return request string with array/list attributes, their associated values/members, and single entity operations."""

    return '{"name":"' + var_entity_id + '", "entityType":"' + var_entity_type + '", "operations":[' + single_entity_operations + ']}'

def create_upsert_request(tsv, array_attr_cols=None):
    """Generate the request body for batchUpsert API."""

    # check tsv format: data model load tsv requirement "entity:table_name_id" or "membership:table_name_id" -> else exit
    entity_type_col_name = tsv.columns[0]                               # entity:entity_name_id
    entity_type = entity_type_col_name.rsplit("_", 1)[0].split(":")[1]  # entity_name

    if not entity_type_col_name.startswith(("entity:", "membership:")):
        print("Invalid tsv. The .tsv does not start with column entity:[table_name]_id or membership:[table_name]_id. Please correct and try again.")
        return

    # replace the "entity:col_name_id" with just "col_name" in df
    # if not replaced, "attributeName" in the template_make_single_attr becomes "entity:entity_name_id" instead of just entity_name
    # when the API request is made, its read as multiple columns with the "entity" prefix which is illegal
    # this is specific just to the first column where the format is required for terra load tsv files
    tsv.rename(columns={entity_type_col_name: entity_type}, inplace=True)

    # initiate string to capture all operation requests for all rows (entities) in given tsv file
    all_entities_request = []

    # for every row (entity) in df
    for index, row in tsv.iterrows():
        # initialize string to capture request for one row (entity) in tsv
        single_entity_operations = ''''''
        # get the entity_id (row id - must be unique)
        entity_id = str(row[0])

        # if array columns/attributes provided
        if array_attr_cols:
            # for each array column
            for col in array_attr_cols:
                # get operation json to make the array/list attribute with column name
                single_entity_operations += create_list_attr_operation(col)
                # convert string value -> back into an array: [foo,bar] (str) --> ['foo', 'bar'] (list)
                attr_values = convert_string_to_list(row[col])
                # for each item in array (values pertaining to the list attribute as defined above)
                for val in attr_values:
                    # get operation json to add each array/list attribute's values to the request
                    single_entity_operations += add_list_member_operation(col, val)

            # get non-array/list attributes based on which of the full tsv columns are array attributes
            single_attr_cols = list(set(list(tsv.columns)) - set(array_attr_cols))
        # if no array columns/attributes provided
        else:
            single_attr_cols = list(tsv.columns)

        # if there are non-array/list attribute columns - cases where all columns are arrays, single_attr_cols would be empty
        if single_attr_cols:
            # for each column that is not an array
            for col in single_attr_cols:
                # get value in col from df
                attr_value = str(row[col])
                # get operation json with the row (entity) value to the non-array column (attribute)
                single_entity_operations += create_non_array_attr_operation(col, attr_value)

        # remove trailing comma from the last request template
        single_entity_operations = single_entity_operations[:-1]

        # fill in entity_type (table name), entity_name (row id), and all entity operations into request body template
        single_entity_request = create_single_entity_request(entity_id, entity_type, single_entity_operations)

        # add the request pertinent to the rows (entities) of a single workspace to list of all workspace requests
        all_entities_request.append(single_entity_request)

    # remove quotes around elements of the list but keep the brackets - using sep() or join() get rid of the brackets
    # ['{entity1}', '{entity2}', '{..}'] (api fails) --> [{entity1}, {entity2}, {..}] (api succeeds)
    all_entities_request_formatted = '[%s]' % ','.join(all_entities_request)

    return all_entities_request_formatted

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='')
    parser.add_argument('-w', '--workspace_name', required=True, help='name of workspace in which to make changes')
    parser.add_argument('-p', '--project', required=True, help='billing project (namespace) of workspace in which to make changes')
    parser.add_argument('-t', '--tsv', required=True, help='.tsv file formatted in load format to Terra UI')

    args = parser.parse_args()

    # create request body for batchUpsert
    request = create_upsert_request(args.tsv)
    # call batchUpsert API (rawls)
    call_rawls_batch_upsert(args.workspace_name, args.project, request)


from collections import deque
n = int(input())

arr = [True for i in range(n+1)]

for i in range(2, n):
    if arr[i]:
        for j in range(i*2, n+1,i):
            arr[j] = False

prime_num = []
for i in range(2, n+1):
    if arr[i]:
        prime_num.append(i)

result = 0
total_sum = 0
q = deque([])
for i in prime_num:

    q.append(i)
    total_sum+=i

    if total_sum == n:
        result +=1

    while total_sum >= n:
        a = q.popleft()
        total_sum-=a

        if total_sum == n:
            result+=1

print(result)