# SOURCE: https://www.django-rest-framework.org/tutorial/3-class-based-views/
# Create your views here.
#from this import d
from accounts.models import Account
from accounts.serializers import *
from rest_framework.views import APIView
from rest_framework.response import Response
from rest_framework import status
from stocks.financeAPI import Stock_info as si
from datetime import datetime
from accounts.utils.historicalPortfolioValueLoader import PortfolioValue
from accounts.utils.transactionHandler import Transaction
from accounts.utils.balanceHandler import Balance
from accounts.utils.userStockHandler import UserStocks
from accounts.utils.stockListsHandler import StockLists

from django.contrib.auth.models import User # we are extending Django's User

from django.conf import settings
from social_django.utils import psa
from decimal import Decimal

@psa()
def exchange_token(request, backend):
    """
    Exchange an OAuth2 access token for one for this site.
    This simply defers the entire OAuth2 process to the front end.
    The front end becomes responsible for handling the entirety of the
    OAuth2 process; we just step in at the end and use the access token
    to populate some user identity.
    The URL at which this view lives must include a backend field, like:
        url(API_ROOT + r'social/(?P<backend>[^/]+)/$', exchange_token),
    Using that example, you could call this endpoint using i.e.
        POST API_ROOT + 'social/facebook/'
        POST API_ROOT + 'social/google-oauth2/'
    Note that those endpoint examples are verbatim according to the
    PSA backends which we configured in settings.py. If you wish to enable
    other social authentication backends, they'll get their own endpoints
    automatically according to PSA.
    ## Request format
    Requests must include the following field
    - `access_token`: The OAuth2 access token provided by the provider
    """
    token = request.query_params.get('token', None)
    serializer = AuthSerializer(data={"access_token": token})
    if serializer.is_valid(raise_exception=True):
        # set up non-field errors key
        # http://www.django-rest-framework.org/api-guide/exceptions/#exception-handling-in-rest-framework-views
        try:
            nfe = settings.NON_FIELD_ERRORS_KEY
        except AttributeError:
            nfe = 'non_field_errors'

        try:
            print("AUTHENTICATING USER")
            user = request.backend.do_auth(serializer.validated_data['access_token'])
            print("SUCCESS")
            return {
                    "result": {
                        "token": "valid",
                        "detail": "Succesfully retreived user"
                    },
                    "data": {
                        "name": user.first_name + " " + user.last_name,
                        "email": user.email, "google_user_id": user.username
                    }
            }
        except BaseException as e:
            print("FAILED")
            # An HTTPError bubbled up from the request to the social auth provider.
            # This happens, at least in Google's case, every time you send a malformed
            # or incorrect access key.
            return {
                'result': {
                    'token': 'invalid',
                    'detail': str(e),
                }
            }

def get_object(email, *args, **kwargs):
    #pk = kwargs.get('goog_id')
    pk = email
    try:
        return Account.objects.get(pk=pk)
    except Account.DoesNotExist:
        return None

def getDataFromToken(request):
    data = exchange_token(request, "google-oauth2")
    if data["result"]["token"] == "invalid":
        return (False, Response(None, status=status.HTTP_401_UNAUTHORIZED))
    return (True, data["data"])

class AccountList(APIView):
    """
    List all accounts, or create a new Account.
    """
    def get(self, request):
        accounts = Account.objects.all()
        serializer = AccountSerializer(accounts, many=True)
        return Response(serializer.data, status=status.HTTP_200_OK)

class AccountReset(APIView):

    def get(self, request, goog_id):
        (valid, data) =  getDataFromToken(request)
        if not valid:
            return data

        account = get_object(data["email"])
        account.start_date = datetime.today().strftime("%Y-%m-%d")
        account.portfolio_value_history = {"data": {}}
        account.transaction_history = {"history": []}
        account.ownedStocks = {}
        account.balance = 100000
        account.save()
        serializer = AccountSerializer(account)#, data=request.data)
        return Response(serializer.data, status=status.HTTP_205_RESET_CONTENT)
        #if serializer.is_valid():
        #    serializer.save() # saves to DB
        #    return Response(serializer.data, status=status.HTTP_205_RESET_CONTENT)
        #return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

class AccountDetail(APIView):
    """
    Create, retrieve, update or delete a Account instance.
    """
    def post(self, request, *args, **kwargs):
        # EDIT: need way to add User's pk to req. data to map this Account -> User.
        # tried: User.objects.get(email=request.data['email']), request.data['email'].split('@')[0]
        # request.data['user'] = ?
        # print("UPDATED REQ. DATA", request.data)
        (valid, data) = getDataFromToken(request)
        if not valid:
            return data
        serializer = AccountSerializer(data=data)
        if serializer.is_valid():
            serializer.save() # saves to DB
            self.__initializeUser(data["email"])
            return Response(serializer.data, status=status.HTTP_201_CREATED)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

    def __initializeUser(self, email):
        user = get_object(email)
        user.watchList = {"stocks": []}
        user.transaction_history = {"history": []}
        user.portfolio_value_history = {"data": {}}
        user.start_date = datetime.today().strftime("%Y-%m-%d")
        user.save()

    def get(self, request, goog_id):

        (valid, data) =  getDataFromToken(request)
        if not valid:
            return data

        account = get_object(data["email"])
        if account != None: # account exists
            stock_value = UserStocks.calculateCurrentPortfolioValue(account.ownedStocks)
            portfolio_value = Decimal.from_float(stock_value) + account.balance
            account.portfolio_value = portfolio_value
            serializer = AccountSerializer(account)
            PortfolioValue.load(account)
            return Response(serializer.data, status=status.HTTP_200_OK)
        else: # account doesn't exist, create new
            return Response(None, status=status.HTTP_404_NOT_FOUND)

    def put(self, request, goog_id):
        (valid, data) =  getDataFromToken(request)
        if not valid:
            return data

        account = get_object(data["email"])
        serializer = AccountSerializer(account, data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data, status=status.HTTP_200_OK)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

    def delete(self, request, goog_id):
        (valid, data) =  getDataFromToken(request)
        if not valid:
            return data

        account = get_object(data["email"])
        account.delete()
        return Response(status=status.HTTP_204_NO_CONTENT)

# class AccountBalance(APIView):
#     """
#     Update an Account's buying power.
#     """
#     def put(self, request, pk):
#         Account = self.get_object(pk)
#         print("AMOUNT SENT IN BP PUT: ", request.data['amount'], type(request.data['amount']))
#         serializer = AccountSerializer(Account, data=request.data)
#         # if serializer.is_valid():
#         #     serializer.save()
#         #     return Response(serializer.data)
#         return Response(None)

class AccountTransactionHistory(APIView):

    def get(self, request, goog_id):
        (valid, data) =  getDataFromToken(request)
        if not valid:
            return data

        account = get_object(data["email"])
        history = account.transaction_history["history"][::-1]
        serializer = TransactionHistorySerializer({"transaction_history":history})
        return Response(serializer.data, status=status.HTTP_200_OK)

class AccountStocksOwned(APIView):
    '''
    When making a get request, the data only needs one field "info". It may also need "symbol" depending on what you want
    info options:
    stock_list_detailed     includes every transaction for stocks currently owned with date purchased and purchase price
    num_of_ticker_stocks    the amount of shares they own of the stock. Must specify "symbol" in the request
    portfolio_value         The total value of the users portfolio
    stock_list_display      A list of every stock the user owns, its quantity, current price, change %, and change direction

    Examples:
    curl -d '{"info":"stock_list_display"}' -H "Content-Type: application/json" -X GET http://127.0.0.1:8000/accounts/[ACCOUNT_GOOG_ID]/getStocks

    curl -d '{"info":"num_of_ticker_stocks", "symbol": "tsla"}' -H "Content-Type: application/json" -X GET http://127.0.0.1:8000/accounts/[ACCOUNT_GOOG_ID]/getStocks
    '''
    def get(self, request, goog_id):
        (valid, data) = getDataFromToken(request)
        if not valid:
            return data

        account = get_object(data["email"])
        if account == None:
            return Response(None, status=status.HTTP_404_NOT_FOUND)

        data = request.query_params.get('info', None)
        if data == "stock_list_detailed":            #Returns list of all their transactions with stocks they currently own.
            serializer = StockListSerializer({"stock_list":account.ownedStocks})
            return Response(serializer.data)
        # EDIT: need this for new stuff on frontend.
        elif data == "stocks_symbols": # Returns a list of just the symbols for user owned stocks.
            serializer = StockSymbolsSerializer({"stock_symbols":account.ownedStocks.keys()})
            return Response(serializer.data, status=status.HTTP_200_OK)
        elif data == "num_of_ticker_stocks":        #returns the amount of the stock they have for the given ticker
            symbol = request.query_params.get('symbol', None)
            numOfStock = UserStocks.countNumberOfSharesOwned(account.ownedStocks, symbol)
            serializer = StockNumSerializer({"quantity_owned":numOfStock})
            return Response(serializer.data)
        elif data == "portfolio_value":             #returns just the portfolio value
            stock_value = UserStocks.calculateCurrentPortfolioValue(account.ownedStocks)
            portfolio_change = UserStocks.calculatePortfolioChange(account.ownedStocks, stock_value)
            portfolio_value = Decimal.from_float(stock_value) + account.balance
            serializer = PortfolioValueSerializer({"portfolio_value": portfolio_value, "percent_change": portfolio_change, "change_direction": portfolio_change > 0})
            return Response(serializer.data)
        elif data == "stock_list_display":          #returns each stock they own with how much they own, price, %change, and changeDir
            stockList = StockLists.ownedStocks(account.ownedStocks)
            serializer = StockListSerializer({"stock_list":stockList})
            return Response(serializer.data)
        elif data == "buying_power_history":
            history = Balance.buildBuyingPowerHistory(account)
            serializer = TransactionHistorySerializer({"transaction_history":history})
            return Response(serializer.data)
        else:
            return Response(None, status=status.HTTP_400_BAD_REQUEST)

    '''
    Buys and sells stock and updates user list accordingly
    Requires data to have "action" set to either "buy" or "sell", the "stock" to be the symbol for the stock the user
    wishes to buy, and the "quantity" that they want of that stock.

    example:
    curl -d {"action":"buy", "stock":"tsla", "quantity":100} -H "Content-Type: application/json" -X PUT http://127.0.0.1:8000/accounts/[ACCOUNT_GOOG_ID]/getStocks

    ----This needs to be updated to work with the client sending the price
    ----This may need to include updating the balance
    '''
    def put(self, request, goog_id):
        (valid, data) =  getDataFromToken(request)
        if not valid:
            return data

        account = get_object(data["email"])
        if account == None:
            return Response(None, status=status.HTTP_404_NOT_FOUND)

        data = request.data
        valid = False
        if data["action"] == "buy":
            valid = Transaction.buy(data, account)
        elif data["action"] == "sell": # EDIT: this might accept nulls, try dif. err checking here
            valid = Transaction.sell(data, account)
        if not valid:
             return Response(None, status=status.HTTP_400_BAD_REQUEST)            #idk which error to put
        account.save()
        serializer = StockListSerializer({"stock_list":account.ownedStocks})
        return Response(serializer.data, status=status.HTTP_200_OK)

class AccountWatchList(APIView):
    def get(self, request, goog_id):
        (valid, data) =  getDataFromToken(request)
        if not valid:
            return data

        account = get_object(data["email"])
        data = request.query_params.get('info', None)

        if account == None:
            return Response(None, status=status.HTTP_404_NOT_FOUND)

        if data == "stocks": # Returns a list of just the symbols for watch list stocks
            serializer = StockListSerializer({"stock_list":account.watchList["stocks"]})
            return Response(serializer.data, status=status.HTTP_200_OK)
        elif data == "detailed_stocks": # Returns list with each stock's price, percent change, and change direction
            detailedList = StockLists.watchList(account.watchList)
            serializer = StockListSerializer({"stock_list":detailedList})
            return Response(serializer.data, status=status.HTTP_200_OK)
        elif data == "check_stock":
            serializer = BoolSerializer({"isPresent": request.query_params.get('symbol', None) in account.watchList["stocks"]})
            return Response(serializer.data, status=status.HTTP_200_OK)
        else:
            return Response(None, status=status.HTTP_400_BAD_REQUEST)

    #Acts as a switch. Simply make a put request with the "symbol" of the data being the
    #Ticker you wish to toggle. If the ticker is in the list it is removed, if it is
    #not in the list it is added. So if there is say a start symbol next to stocks,
    #clicking it will add it, but clicking it again will remove it.
    def put(self, request, goog_id):

        (valid, data) =  getDataFromToken(request)
        if not valid:
            return data

        account = get_object(data["email"])
        if account == None:
            return Response(None, status=status.HTTP_404_NOT_FOUND)
        symbol = request.data["symbol"]
        self.__toggleStockInWatchList(symbol, account)
        account.save()
        return Response(None, status=status.HTTP_204_NO_CONTENT)

    #Adds a stock into the watch list if not present, otherwise removes it
    def __toggleStockInWatchList(self, symbol, account):
        watched = self.__getWatchList(account)
        try:
            watched.pop(watched.index(symbol.upper()))
        except ValueError: #If the stock is not being watched, add it to the watch list
            watched.append(symbol.upper())

    #If the watch list hasn't been set up, set it up and then return the set
    def __getWatchList(self, account):
        if account.watchList == {} or account.watchList is None:
            account.watchList["stocks"] = []
        return account.watchList["stocks"]

class AccountStockIndustries(APIView):

    def get(self, request, goog_id):
        (valid, data) =  getDataFromToken(request)
        if not valid:
            return data

        account = get_object(data["email"])
        if account != None:
            symbols = list(account.ownedStocks.keys())
            industries = si.get_industries(symbols)
            diversity = self.__accumulate(industries)
            serializer = IndustryDiversitySerializer({"industry_makeup":diversity})
            return Response(serializer.data, status=status.HTTP_200_OK)
        else:
            return Response(None, status=status.HTTP_404_NOT_FOUND)

    def __accumulate(self, industries):
        counts = {}
        for industry in industries.values():
            if industry in counts.keys():
                counts[industry] += 1
            else:
                counts[industry] = 1
        return counts

class AccountHistoricPV(APIView):

    def get(self, request, goog_id):
        (valid, data) =  getDataFromToken(request)
        if not valid:
            return data

        account = get_object(data["email"])

        if account != None:
            balanceHistory = Balance.buildBuyingPowerHistory(account)
            Balance.addBalance(account.portfolio_value_history["data"], balanceHistory["data"])
            data = self.convertToListOfDicts(account.portfolio_value_history["data"])
            serializer = HistoricPortfolioValueSerializer({"pv":data})
            return Response(serializer.data, status=status.HTTP_200_OK)
        else:
            return Response(None, status=status.HTTP_404_NOT_FOUND)

    #converts the portfolio value history into a format that frontend likes
    def convertToListOfDicts(self, data):
        returnList = []
        keys = sorted(data)
        starting = data[keys[0]]
        for key in sorted(data):
            returnList.append({
                "date": key,
                "value": data[key],
                "value_change": data[key] - starting,
                "percent_change": 100*(data[key] - starting)/starting,
                "change_direction": data[key] - starting > 0
            })
        return returnList


import pygame
import os
import random
pygame.font.init()

WIN_WIDTH = 500
WIN_HEIGHT = 500

STAT_FONT = pygame.font.SysFont("comicsans", 50)

SNAKE_IMG = pygame.image.load(os.path.join("imgs", "Snake.png"))
FOOD_IMG = pygame.image.load(os.path.join("imgs", "Food.png"))
BG_IMG = pygame.image.load(os.path.join("imgs", "bg.png"))

class Snake:
    LENGTH = 1
    SNAKE = SNAKE_IMG

    def __init__(self):

        self.x = 250
        self.y = 250

    def draw(self, win):
        win.blit(self.SNAKE, (self.x, self.y))

class Food:

    def __init__(self):

        self.x = 0
        self.y = 0

        self.FOOD = FOOD_IMG

        self.set_position()

    def draw(self, win):
        win.blit(self.FOOD, (self.x, self.y))

    def set_position(self):
        self.x = random.randrange(0, 500)
        self.y = random.randrange(0, 500)

def draw_field(win, snake, foods, score):
    win.blit(BG_IMG, (0, 0))

    text = STAT_FONT.render("Score: " + str(score), True, (0, 255, 255))
    win.blit(text, (WIN_WIDTH - 10 - text.get_width(), 10))

    snake.draw(win)
    for food in foods:
        food.draw(win)

    pygame.display.update()

def main():

    clock = pygame.time.Clock()
    x = 50
    y = 50

    snake = Snake
    foods = [Food]
    score = 0

    win = pygame.display.set_mode((WIN_WIDTH, WIN_HEIGHT))

    while True:
        clock.tick(30)
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                pygame.quit()
                quit()
        draw_field(win, snake, foods, score)

main()


import csv
import boto3

with open('new_user_credentials.csv', 'r') as input:
    next(input)
    reader = csv.reader(input)
    for line in reader:
        access_key_id = line[2]
        secret_access_key = line[3]

photo = 'train.jpg'

# Creates a client of the rekognition API
client = boto3.client('rekognition',
                      aws_access_key_id=access_key_id,
                      aws_secret_access_key=secret_access_key)

with open(photo, 'rb') as source_image:
    source_bytes = source_image.read()

response = client.detect_labels(Image={'Bytes': source_bytes},
                                MaxLabels=10)

person = 'mercelis.jpg'

with open(person, 'rb') as source_image:
    source_bytes = source_image.read()

personResponse = client.detect_faces(Image={'Bytes': source_bytes},
                                     Attributes=['ALL'])

print("Train response: ", response)
print("Person response: ", personResponse)


from flask import Blueprint, render_template, abort
from jinja2 import TemplateNotFound
from blueprints import default_blueprint

@default_blueprint.route('/', defaults={'page': 'index'})
@default_blueprint.route('/<page>')
def show(page):
    try:
        return render_template('pages/%s.html' % page)
    except TemplateNotFound:
        abort(404)


import requests

from timer import timer

URL = 'https://httpbin.org/uuid'

def fetch(web, url):
    with web.get(url) as response:
        print(response.json()['uuid'])

@timer(1, 1)
def main():
    with requests.Session() as web:
        for i in range(100):
            fetch(web, URL)


import ee
import yaml
import time
import os
import json
import time
from argparse import ArgumentParser
from utils import clipToROI, exportImageCollectionToGCS, exportImageToGCS, sentinel2CloudScore, calcCloudCoverage, calcCloudCoverageTile, inject_B10, sentinel2ProjectShadows, computeQualityScore, mergeCollection
from utils import GEETaskManager

from gevent.fileobject import FileObjectThread

def makeFilterList(sensor):
	filters_before = None
	filters_after = None

	def _build_filters(filter_list):
		filters = []
		for f in filter_list:
			key = list(f.keys())[0]
			op = list(list(f.values())[0].keys())[0]
			val = list(list(f.values())[0].values())[0]
			filters.append(getattr(ee.Filter, op)(key, val))

		return filters

	if 'filters_before' in sensor:
		filters_before = _build_filters(sensor['filters_before'])

	if 'filters_after' in sensor:
		filters_after = _build_filters(sensor['filters_after'])

	return filters_before, filters_after

def makeImageCollection(sensor, roi, start_date, end_date, modifiers=[]):
	filters_before, filters_after = makeFilterList(sensor)
	print(modifiers)

	collection = ee.ImageCollection(sensor['name']) \
				.filterDate(ee.Date(start_date), ee.Date(end_date)) \
				.filterBounds(roi) \
				.map(lambda x: clipToROI(x, ee.Geometry(roi)))

	print("size of collection:", collection.size().getInfo())

	if filters_before is not None:
		collection = collection.filter(filters_before)

	if modifiers and len(modifiers) > 0:
		for m in modifiers:
			print(f'Applying modifier {m}')
			collection = collection.map(m)

	if filters_after:
		collection = collection.filter(filters_after)

	return collection

def makeTileCollection(sensor, tile, main_collection, modifiers=[]):
    filters_before, filters_after = makeFilterList(sensor)

    collection = main_collection.filterMetadata(
        'system:index', 'contains', tile)

    if filters_before is not None:
        collection = collection.filter(filters_before)

    if modifiers and len(modifiers) > 0:
        for m in modifiers:
            collection = collection.map(m)

    if filters_after:
        collection = collection.filter(filters_after)

    return collection

def process_datasource(
    source,
    sensor,
    export_folder,
    feature_list=None,
     pre_mosaic_sort='CLOUDY_PERCENTAGE'):
# 	feature_list = ee.FeatureCollection(source['features_src'])
	feature_list = feature_list.sort(
    source['sort_by']).toList(
        feature_list.size())
	n_features = feature_list.size().getInfo()

	print("{} features have been loaded".format(n_features))

	# task_list = []

	exports = []

	### ERROR? ###
	# Originally this was range(1, n_features), but we're pretty sure
	# that should be 0 so we changed it.
	for i in range(0, n_features):
		feature_point = ee.Feature(feature_list.get(i))

		# if source['geometry'] == "point":
		#	feature_point = feature_point.buffer(source['size']).bounds()

		roi = feature_point.geometry()
		roi = roi.coordinates().getInfo()

		### should be done outside the for loop ###
		if isinstance(source['name'], str):
			source['name'] = [source['name']]

		### ERROR? ###
		# The following conditional should be moved under
		# the conditional after it, or else we'll error out
		# if sensor doesn't have a "prefix" key.
		if isinstance(sensor['prefix'], str):
			sensor['prefix'] = [sensor['prefix']]

		if 'prefix' in sensor:
			filename_parts = sensor['prefix'] + source['name']
		else:
			filename_parts = source['name']
		### end of part that should be done outside the for loop ###

		time_stamp = "_".join(time.ctime().split(" ")[1:])
		filename = "_".join([str(i + 1)] + source['name'] + [time_stamp])
		print("processing ", filename)
		dest_path = "/".join(filename_parts + [filename])

		export_params = {
			'bucket': export_folder,
			'resolution': source['resolution'],
			'filename': filename,
			'dest_path': dest_path
		}

		# task_params = {
		# 	'action': export_single_feature,
		# 	'id': "_".join(filename_parts + [str(i)]), # This must be unique per task, to allow to track retries
		# 	'kwargs': {
		# 		'roi': roi,
		# 		'export_params': export_params,
		# 		'sensor': sensor,
		# 		'date_range': {'start_date': source['start_date'], 'end_date': source['end_date'],
		# 		'sort_by': pre_mosaic_sort}
		# 	}
		# }

		# task_queue.add_task(task_params, blocking=True)

		export = export_single_feature(
			roi=roi,
			sensor=sensor,
			date_range={
    'start_date': source['start_date'],
     'end_date': source['end_date']},
			export_params=export_params,
			sort_by=pre_mosaic_sort
		)

		exports.append(export)

	return exports

def process_datasource_tiles(
    source,
    sensor,
    export_folder,
    tile_hash_table,
    main_polygon=None,
    pre_mosaic_sort='CLOUDY_PERCENTAGE',
	limit=None):

    tile_list = list(tile_hash_table.keys())

    n_tiles = len(tile_list)

    print(f'{n_tiles} tiles have been loaded')

    exports = []

    start_date = source['start_date']
    end_date = source['end_date']

    tile = tile_list[0]
    roi = json.loads(tile_hash_table[tile]['Polygon'])
    #print('ROI:', roi)
    roi_ee = ee.Geometry.Polygon(roi)

    main_collection = ee.ImageCollection(sensor['name']) \
                        .filterDate(ee.Date(start_date), ee.Date(end_date)) \
						.filterBounds(roi_ee)

    # if main_polygon:
    #     main_polygon = ee.Geometry.Polygon(main_polygon)
    #     main_collection = main_collection.filterBounds(main_polygon)

    if isinstance(source['name'], str):
        source['name'] = [source['name']]

    if 'prefix' in sensor:
        if isinstance(sensor['prefix'], str):
            sensor['prefix'] = [sensor['prefix']]
        filename_parts = sensor['prefix'] + source['name']
    else:
        filename_parts = source['name']

    if not limit:
        limit = n_tiles

    for i in range(limit):
        tile = tile_list[i]
        print(f'Processing tile {tile}')

        time_stamp = "_".join(time.ctime().split(" ")[1:])
        filename = "_".join([str(i + 1)] + source['name'] + [time_stamp])
        print("processing ",filename)
        dest_path = "/".join(filename_parts + [filename])

        # roi = json.loads(tile_hash_table[tile]['Polygon'])
        # print('ROI:', roi)
        # roi_ee = ee.Geometry.Polygon(roi)

        export_params = {
            'bucket': export_folder,
            'resolution': source['resolution'],
            'filename': filename,
            'dest_path': dest_path,
			#'roi': roi_ee
        }

        export = export_single_tile(
            tile=tile,
            main_collection=main_collection,
            main_polygon=main_polygon,
            sensor=sensor,
            export_params=export_params,
            sort_by=pre_mosaic_sort
        )

        exports.append(export)

    return exports

def export_single_feature(roi=None, sensor=None, date_range=None, export_params=None, sort_by='CLOUDY_PERCENTAGE'):
	modifiers = []
	if sensor['name'].lower() == "copernicus/s2_sr":
		print('Inject B10')
		modifiers.append(inject_B10)
	if sensor['type'].lower() == "opt":
		# print(sensor['type'])
		modifiers += [sentinel2CloudScore, calcCloudCoverage, sentinel2ProjectShadows, computeQualityScore]
		print(modifiers)

	# print('Modifiers:', modifiers)
	roi_ee = ee.Geometry.Polygon(roi[0])
	image_collection = makeImageCollection(sensor, roi_ee, date_range['start_date'], date_range['end_date'], modifiers=modifiers)
	# sort was not in the original version
	image_collection = image_collection.sort(sort_by)
	# below line was in the original verson;
	# changing to the JS version
	# img = image_collection.mosaic().clip(roi_ee)
	cloudFree = mergeCollection(image_collection).clip(roi_ee)
	cloudFree = cloudFree.reproject('EPSG:4326', None, 10)
	# Do we need to mosaic it now???
# 	print('cloudFree info:', cloudFree.getInfo())
	# print('Mosaic type:', type(img))

	new_params = export_params.copy()
	new_params['img'] = cloudFree
	new_params['roi'] = roi
	new_params['sensor_name'] = sensor['name'].lower()

	return exportImageToGCS(**new_params)

def export_single_tile(tile, main_collection, main_polygon=None, sensor=None, export_params=None, sort_by='CLOUDY_PERCENTAGE'):
    modifiers = []
    if sensor['name'].lower() == "copernicus/s2_sr":
        print('Inject B10')
        modifiers.append(inject_B10)
    if sensor['type'].lower() == "opt":
        # print(sensor['type'])
        modifiers += [sentinel2CloudScore, calcCloudCoverage, sentinel2ProjectShadows, computeQualityScore]

    print('Making tile collection')
    tile_collection = makeTileCollection(sensor, tile, main_collection, modifiers=modifiers)
    tile_collection = tile_collection.sort(sort_by)

    print('Making mosaic')
    cloudFree = mergeCollection(tile_collection)

    #cloudFree = cloudFree.clip(export_params['roi'])
    # if main_polygon:
    #     print('Clipping to polygon')
    #     cloudFree = cloudFree.clip(main_polygon)
    cloudFree = cloudFree.reproject('EPSG:4326', None, 10)

    new_params = export_params.copy()
    new_params['img'] = cloudFree
    # if main_polygon:
    # 	new_params['roi'] = main_polygon
    new_params['sensor_name'] = sensor['name'].lower()

    # return cloudFree

    return exportImageToGCS(**new_params)

def _serialise_task_log(task_log):
	for k,v in task_log.items():
		task_log[k]['task_def']['action'] = "export_single_feature"

	return task_log

def load_task_log(filename='task_log.json'):
	with open(filename, 'r') as f:
		task_log = json.load(f)

	for k, v in task_log.items():
		task_log[k]['task_def']['action'] = globals()[task_log[k]['task_def']['action']]

	return task_log

def monitor_tasks(task_log):
	print("SAVING LOG")
	f_raw = open('task_log.json', 'w')
	with FileObjectThread(f_raw, 'w') as handle:
		task_log = _serialise_task_log(task_log)
		json.dump(task_log, handle)

	f_raw.close()

# def load_config(path):
# 	with open(path, 'r') as stream:
# 		try:
# 			return yaml.load(stream)
# 		except yaml.YAMLError as exc:
# 			print(exc)

def load_config(config_file):
    stream = open(config_file, 'r')
    return yaml.load(stream)

if __name__ == "__main__":
	parser = ArgumentParser()
	parser.add_argument("-c", "--config", default=None, help="Config file for the download")
	args = parser.parse_args()

	assert args.config, "Please specify a config file for the download"
	config = load_config(args.config)
	print(config)

	ee.Initialize()

	task_queue = GEETaskManager(n_workers=config['max_tasks'], max_retry=config['max_retry'], wake_on_task=True, log_file=config['log_file'], process_timeout=config['task_timeout'])
	task_queue.register_monitor(monitor_tasks)

	if os.path.exists('task_log.json'):
		task_log = load_task_log(filename='task_log.json')
		task_queue.set_task_log(task_log)

	pre_mosaic_sort = config['pre_mosaic_sort']
	for data_list in config['data_list']:
		for sensor_idx in data_list['sensors']:
			sensor = config['sensors'][sensor_idx]
			tasks = process_datasource(task_queue, data_list, sensor, config['export_to'], config['export_dest'], pre_mosaic_sort)

	print("Waiting for completion...")
	task_queue.wait_till_done()


from flask import render_template, request, jsonify
import requests
import pycurl
from io import BytesIO
import json
from __main__ import *
import time
import pandas as pd

#Get the URL of the Proxy
cURL = pycurl.Curl()
rm_url = 'http://10.140.17.105:3000'

def index():
    # Making sure the cloud is online
    try:
        stat = get_cloud_status()
        init = "Connected"

    except Exception as e:
        print(e)
        stat = []
        init = "Error Clound not Launched"

    lt_avail = "0"
    md_avail = "0"
    hv_avail = "0"
    if len(stat) > 0:
        nd_ls = get_cloud_nodes(stat)
        for nodes in nd_ls:
            if 'LIGHT' == nodes['Pod']:
                lt_avail = str(len(nodes) - 1)
            if 'MEDIUM' == nodes['Pod']:
                md_avail = str(len(nodes) - 1)
            else:
                hv_avail = str(len(nodes) - 1)

    #get HAProxy stats
    lt_stats, md_stats, hv_stats = get_proxy_stats()
    total_req = int(lt_stats["requests"]) + int(md_stats["requests"]) + int(hv_stats["requests"])

    return render_template("index.html", cloud_status=init, total_requests=total_req,
                           light_avail=lt_avail, med_avail=md_avail, heavy_avail=hv_avail,
                           lt_resumed=lt_stats["resumed"], lt_process=lt_stats["processing"], lt_request=lt_stats["requests"],
                           med_resumed=md_stats["resumed"], med_process=md_stats["processing"], med_request=md_stats["requests"],
                           hv_resumed=hv_stats["resumed"], hv_process=hv_stats["processing"], hv_request=hv_stats["requests"])

def stats():
    data = BytesIO()
    cURL.setopt(cURL.URL, rm_url + f'/cloud/haproxy/stats')
    cURL.setopt(cURL.WRITEFUNCTION, data.write)
    cURL.perform()
    dct = json.loads(data.getvalue())

    return render_template("stats.html", info_dct=dct, len=len(dct["pxname"]))

def clusters():
    dct = get_cloud_nodes()
    val = []
    for k in dct:
        l = dct[k].split(",")
        name = l[0].split()[1]
        id_num = l[1].split()[1]
        num_nodes = l[2].split()[1]
        val.append([name + id_num, id_num, num_nodes])

    return render_template("clusters.html", lst=val)

def pods(pod_id):
    dct = get_cloud_nodes([pod_id])
    if len(dct) > 0:
        dct = dct[0]
        dct.pop('Pod')
    val = []

    data = BytesIO()
    cURL.setopt(cURL.URL, rm_url + f'/dashboard/status/{pod_id}')
    cURL.setopt(cURL.WRITEFUNCTION, data.write)
    cURL.perform()
    dct_2 = json.loads(data.getvalue())
    paused = dct_2['result']

    if pod_id == 'L':
        pod_id = 'Light Pod'
    elif pod_id == 'M':
        pod_id = 'Medium Pod'
    else:
        pod_id = 'Heavy Pod'

    if paused:
        paused = "Paused"
    else:
        paused = "Resumed"

    for k in dct:
        l = dct[k].split(" - ")
        print(l)
        id_num = k
        nd_name = l[0].split()[1]
        num_port = l[1].split()[1]
        nd_status = l[2].split()[1]
        val.append([nd_name, id_num, num_port, nd_status])

    val.sort(key=lambda v : int(v[1]))

    proxy_dct = get_running_jobs()

    for ls in val:
        if ls[0] in proxy_dct["svname"]:
            for i in range(len(proxy_dct["svname"])):
                if proxy_dct["svname"][i] == ls[0]:
                    if proxy_dct["scur"][i] == "0":
                        ls.append(False)
                    else:
                        ls.append(True)
                    ls.append(proxy_dct["stot"][i])

    return render_template("cluster_overview.html", pod_id=pod_id, lst=val, running=paused, unique_id=pod_id[0])

#--------------------------HELPER FUNCTIONS-------------------------
def get_nodes_info(node_ls):
    pass

def get_cloud_status():
    pod_ls = ("L", "M", "H")

    result = []

    for id in pod_ls:
        #Logic to invoke RM-Proxy
        data = BytesIO()
        cURL.setopt(cURL.URL, rm_url + f'/cloud/node/ls/{id}')
        cURL.setopt(cURL.WRITEFUNCTION, data.write)
        cURL.perform()
        dct = json.loads(data.getvalue())
        print(dct)

        if (dct['result'] == 'Success'):
            result.append(id)

    return result

def get_cloud_nodes(pod_ls):
    result = []

    for id in pod_ls:
        #Logic to invoke RM-Proxy
        data = BytesIO()

        cURL.setopt(cURL.URL, rm_url + f'/cloud/node/ls/{id}')
        cURL.setopt(cURL.WRITEFUNCTION, data.write)
        cURL.perform()
        dct = json.loads(data.getvalue())

        if (dct['result'] == 'Success'):
            dct.pop('result')
            result.append(dct)

    return result

def get_running_jobs():
    #request HAProxy stats from Resource Manager
    data = BytesIO()
    cURL.setopt(cURL.URL, rm_url + '/cloud/haproxy/stats')
    cURL.setopt(cURL.WRITEFUNCTION, data.write)
    cURL.perform()
    #converts into a dictionary
    data_dct = json.loads(data.getvalue())

    print(data_dct)

    return data_dct

#Process HAProxy stats for the different pods
def get_proxy_stats():
    dct = get_running_jobs()

    lt_stats = {"resumed": -1, "processing": 0, "requests": 0}
    md_stats = {"resumed": -1, "processing": 0, "requests": 0}
    hv_stats = {"resumed": -1, "processing": 0, "requests": 0}

    i = -1
    # getting current number of requests for heavy
    for v in dct["pxname"]:
        i += 1
        if v == "heavy-servers" and dct["svname"][i] == "BACKEND":
            hv_stats["processing"] = dct["scur"][i]
            hv_stats["resumed"] += 1
            hv_stats["requests"] = dct["stot"][i]

        elif v == "heavy-servers":
            hv_stats["resumed"] += 1

        if v == "light-servers" and dct["svname"][i] == "BACKEND":
            lt_stats["processing"] = dct["scur"][i]
            lt_stats["resumed"] += 1
            lt_stats["requests"] = dct["stot"][i]

        elif v == "light-servers":
            lt_stats["resumed"] += 1

        if v == "medium-servers" and dct["svname"][i] == "BACKEND":
            md_stats["processing"] = dct["scur"][i]
            md_stats["resumed"] += 1
            md_stats["requests"] = dct["stot"][i]

        elif v == "medium-servers":
            md_stats["resumed"] += 1

    return lt_stats, md_stats, hv_stats


from .solvers import DefaultSolver
import itertools

N = 25

class Solver(DefaultSolver):
    @staticmethod
    def process_input(lines):
        return list(map(int, lines))

    @staticmethod
    def part_1(data):
        return get_bad_num(data)

    @staticmethod
    def part_2(data):
        target = get_bad_num(data)

        result = get_summing_nums(data, target)
        if result:
            start, end = result
            nums = data[start:end]
            return min(nums) + max(nums)

        return None

def get_summing_nums(data, target):
    total = 0
    j = 0
    for i in range(len(data)):
        while total < target:
            j += 1
            total += data[j - 1]
        if total == target:
            return i, j
        total -= data[i]
    return None

def get_bad_num(data):
    for idx in range(N, len(data)):
        prev_set = set(data[idx - N : idx])
        if not get_matching_set(prev_set, 2, data[idx]):
            return data[idx]
    return None

def get_matching_set(number_set, count, _sum):
    for tpl in itertools.combinations(number_set, count):
        if sum(tpl) == _sum:
            return tpl
    return None


import pandas as pd
from sklearn.linear_model import LinearRegression

df = pd.read_csv('carprices.csv')

# prepare data
dummies = (pd.get_dummies(df['Car Model']).astype(int))

df = pd.concat([df,dummies],axis=1)

df = df.drop(['Car Model','Mercedez Benz C class'],axis=1)
print(df)

X = df.drop('Sell Price($)',axis=1).values
y = df['Sell Price($)'].values

# train
model = LinearRegression()
model.fit(X,y)

# predict
print(model.predict([[45000,4,0,0]]))
print(model.predict([[86000,7,0,1]]))
print(model.score(X,y))

v5 - Dummy Variables & One Hot Encoding/exercise.py


#!/bin/python3

# Require: yum install bind-utils jq (dig command)
# Require: python -m pip install paramiko

from telegram import ReplyKeyboardMarkup, ReplyKeyboardRemove, Update, ParseMode
from telegram.ext import (
    Updater,
    CommandHandler,
    MessageHandler,
    Filters,
    ConversationHandler,
    CallbackContext,
)
import os, logging, json
from datetime import date,datetime

from LIB_botTelegram import aaPanel

#Global Variables
gensslStep = 0
DOMAIN, COMPANY, CITY, EMAIL = range(4)
domainG=""
companyG=""
cityG=""
emailG=""

# Enable logging
logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
                    level=logging.INFO)
logger = logging.getLogger(__name__)

def check(update, context):
    chat_id = update.message.chat_id
    statement = "sh LIB_botTelegram/check.sh " + str(chat_id)
    os.system(statement)

def ssl(update, context):
    domain = str(" ".join(str(update.message.text).strip().split())).split()[1]
    #print("Message: ", " ".join(str(update.message.text).strip().split()))
    chat_id = update.message.chat_id
    statement = "sh LIB_botTelegram/ssl.sh " + str(chat_id) + " " + str(domain)
    os.system(statement)
    #update.message.reply_text("This is the SSL")

# CONVERSATION

def genssl(update: Update, context: CallbackContext) -> int:
    #Starts the conversation and ask for domain
    update.message.reply_text(
        'Welcome to IT CHICKEN BOT, I will GEN new CSR, KEY for your domain!'
        '\nSend /cancel to stop talking to me.\n\n'
        'What is your domain?',parse_mode=ParseMode.HTML
        # reply_markup=ReplyKeyboardMarkup(
        #     one_time_keyboard=True, input_field_placeholder='What is your domain?'
        # ),
    )

    return DOMAIN

def domain(update: Update, context: CallbackContext) -> int:
    #Get domain and ask for company
    global domainG
    domainG = update.message.text
    update.message.reply_text(
        'Your domain: ' + domainG + ''
        '\nSend /cancel to stop talking to me.\n\n'
        'What is your company?',parse_mode=ParseMode.HTML
        # reply_markup=ReplyKeyboardMarkup(
        #     one_time_keyboard=True,input_field_placeholder='What is your company?'
        # ),
    )

    return COMPANY

def company(update: Update, context: CallbackContext) -> int:
    #Get company and ask for city
    global companyG
    companyG = update.message.text
    update.message.reply_text(
        'Your company: ' + companyG + ''
        '\nSend /cancel to stop talking to me.\n\n'
        'What is your city?',parse_mode=ParseMode.HTML
        # reply_markup=ReplyKeyboardMarkup(
        #     one_time_keyboard=True, input_field_placeholder='What is your city?'
        # ),
    )

    return CITY

def city(update: Update, context: CallbackContext) -> int:
    #Get city and ask for email
    global cityG
    cityG = update.message.text
    update.message.reply_text(
        'Your city: ' + cityG + ''
        '\nSend /cancel to stop talking to me.\n\n'
        'What is your email?',parse_mode=ParseMode.HTML
        # reply_markup=ReplyKeyboardMarkup(
        #     one_time_keyboard=True, input_field_placeholder='What is your email?'
        # ),
    )

    return EMAIL

def email(update: Update, context: CallbackContext) -> int:
    #Get email and send csr, key
    global emailG
    emailG = update.message.text
    update.message.reply_text(
        'Your email: ' + emailG + ''
        '\nSend /cancel to stop talking to me.\n\n'
        'Your CSR, KEY will be here now!',parse_mode=ParseMode.HTML
    )
    genkey(update, context)
    return ConversationHandler.END

def genkey(update: Update, context: CallbackContext) -> int:
    #Gen and send key
    # context.bot.send_message(text=
    #     'Your domain:   ' + domainG + ''
    #     '\nYour comany:   ' + companyG + ''
    #     '\nYour city:     ' + cityG + ''
    #     '\nyour email:    ' + emailG + '', chat_id=update.message.chat_id, parse_mode=ParseMode.HTML
    # )
    try:
        chat_id = update.message.chat_id
        statement = "sh LIB_botTelegram/genSSL.sh " + str(chat_id) + " \"" + str(domainG) + "\" \"" + str(companyG) + "\" \"" + str(cityG) + "\" \"" + str(emailG) + "\""
        os.system(statement)
    except:
        context.bot.send_message(text="⚠ Can't Create CSR, KEY for " + domainG + " ⚠", chat_id=update.message.chat_id, parse_mode=ParseMode.HTML)
        return ConversationHandler.END

def cancel(update: Update, context: CallbackContext) -> int:
    #Cancels and ends the conversation.
    user = update.message.from_user
    logger.info("User %s canceled the conversation.", user.first_name)
    global domainG
    global companyG
    global cityG
    global emailG
    # RESET VARIABLES
    domainG=""
    companyG=""
    cityG=""
    emailG=""

    update.message.reply_text(
        'Creating SSL KEY PAIR task has cancelled!', reply_markup=ReplyKeyboardRemove()
    )

    return ConversationHandler.END

# END CONVERSATION

def error(update, context):
    """Log Errors caused by Updates."""
    logger.warning('Update "%s" caused error "%s"', update, context.error)

def echo(update, context):
    #update.message.reply_text(update.message.text)
    return 0

def bank(update, context):
    id = str(update.message.text).split(" ")[1]
    filePath = "./LIB_botTelegram/db.json"
    if ( not os.path.isfile(filePath)):
        update.message.reply_text("Bank ID invalid!!")
    f = open(filePath, )
    userData = json.load(f)
    f.close()
    if (userData.get(id, -1) == -1):
        update.message.reply_text("Bank ID invalid!!")

    total = 0
    resultBill="🏛 Checking profit for " + userData[id][0].get('name') + " 🏛\n==========\n"
    for bill in userData[id]:
        cost = bill.get('cost')
        date_format = "%d/%m/%Y"
        startDate = datetime.strptime(bill.get('date'), date_format)
        now = datetime.today()
        numDates = now - startDate
        numDate = numDates.days
        pay = bill.get('pay')
        currency = "{:,.2f} VND".format(cost*1000)
        profit = "{:,.2f} VND".format(cost*1000*numDate*pay/365)
        resultBill += " + Capital: " + currency + "\n + Start Date: " + startDate.strftime(date_format) + "\n + Days: " + str(numDate) + "\n + IR: " + str(pay*100 )+ "%\n + Profit: " + profit + "\n----------\n"
        total += cost*1000*(numDate*pay/365 + 1)

    resultBill += " * Total: " + "{:,.2f} VND".format(total)
    update.message.reply_text(resultBill)

def dm(update, context):
    domain = str(" ".join(str(update.message.text).strip().split())).split()[1]
    #domain = domain.strip()
    chat_id = update.message.chat_id
    statement = "sh LIB_botTelegram/checkDomain.sh " + str(chat_id) + " " + str(domain) + " " + "O"
    os.system(statement)
    #update.message.reply_text("This is the SSL")
    return 0

def curl(update, context):
    url = str(" ".join(str(update.message.text).strip().split())).split()[1]
    chat_id = update.message.chat_id
    statement = "sh LIB_botTelegram/curl.sh " + str(chat_id) + " " + str(url)
    os.system(statement)
    #update.message.reply_text("This is the SSL")
    return 0

def dmrecord(update, context):
    domain = str(" ".join(str(update.message.text).strip().split())).split()[1]
    chat_id = update.message.chat_id
    statement = "sh LIB_botTelegram/checkDomain.sh " + str(chat_id) + " " + str(domain) + " " + "F"
    os.system(statement)
    #update.message.reply_text("This is the SSL")
    return 0

def passWord(update, context):
    try:
        length = str(update.message.text).split(" ")[1]
        if (int(length) <= 0 or int(length) >= 500):
            update.message.reply_text("Getshit DONE!!!")
            return 0
    except:
        length = 0
    chat_id = update.message.chat_id
    statement = "sh LIB_botTelegram/randomPass.sh " + str(chat_id) + " " + str(length)
    os.system(statement)
    return 0

def ping(update, context):
    ip_check = str(" ".join(str(update.message.text).strip().split())).split()[1]
    try:
        options = str(" ".join(str(update.message.text).strip().split())).split()[2]
    except:
        options = "L"
    if (len(str(options)) <= 0): options = "L"
    #print("Options: ", options)
    options = options.strip()
    chat_id = update.message.chat_id
    statement = "sh LIB_botTelegram/checkIP.sh " + str(chat_id) + " " + str(ip_check) + " " + str(options).upper()
    os.system(statement)
    return 0

def tag(update, context):
    mode = str(" ".join(str(update.message.text).strip().split())).split()[1]
    listUser = {'Nam Lưu': 1076498380, 'Tuấn Đặng': 1753149166, 'Huy Ngô': 1458296682, 'Hải Anh': 1740013393, 'Đạt Trần': 2138787778, 'Hoà Lê': 1800256365, 'Thanh Hoàng': 5567151417, 'Doãn Anh': 1211523861}
    result = ''
    if len(mode) <= 0: mode = 'err'
    if mode == 'all':
        for user in listUser:
            #print(listUser[user])
            result += "["+ user +"](tg://user?id="+str(listUser[user])+"), "
        #user = "[Tuan Dang](tg://user?id="+str(id)+")"

        update.message.reply_text(result, parse_mode='markdown')
    else:
        update.message.reply_text('Wrong command!', parse_mode='markdown')
    return 0

def id(update, context):
    first_name = update.message.from_user.first_name
    id = update.message.from_user.id
    last_name = update.message.from_user.last_name
    username = update.message.from_user.username
    result = (
        f"Hello @{username}! Welcome to IT Chicken Bot\n"
        f"Your ID: {id}\n"
        f"Your name: {first_name} {last_name}"
    )
    update.message.reply_text(result)
    return 0

def roomid(update, context):
    id = update.message.chat.id
    name = update.message.chat.title
    result = (
        f"Hello @! Welcome to IT Chicken Bot\n"
        f"Room ID: {id}\n"
        f"Room name: {name}"
    )
    update.message.reply_text(result)
    return 0

def main():
    updater = Updater('1741302312:AAHUJEV2WsKzCu8wBF6Uq9zwBPL7F724wYo', use_context=True)
    dp = updater.dispatcher
    dp.add_handler(CommandHandler('check',check))
    dp.add_handler(CommandHandler('ssl', ssl))

    # Domain check
    dp.add_handler(CommandHandler('dm', dm))
    dp.add_handler(CommandHandler('dmrecord', dmrecord))

    # Curl
    dp.add_handler(CommandHandler('curl', curl))

    #bank
    dp.add_handler(CommandHandler('bank', bank))

    #pass
    dp.add_handler(CommandHandler('pass', passWord))

    #tag
    dp.add_handler(CommandHandler('tag', tag))

    #id
    dp.add_handler(CommandHandler('id', id))

    #roomid
    dp.add_handler(CommandHandler('roomid', roomid))

    #check International
    dp.add_handler(CommandHandler('ping', ping))

    # Add conversation handler for create SSL
    genssl_conv_handler = ConversationHandler(
        entry_points=[CommandHandler('genssl', genssl)],
        states={
            DOMAIN: [MessageHandler(Filters.text & ~Filters.command, domain)],
            COMPANY: [MessageHandler(Filters.text & ~Filters.command, company)],
            CITY: [MessageHandler(Filters.text & ~Filters.command, city)],
            EMAIL: [MessageHandler(Filters.text & ~Filters.command, email)],
        },
        fallbacks=[CommandHandler('cancel', cancel)],
    )
    dp.add_handler(genssl_conv_handler)

     # on noncommand i.e message - echo the message on Telegram
    dp.add_handler(MessageHandler(Filters.text, echo))

    # log all errors
    dp.add_error_handler(error)
    updater.start_polling()
    updater.idle()

main()


import datetime
from collections import namedtuple

import openPLM.plmapp.models as models
from openPLM.plmapp.units import DEFAULT_UNIT
from openPLM.plmapp.controllers.plmobject import PLMObjectController
from openPLM.plmapp.controllers.base import get_controller

Child = namedtuple("Child", "level link")
Parent = namedtuple("Parent", "level link")

class PartController(PLMObjectController):
    u"""
    Controller for :class:`.Part`.

    This controller adds methods to manage Parent-Child links between two
    Parts.
    """

    def check_add_child(self, child):
        """
        Checks if *child"* can be added to *self*.
        If *child* can not be added, an exception is raised.

        :param child: child to be added
        :type child: :class:`.Part`

        :raises: :exc:`ValueError` if *child* is already a child or a parent.
        :raises: :exc:`.PermissionError` if :attr:`_user` is not the owner of
            :attr:`object`.
        """
        self.check_permission("owner")
        self.check_editable()
        if not child.is_part:
            raise TypeError("Can not add child: not a Part")
        # check if child is not a parent
        if child.id == self.object.id:
            raise ValueError("Can not add child: child is current object")
        get_controller(child.type)(child, self._user).check_readable()
        parents = (p.link.parent.pk for p in self.get_parents(-1))
        if child.pk in parents:
            raise ValueError("Can not add child %s to %s, it is a parent" %
                                (child, self.object))
        link = self.parentchildlink_parent.filter(child=child, end_time=None)
        if link:
            raise ValueError("Can not add child, %s is already a child of %s" %
                                (child, self.object))

    def can_add_child(self, child):
        """
        Returns True if *child* can be added to *self*.
        """

        can_add = False
        try:
            self.check_add_child(child)
            can_add = True
        except StandardError:
            pass
        return can_add

    def add_child(self, child, quantity, order, unit=DEFAULT_UNIT, **extension_data):
        """
        Adds *child* to *self*.

        :param child: added child
        :type child: :class:`.Part`
        :param quantity: amount of *child*
        :type quantity: positive float
        :param order: order
        :type order: positive int
        :param unit: a valid unit

        :raises: :exc:`ValueError` if *child* is already a child or a parent.
        :raises: :exc:`ValueError` if *quantity* or *order* are negative.
        :raises: :exc:`.PermissionError` if :attr:`_user` is not the owner of
            :attr:`object`.
        :raises: :exc:`.PermissionError` if :attr:`object` is not editable.
        """

        if isinstance(child, PLMObjectController):
            child = child.object
        self.check_add_child(child)
        # check if child is not already a direct child
        if child.pk in (c.link.child.pk for c in self.get_children(1)):
            raise ValueError("%s is already a child of %s" % (child, self.object))
        if order < 0 or quantity < 0:
            raise ValueError("Quantity or order is negative")
        # data are valid : create the link
        link = models.ParentChildLink()
        link.parent = self.object
        link.child = child
        link.quantity = quantity
        link.order = order
        link.unit = unit
        link.save()
        # handle plces
        for PCLE in models.get_PCLEs(self.object):
            name = PCLE._meta.module_name
            if name in extension_data and PCLE.one_per_link():
                ext = PCLE(link=link, **extension_data[name])
                ext.save()
        # records creation in history
        self._save_histo(link.ACTION_NAME,
                         "parent : %s\nchild : %s" % (self.object, child))

    def delete_child(self, child):
        u"""
        Deletes *child* from current children and records this action in the
        history.

        .. note::
            The link is not destroyed: its end_time is set to now.

        :raises: :exc:`.PermissionError` if :attr:`_user` is not the owner of
            :attr:`object`.
        :raises: :exc:`.PermissionError` if :attr:`object` is not editable.
        """

        self.check_permission("owner")
        self.check_editable()
        if isinstance(child, PLMObjectController):
            child = child.object
        link = self.parentchildlink_parent.get(child=child, end_time=None)
        link.end_time = datetime.datetime.today()
        link.save()
        self._save_histo("Delete - %s" % link.ACTION_NAME, "child : %s" % child)

    def modify_child(self, child, new_quantity, new_order, new_unit,
            **extension_data):
        """
        Modifies information about *child*.

        :param child: added child
        :type child: :class:`.Part`
        :param new_quantity: amount of *child*
        :type new_quantity: positive float
        :param new_order: order
        :type new_order: positive int

        :raises: :exc:`.PermissionError` if :attr:`_user` is not the owner of
            :attr:`object`.
        :raises: :exc:`.PermissionError` if :attr:`object` is not editable.
        """

        self.check_permission("owner")
        self.check_editable()
        if isinstance(child, PLMObjectController):
            child = child.object
        if new_order < 0 or new_quantity < 0:
            raise ValueError("Quantity or order is negative")
        link = models.ParentChildLink.objects.get(parent=self.object,
                                                  child=child, end_time=None)
        original_extension_data = link.get_extension_data()

        if (link.quantity == new_quantity and link.order == new_order and
            link.unit == new_unit and original_extension_data == extension_data):
            # do not make an update if it is useless
            return
        link.end_time = datetime.datetime.today()
        link.save()
        # make a new link
        link2, extensions = link.clone(quantity=new_quantity, order=new_order,
                       unit=new_unit, end_time=None, extension_data=extension_data)
        details = ""
        if link.quantity != new_quantity:
            details += "quantity changes from %f to %f\n" % (link.quantity, new_quantity)
        if link.order != new_order:
            details += "order changes from %d to %d" % (link.order, new_order)
        if link.unit != new_unit:
            details += "unit changes from %s to %s" % (link.unit, new_unit)

        # TODO: details of extension changes

        self._save_histo("Modify - %s" % link.ACTION_NAME, details)
        link2.save(force_insert=True)
        # save cloned extensions
        for ext in extensions:
            ext.link = link2
            ext.save(force_insert=True)
        # add new extensions
        for PCLE in models.get_PCLEs(self.object):
            name = PCLE._meta.module_name
            if (name in extension_data and name not in original_extension_data
                and PCLE.one_per_link()):
                ext = PCLE(link=link2, **extension_data[name])
                ext.save()

    def get_children(self, max_level=1, current_level=1, date=None):
        """
        Returns a list of all children at time *date*.

        :rtype: list of :class:`Child`
        """

        if max_level != -1 and current_level > max_level:
            return []
        if not date:
            links = self.parentchildlink_parent.filter(end_time__exact=None)
        else:
            links = self.parentchildlink_parent.filter(ctime__lt=date).exclude(end_time__lt=date)
        res = []
        for link in links.order_by("order", "child__reference"):
            res.append(Child(current_level, link))
            pc = PartController(link.child, self._user)
            res.extend(pc.get_children(max_level, current_level + 1, date))
        return res

    def get_parents(self, max_level=1, current_level=1, date=None):
        """
        Returns a list of all parents at time *date*.

        :rtype: list of :class:`Parent`
        """

        if max_level != -1 and current_level > max_level:
            return []
        if not date:
            links = self.parentchildlink_child.filter(end_time__exact=None)
        else:
            links = self.parentchildlink_child.filter(ctime__lt=date).exclude(end_time__lt=date)
        res = []
        for link in links:
            res.append(Parent(current_level, link))
            pc = PartController(link.parent, self._user)
            res.extend(pc.get_parents(max_level, current_level + 1, date))
        return res

    def update_children(self, formset):
        u"""
        Updates children informations with data from *formset*

        :param formset:
        :type formset: a modelfactory_formset of
                        :class:`~plmapp.forms.ModifyChildForm`

        :raises: :exc:`.PermissionError` if :attr:`_user` is not the owner of
            :attr:`object`.
        :raises: :exc:`.PermissionError` if :attr:`object` is not editable.
        """

        self.check_permission("owner")
        self.check_editable()
        if formset.is_valid():
            for form in formset.forms:
                parent = form.cleaned_data["parent"]
                if parent.pk != self.object.pk:
                    raise ValueError("Bad parent %s (%s expected)" % (parent, self.object))
                delete = form.cleaned_data["delete"]
                child = form.cleaned_data["child"]
                if delete:
                    self.delete_child(child)
                else:
                    quantity = form.cleaned_data["quantity"]
                    order = form.cleaned_data["order"]
                    unit = form.cleaned_data["unit"]
                    self.modify_child(child, quantity, order, unit,
                            **form.extensions)

    def revise(self, new_revision):
        # same as PLMObjectController + add children
        new_controller = super(PartController, self).revise(new_revision)
        for level, link in self.get_children(1):
            link.clone(save=True, parent=new_controller.object)
        return new_controller

    def attach_to_document(self, document):
        """
        Links *document* (a :class:`.Document`) with
        :attr:`~PLMObjectController.object`.

        :raises: :exc:`.PermissionError` if :attr:`_user` is not the owner of
            :attr:`object`.
        """

        self.check_attach_document(document)
        if isinstance(document, PLMObjectController):
            document = document.object
        self.documentpartlink_part.create(document=document)
        self._save_histo(models.DocumentPartLink.ACTION_NAME,
                         "Part : %s - Document : %s" % (self.object, document))

    def detach_document(self, document):
        """
        Delete link between *document* (a :class:`.Document`)
        and :attr:`~PLMObjectController.object`.

        :raises: :exc:`.PermissionError` if :attr:`_user` is not the owner of
            :attr:`object`.
        """

        self.check_permission("owner")
        if isinstance(document, PLMObjectController):
            document = document.object
        link = self.documentpartlink_part.get(document=document)
        link.delete()
        self._save_histo(models.DocumentPartLink.ACTION_NAME + " - delete",
                         "Part : %s - Document : %s" % (self.object, document))

    def get_attached_documents(self):
        """
        Returns all :class:`.Document` attached to
        :attr:`~PLMObjectController.object`.
        """
        return self.documentpartlink_part.all()

    def is_document_attached(self, document):
        """
        Returns True if *document* is attached to the current part.
        """

        if isinstance(document, PLMObjectController):
            document = document.object
        return bool(self.documentpartlink_part.filter(document=document))

    def check_attach_document(self, document):
        self.check_permission("owner")
        if not hasattr(document, "is_document") or not document.is_document:
            raise TypeError("%s is not a document" % document)

        if isinstance(document, PLMObjectController):
            document.check_readable()
            document = document.object
        else:
            get_controller(document.type)(document, self._user).check_readable()
        if self.is_document_attached(document):
            raise ValueError("Document is already attached.")

    def can_attach_document(self, document):
        """
        Returns True if *document* can be attached to the current part.
        """
        can_attach = False
        try:
            self.check_attach_document(document)
            can_attach = True
        except StandardError:
            pass
        return can_attach

    def update_doc_cad(self, formset):
        u"""
        Updates doc_cad informations with data from *formset*

        :param formset:
        :type formset: a modelfactory_formset of
                        :class:`~plmapp.forms.ModifyChildForm`

        :raises: :exc:`.PermissionError` if :attr:`_user` is not the owner of
            :attr:`object`.
        """

        self.check_permission("owner")
        if formset.is_valid():
            for form in formset.forms:
                part = form.cleaned_data["part"]
                if part.pk != self.object.pk:
                    raise ValueError("Bad part %s (%s expected)" % (part, self.object))
                delete = form.cleaned_data["delete"]
                document = form.cleaned_data["document"]
                if delete:
                    self.detach_document(document)


import numpy, pandas, sys
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt, mpld3

PRICE = 0.00001667
WORKLOAD = 100000

try:
    data = pandas.read_csv('data.csv')
    data = data.sort_values(by=['MemorySize'])
    data['cost100k'] = data.apply(lambda row: ((WORKLOAD * row.BilledDuration)/1000) * (row.MemorySize / 1024) * PRICE, axis=1)
    data.round(2).sort_values(by=['cost100k']).to_csv('totalcost-100000.csv')
    # print(data)
except Exception as error:
    print('Error:', error)


import pytest
import p10s.terraform as tf
import shutil
from copy import deepcopy

def test_init():
    c = tf.Context()
    assert c.data == {}

def test_eq1():
    a = tf.TerraformBlock({'type': {'name': 'value'}})
    assert a == a

def test_eq2():
    a = tf.TerraformBlock({'type': {'name': 2}})
    b = tf.TerraformBlock({'type': {'name': 1 + 1}})
    assert a == b

def test_neq1():
    a = tf.TerraformBlock({'type': {'name': 2}})
    b = tf.TerraformBlock({'type': {'name': 1}})
    assert a != b

def test_neq2():
    a = tf.TerraformBlock({'type': {'name': 2}})
    assert a != "not a block"

def test_variable1():
    c = tf.Context()
    c += tf.Variable('foo')
    assert {'variable': {'foo': {}}} == c.data

def test_variable2():
    c = tf.Context()
    c += tf.Variable('foo', dict(
        type='string'
    ))
    assert {'variable': {'foo': {'type': 'string'}}} == c.data

def test_add():
    a = tf.Context()
    assert {} == a.data
    b = a + tf.Variable('foo', {'type': 'string'})
    assert {} == a.data
    assert {'variable': {'foo': {'type': 'string'}}} == b.data

def test_iadd():
    a = tf.Context()
    assert {} == a.data
    a += tf.Variable('foo', {'type': 'string'})
    assert {'variable': {'foo': {'type': 'string'}}} == a.data

def test_multiple_variables():
    c = tf.Context()
    c += tf.Variable('foo', dict(
        type='string'
    ))
    c += tf.Variable('bar', dict(
        default='whatever'
    ))
    assert {'variable': {'foo': {'type': 'string'},
                         'bar': {'default': 'whatever'}}} == c.data

def test_module():
    c = tf.Context()
    c += tf.Module('foo1', {
        'source': '../../module/',
        'list_of_things': [1, 2, 3]
    })
    assert {'module': {'foo1': {'source': '../../module/',
                                'list_of_things': [1, 2, 3]}}} == c.data

def test_resource():
    c = tf.Context()
    c += tf.Resource("type", "name", dict(
        var='value'
    ))
    assert {'resource': {'type': {'name': {'var': 'value'}}}} == c.data

def test_update1():
    r = tf.Resource(type="t", name="n", body=dict(
    ))
    r.update(dict(
        type="other",
        name="nother"
    ))

    assert {'resource': {'t': {'n': {'type': 'other', 'name': 'nother'}}}} == r.data

def test_hcl_as_object():
    c1 = tf.Context()
    c1 += tf.many_from_hcl("""
    terraform {
      foo = "bar"
    }

    resource "type" "name" {
      whatever {
        here = "there"
      }
    }

    """)
    c2 = tf.Context()
    c2 += tf.Terraform({'foo': 'bar'})
    c2 += tf.Resource('type', 'name', {
        'whatever': {
            'here': 'there'
        }
    })
    assert c1.data == c2.data

def test_single_hcl_raises():
    with pytest.raises(Exception):
        tf.from_hcl("""
            terraform {
              foo = "bar"
            }

            resource "type" "name" {
              whatever {
                here = "there"
              }
            }
        """)

def test_many_from_hcl_unknown_key():
    with pytest.raises(tf.HCLUnknownBlockError):
        tf.many_from_hcl("""key { }""")

def test_many_from_hcl_no_data():
    with pytest.raises(tf.HCLParseError):
        tf.many_from_hcl(""" """)

def test_from_hcl_no_data():
    with pytest.raises(tf.HCLParseError):
        tf.from_hcl(""" """)

def test_from_hcl_no_data2():
    try:
        tf.from_hcl(""" """)
    except tf.HCLParseError as e:
        assert str(e).startswith("Unable to parse")
    else:
        pytest.fail("from_hcl did not raise an exception")

def test_single_parse():
    c1 = tf.Context()
    c1 += tf.many_from_hcl("""
        terraform {
          foo = "bar"
        }
    """)
    c2 = tf.Context()
    c2 += tf.Terraform({'foo': 'bar'})
    assert c1.data == c2.data

def test_many_from_hcl_order():
    c = tf.Context()
    many = tf.many_from_hcl("""
    variable "" {
      default = "1"
    }

    variable "a" {
      default = "1"
    }

    variable "a/b" {
      default = "1"
    }

    variable "a/b/c" {
      default = "1"
    }

    """)
    assert [tf.Variable(name="", body={'default': '1'}),
            tf.Variable(name="a", body={'default': '1'}),
            tf.Variable(name="a/b", body={'default': '1'}),
            tf.Variable(name="a/b/c", body={'default': '1'})] == many

@pytest.mark.parametrize("hcl,data", [("""terraform {
                                            foo = "bar"
                                          }
                                          resource "type" "name" {
                                             whatever {
                                               here = "there"
                                             }
                                           }
                                          """,
                                       {'terraform': {'foo': 'bar'},
                                        'resource': {
                                            'type': {
                                                'name': {
                                                    'whatever': {
                                                        'here': 'there'
                                                    }
                                                }
                                            }
                                        }}),
                                      ("""locals {
                                            other {
                                              nested {
                                                value = "value"
                                              }
                                            }
                                          }

                                          locals {
                                            locals = "whatever"
                                          }
                                      """,
                                       {'locals': {
                                           'other': {
                                               'nested': {
                                                   'value': 'value'
                                               }
                                           },
                                           'locals': 'whatever'
                                       }
                                       }),
                                      ("""resource "foo" "bar" {
                                            data1 = <<HEREDOC
                                             this is a test
                                          HEREDOC
                                            data2 = "this is not a test"
                                          }
                                      """,
                                       {'resource':
                                        {'foo': {
                                            'bar': {
                                                'data1': '                                             this is a test\n'
                                                         '                                          ',
                                                'data2': 'this is not a test'}}}})],
                         ids=['two blocks', 'two locals blocs', 'heredoc'])
def test_hcl_as_data(hcl, data):
    c1 = tf.Context()
    c1 += tf.many_from_hcl(hcl)
    assert data == c1.data

def test_hcl_nested_named_block():
    c = tf.Context()
    c += tf.many_from_hcl("""
terraform {
  backend "foo" {
    foo = "bar"
    bar = ["baz"]

    map = {
      a = "b"
    }
  }
}
""")
    assert {
        'terraform': {
            'backend': {
                'foo': {
                    'foo': 'bar',
                    'bar': ['baz'],
                    'map': {
                        'a': 'b'
                    }
                }
            }
        }
    } == c.data

def test_modify_resource():
    resource = tf.Resource("foo", "bar", {
        "count": 7
    })
    assert resource.body['count'] == 7
    assert resource.type == 'foo'
    assert resource.name == 'bar'
    resource.body['count'] = 0
    resource.name = 'name'
    resource.type = 'type'
    assert resource.body['count'] == 0
    assert resource.name == 'name'
    assert resource.type == 'type'

def test_modify_module():
    module = tf.Module("foo", {
        "count": 7
    })

    module.name = "other"
    module.body['count'] = 0

    assert {'module': {'other': {'count': 0}}} == module.data

def test_share_resource():
    t = tf.Variable("var1", dict(default=1))
    c = tf.Context()

    c += t
    t.name = 'var2'
    c += t

    assert c.data == {'variable': {'var2': {'default': 1}}}

def test_copy_resource():
    t = tf.Variable("var1", dict(default=1))
    c = tf.Context()

    c += deepcopy(t)
    t.name = 'var2'
    c += t

    assert c.data == {'variable': {'var1': {'default': 1},
                                   'var2': {'default': 1}}}

def test_duplicate_variable():
    v1 = tf.Variable("var1", dict(default=1))
    v2 = tf.Variable("var1", dict(other=42))

    c = tf.Context(strict=True)
    c += v1
    with pytest.raises(tf.DuplicateBlockError):
        c += v2

def test_duplicate_resource():
    r1 = tf.Resource("type", "name", dict(var1='val1'))
    r2 = tf.Resource("type", "name", dict(var2='val2'))

    c = tf.Context(strict=True)
    c += r1
    with pytest.raises(tf.DuplicateBlockError):
        c += r2

def test_duplicate_terraform():
    t1 = tf.Terraform(dict(var1='val1'))
    t2 = tf.Terraform(dict(var2='val2'))

    c = tf.Context(strict=True)
    c += t1
    with pytest.raises(tf.DuplicateBlockError):
        c += t2

def test_output1():
    o = tf.Output(name='foo')
    assert o.data == {'output': {'foo': {}}}

def test_output2():
    o = tf.Output(name='foo', body='bar')
    assert o.data == {'output': {'foo': 'bar'}}

def test_output3():
    o = tf.Output(aws_whatever="${var.other_thing}")
    assert o.data == {'output': {'aws_whatever': {'value': "${var.other_thing}"}}}

def test_output4():
    with pytest.raises(Exception):
        tf.Output(a='b', c='d')

def test_module_name():
    m = tf.from_hcl("""module "X" { var = "value" }""")
    m.name = "X"
    assert {'module': {'X': {'var': 'value'}}} == m.data

def test_create_output_dir(fixtures_dir):
    output_dir = fixtures_dir / 'generator_data' / 'simple' / 'does_not_exist'
    output_file = output_dir / 'output.tf.json'
    shutil.rmtree(str(output_dir), ignore_errors=True)
    c = tf.Context(output=output_file)
    c += tf.Terraform(dict(foo=42))
    c.render()

    assert output_file.exists()

def test_variables():
    c = tf.Context()
    c += tf.variables(
        foo=42,
        bar='whatever'
    )
    assert {'variable': {'foo': {'default': 42},
                         'bar': {'default': 'whatever'}}} == c.data

def test_outputs():
    c = tf.Context()
    c += tf.outputs(
        a='b',
        c='${var.c}')
    assert {'output': {'a': {'value': 'b'},
                       'c': {'value': '${var.c}'}}} == c.data

def test_false_body():
    t = tf.Resource("type", "name", 0)
    assert t.data == {'resource': {'type': {'name': 0}}}

    t = tf.Module("name", "")
    assert t.data == {'module': {'name': ""}}

def test_implicit_resource():
    a = tf.Context()

    a.resource.a.b = {'k': 'v1'}
    a.resource.a.c = {'k': 'v2'}

    b = tf.Context()
    b += tf.Resource("a", "b", {'k': 'v1'})
    b += tf.Resource("a", "c", {'k': 'v2'})

    assert a.data == b.data

def test_implicit_module1():
    c = tf.Context()

    c.module.name = {}

    assert {'module': {'name': {}}} == c.data

def test_implicit_module2():
    c = tf.Context()

    c.module["name"] = {'source': 'here'}

    assert {'module': {'name': {'source': 'here'}}} == c.data

def test_implicit_variable1():
    c = tf.Context()

    c.variable.name = 'foo'

    assert {'variable': {'name': {'default': 'foo'}}} == c.data

def test_implicit_variable2():
    c = tf.Context()

    c.variable["name"] = 'foo'

    assert {'variable': {'name': {'default': 'foo'}}} == c.data


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Compatibility tools for various data structure inputs
"""
from six.moves import range
import numpy as np
import pandas as pd

# upstream this is in discrete_model._numpy_to_dummies
def numpy_to_dummies(endog):
    from sm2.tools import tools
    if endog.dtype.kind in ['S', 'O']:
        endog_dummies, ynames = tools.categorical(endog, drop=True,
                                                  dictnames=True)
    elif endog.ndim == 2:
        endog_dummies = endog
        ynames = range(endog.shape[1])
    else:
        endog_dummies, ynames = tools.categorical(endog, drop=True,
                                                  dictnames=True)
    return endog_dummies, ynames

# upstream this is in discrete_model._pandas_to_dummies
def pandas_to_dummies(endog):
    if endog.ndim == 2:
        if endog.shape[1] == 1:
            yname = endog.columns[0]
            endog_dummies = pd.get_dummies(endog.iloc[:, 0])
        else:  # series
            yname = 'y'
            endog_dummies = endog
    else:
        yname = endog.name
        endog_dummies = pd.get_dummies(endog)
    ynames = endog_dummies.columns.tolist()

    return endog_dummies, ynames, yname

# upstream this is discrete_margins._get_const_index
def get_const_index(exog):
    """
    Returns a boolean array of non-constant column indices in exog and
    an scalar array of where the constant is or None
    """
    effects_idx = exog.var(0) != 0
    if np.any(~effects_idx):
        const_idx = np.where(~effects_idx)[0]
    else:
        const_idx = None
    return effects_idx, const_idx

# upstream this is discrete_margins._isdummy
def isdummy(X):
    """
    Given an array X, returns the column indices for the dummy variables.

    Parameters
    ----------
    X : array-like
        A 1d or 2d array of numbers

    Examples
    --------
    >>> X = np.random.randint(0, 2, size=(15, 5)).astype(float)
    >>> X[:, 1:3] = np.random.randn(15, 2)
    >>> ind = _isdummy(X)
    >>> ind
    array([0, 3, 4])
    """
    X = np.asarray(X)
    if X.ndim > 1:
        ind = np.zeros(X.shape[1]).astype(bool)
    max = (np.max(X, axis=0) == 1)
    min = (np.min(X, axis=0) == 0)
    remainder = np.all(X % 1. == 0, axis=0)
    ind = min & max & remainder
    if X.ndim == 1:
        ind = np.asarray([ind])
    return np.where(ind)[0]

# upstream this is discrete_margins._get_dummy_index
def get_dummy_index(X, const_idx):
    dummy_ind = isdummy(X)
    dummy = True

    if dummy_ind.size == 0:  # don't waste your time
        dummy = False
        dummy_ind = None  # this gets passed to stand err func
    return dummy_ind, dummy

# upstream this is discrete_margins._iscount
def iscount(X):
    """
    Given an array X, returns the column indices for count variables.

    Parameters
    ----------
    X : array-like
        A 1d or 2d array of numbers

    Examples
    --------
    >>> X = np.random.randint(0, 10, size=(15, 5)).astype(float)
    >>> X[:, 1:3] = np.random.randn(15, 2)
    >>> ind = _iscount(X)
    >>> ind
    array([0, 3, 4])
    """
    X = np.asarray(X)
    remainder = np.logical_and(np.logical_and(np.all(X % 1. == 0, axis=0),
                               X.var(0) != 0), np.all(X >= 0, axis=0))
    dummy = isdummy(X)
    remainder = np.where(remainder)[0].tolist()
    for idx in dummy:
        remainder.remove(idx)
    return np.array(remainder)

def get_count_index(X, const_idx):
    count_ind = iscount(X)
    count = True

    if count_ind.size == 0:  # don't waste your time
        count = False
        count_ind = None  # for stand err func
    return count_ind, count

def _check_period_index(x, freq="M"):  # pragma: no cover
    raise NotImplementedError("_check_period_index not ported from upstream, "
                              "as it is only used in tsaplots, "
                              "which is not (yet) ported")

def is_data_frame(obj):
    return isinstance(obj, pd.DataFrame)

def is_design_matrix(obj):
    from patsy import DesignMatrix
    return isinstance(obj, DesignMatrix)

def is_structured_ndarray(obj):
    return isinstance(obj, np.ndarray) and obj.dtype.names is not None

def interpret_data(data, colnames=None, rownames=None):
    """
    Convert passed data structure to form required by estimation classes

    Parameters
    ----------
    data : ndarray-like
    colnames : sequence or None
        May be part of data structure
    rownames : sequence or None

    Returns
    -------
    (values, colnames, rownames) : (homogeneous ndarray, list)
    """
    if isinstance(data, np.ndarray):
        if _is_structured_ndarray(data):
            if colnames is None:
                colnames = data.dtype.names
            values = struct_to_ndarray(data)
        else:
            values = data

        if colnames is None:
            colnames = ['Y_%d' % i for i in range(values.shape[1])]
    elif is_data_frame(data):
        # XXX: hack
        data = data.dropna()
        values = data.values
        colnames = data.columns
        rownames = data.index
    else:  # pragma: no cover
        raise TypeError('Cannot handle input type {typ}'
                        .format(typ=type(data).__name__))

    if not isinstance(colnames, list):
        colnames = list(colnames)

    # sanity check
    if len(colnames) != values.shape[1]:
        raise ValueError('length of colnames does not match number '
                         'of columns in data')  # pragma: no cover

    if rownames is not None and len(rownames) != len(values):
        raise ValueError('length of rownames does not match number '
                         'of rows in data')  # pragma: no cover

    return values, colnames, rownames

def struct_to_ndarray(arr):
    return arr.view((float, len(arr.dtype.names)), type=np.ndarray)

def is_using_ndarray_type(endog, exog):
    return (type(endog) is np.ndarray and
            (type(exog) is np.ndarray or exog is None))

def is_using_ndarray(endog, exog):
    return (isinstance(endog, np.ndarray) and
            (isinstance(exog, np.ndarray) or exog is None))

def is_using_pandas(endog, exog):
    klasses = (pd.Series, pd.DataFrame, pd.Panel)
    # from sm2.compat.pandas import data_klasses as klasses
    return (isinstance(endog, klasses) or isinstance(exog, klasses))

def _is_array_like(endog, exog):  # pragma: no cover
    raise NotImplementedError("_is_array_like not ported from upstream, "
                              "as it is only used in iolib.foreign, "
                              "which is itself not ported.")

def is_using_patsy(endog, exog):
    # we get this when a structured array is passed through a formula
    return (is_design_matrix(endog) and
            (is_design_matrix(exog) or exog is None))

def is_recarray(data):
    """
    Returns true if data is a recarray
    """
    return isinstance(data, np.core.recarray)

# aliases for backward compatibility
_is_structured_ndarray = is_structured_ndarray
_is_recarray = is_recarray
_is_using_patsy = is_using_patsy
_is_using_pandas = is_using_pandas
_is_using_ndarray = is_using_ndarray
_is_using_ndarray_type = is_using_ndarray_type


from asyncio.futures import Future
import logging
from os import getpid
from random import randint
from time import time
from typing import Callable, Iterable
from bacpypes.apdu import (
    ConfirmedRequestSequence,
    SubscribeCOVRequest,
)
from bacpypes.core import deferred
from bacpypes.iocb import IOCB, IOController
from bacpypes.pdu import Address
from bacpypes.primitivedata import ObjectIdentifier
from bacpypes.task import OneShotTask
from fastapi.exceptions import HTTPException

_logger = logging.getLogger(__name__)

class _BaseRecurringTask(OneShotTask):
    def __init__(self, interval: int | None, offset: float | None = None) \
            -> None:
        self.interval = interval
        self.offset = offset
        self.cancelled = False
        super().__init__()
        _logger.debug("Init %r", self)

    def install_task(self, when: float | None = None,
                     delta: float | None = None) -> None:
        if self.cancelled:
            return
        offset = self.offset if self.offset is not None else 0
        super().install_task(when=time() + offset)

    def process_task(self) -> None:
        _logger.debug("Pocess task %r", self)
        if self.interval and not self.cancelled:
            super().install_task(delta=self.interval)

    def cancel_task(self) -> None:
        _logger.debug("Canceled task %r", self)
        self.cancelled = True

class _BaseIOTask(_BaseRecurringTask):
    def __init__(self, io_controller: IOController, interval: int,
                 offset: float | None = None,
                 callback: Callable[[IOCB], None] | None = None) -> None:
        if offset is None:
            offset = randint(0, interval * 1000 - 1) / 1000.0
        self.io_controller = io_controller
        self.callback = callback
        super().__init__(interval, offset)

    def _add_callback(self, iocb: IOCB) -> None:
        if self.callback is not None:
            iocb.add_callback(self.callback)

    def process_task(self) -> None:
        super().process_task()
        for request in self._build_requests():
            iocb = IOCB(request)
            self._add_callback(iocb)
            deferred(self.io_controller.request_io, iocb)

    def _build_requests(self) -> Iterable[ConfirmedRequestSequence]:
        raise NotImplementedError()

class SubscribeCOVTask(_BaseIOTask):
    def __init__(self, io_controller: IOController, address: Address,
                 object_identifier: ObjectIdentifier, cov_lifetime: int,
                 future: Future[bool]) \
            -> None:
        self.address = address
        self.object_identifier = object_identifier
        self.lifetime = cov_lifetime
        self.future = future
        super().__init__(io_controller, cov_lifetime, 0)

    def _build_requests(self) -> Iterable[ConfirmedRequestSequence]:
        yield SubscribeCOVRequest(
            destination=self.address,
            subscriberProcessIdentifier=getpid(),
            monitoredObjectIdentifier=self.object_identifier,
            issueConfirmedNotifications=False,
            lifetime=self.lifetime
        )

    def process_subscribe_ack(self, iocb: IOCB) -> None:
        if iocb.ioError:
            _logger.error("Failed to subscribe to %r@%r: %r",
                          self.object_identifier, self.address, iocb.ioError)
            if not self.future.done():
                self.future.set_exception(HTTPException(
                    500, f"Failed to subscribe to COV: {iocb.ioError}",
                ))
            self.cancel_task()
        else:
            _logger.debug("Subsribed to %r@%r", self.object_identifier,
                          self.address)
            if not self.future.done():
                self.future.set_result(True)

    def _add_callback(self, iocb: IOCB) -> None:
        iocb.add_callback(self.process_subscribe_ack)


#!/usr/bin/env python3
# @Date    : 2021-12-21
# @Author  : Bright Li (brt2@qq.com)
# @Link    : https://gitee.com/brt2
# @Version : 0.1.5

import os
import re
import datetime
import glob
from collections import defaultdict

from .md_parser import MarkdownParser, NullMarkdownFile
# from .util.imgfmt import png2jpg, resize

# def png2jpg_for_md(path_png):
#     """ 验证png转换后体积明显缩小，
#         否则添加前缀，避免修改后再次上传时重复进行格式转换 """
#     new_file_prefix="keepng_"
#     if os.path.basename(path_png).startswith(new_file_prefix):
#         return path_png

#     path_jpg = png2jpg(path_png, 85)
#     if path_jpg == path_png:
#         path_jpg = os.path.join(os.path.dirname(path_png),
#            new_file_prefix + os.path.basename(path_png))
#         os.rename(path_png, path_jpg)
#     return path_jpg

class MarkdownFormatter(MarkdownParser):

    def format(self, resize_imgs=False):
        # self._update_categories()

        if not self.check_list["find_TOC"]:
            toc_index = self.check_list["index_H2"]
            if toc_index is None:
                toc_index = 0
            self.insert_text(toc_index, "[TOC]\n\n---\n\n")

        if self.check_list["index_H1"]:
            self.pop_text(self.check_list["index_H1"])

        self.update_serial_num()
        self.update_description()
        self.update_meta()  # Front Matter

        # # 图像处理
        # if self.get_images("http") and input("是否尝试下载超链接图片？[Y/n]: ").lower() != "n":
        #     # self.unlock_text()
        #     self.download_img()

        # # 默认启用 png -> jpg
        # self.convert_png2jpg()

        # # 对于高分辨率图像进行压缩
        # if resize_imgs:
        #     self.resize_high_resolution()

        # # 判断下载图像的size，执行resize或压缩
        # self.compress_bigimg()

    def _make_front_matter(self):
        def list_as_str(data: list):
            # str(data) -> 单引号，不符合markdown标准
            return "[\"" + "\",\"".join(data) + "\"]" if data else "[]"

        self.metadata["date"] = str(datetime.date.today())

        # date数据由于使用eval()反序列化，故必须使用""作为字符串
        str_md_info = """<!--
+++
title       = "{}"
description = "{}"
date        = "{}"
tags        = {}
categories  = {}
series      = {}
keywords    = {}
weight      = {}
toc         = true
draft       = false
+++ -->
""".format(self.metadata['title'], self.metadata['description'], self.metadata['date'],
    list_as_str(self.metadata.get('tags')), list_as_str(self.metadata.get('categories')),
    list_as_str(self.metadata.get('series')), list_as_str(self.metadata.get('keywords')),
    self.metadata['weight'])
        return str_md_info

    def update_meta(self):
        meta_line = self._make_front_matter()
        if self.meta_range[0] is not None:
        # def _remove_old_meta(self):
            __text_lines = self.get_text()
            meta_start, meta_end = self.meta_range
            if meta_start is not None:
                self.set_text(__text_lines[:meta_start] + __text_lines[meta_end +1:])

        # def _insert_meta(self):
            self.insert_text(0, meta_line)
            self.meta_range = [None, None]
        else:
            self.modify_text(0, meta_line)

    def update_serial_num(self):
        """ 使用3级序号：1.2.4. xxx """
        x, y, z = 0, 0, 0
        def get_serial():
            serial_num = ""
            for i in [x, y, z]:
                if i:
                    # print("__", serial_num, type(i))
                    serial_num += "{}.".format(i)
                else:
                    break
            return serial_num

        pattern_headline = re.compile(r"(#+) +(\d+\.\S*)? *(.*)")

        def update_line(line):
            serial_num = get_serial()
            # if self.check_list["has_serial_num"]:
            #     prefix, _, text = line.split(maxsplit=2)
            # else:
            #     prefix, text = line.split(maxsplit=1)
            prefix, _, text = re.match(pattern_headline, line).groups()
            return "{} {} {}".format(prefix, serial_num, text)

        self.head2 = []
        for index, line in enumerate(self.get_text()):
            if line.startswith("## "):
                x += 1
                y,z = 0,0

                new_line = update_line(line)
                self.modify_text(index, new_line)
                # h2 = re.match(pattern_headline, line).groups()[2]
                _, h2 = new_line.split(maxsplit=1)
                self.head2.append(h2)
            elif line.startswith("### "):
                y += 1
                z = 0
                self.modify_text(index, update_line(line))
            elif line.startswith("#### "):
                z += 1
                self.modify_text(index, update_line(line))

    def update_description(self):
        if not self.metadata['description']:
            self.metadata['description'] = "; ".join(self.head2)

    def download_img(self):
        from util.imgfmt import download_src

        dict_images = self.get_images("http")
        # 生成图像目录
        dir_img, _ = os.path.splitext(self.file_path)
        if not os.path.exists(dir_img):
            os.makedirs(dir_img)
        self.process_images(dict_images, lambda url: os.path.relpath(download_src(url, dir_img),
                                                            os.path.dirname(self.file_path)))

    def convert_png2jpg(self):
        dict_images = self.get_images("png")
        self.process_images(dict_images, lambda path_png: os.path.relpath(png2jpg_for_md(path_png),
                                                            os.path.dirname(self.file_path)))
        # 删除原png文件
        # for path_img in dict_images.values():
        #     os.remove(path_img)

    def compress_jpg(self):
        pass

    def compress_bigimg(self):
        pass

    # def resize_high_resolution(self, save_as_jpg=True):
    #     dict_images_png = self.get_images("png")
    #     dict_images_jpg = self.get_images("jpg")
    #     callback = lambda url: resize(url, ratio=0.6, min_size=10240,
    #                                   max_shape=[680,680], save_as_jpg=save_as_jpg)
    #     self.process_images({**dict_images_png, **dict_images_jpg}, callback)

#####################################################################

def getopt():
    import argparse

    parser = argparse.ArgumentParser("格式化mkdocs文档", description="")
    parser.add_argument("-p", "--path", action="store", help="解析文件路径，可以是文件或目录")
    return parser.parse_args()

def format_one_doc(fmt, path_file):
    fmt.load_file(path_file)
    fmt.format(resize_imgs=False)

def format_dir(fmt, path_dir):
    list_files = glob.glob("{}/**/*.md" % path_dir, recursive=True)

    err_files = defaultdict(list)
    for path_file in list_files:
        try:
            format_one_doc(fmt, path_file)
        except NullMarkdownFile:
            err_files["NullMarkdownFile"].append(path_file)

    if err_files:
        for key, list_files in err_files.items():
            print(">> 错误文件类型【{}】" % key)
            print("   文件列表： {}" % list_files)

def format_anything(fmt, path):
    if os.path.isdir(path):
        format_dir(fmt, path)
    else:
        format_one_doc(fmt, path)

if __name__ == "__main__":
    args = getopt()
    fmt = MarkdownFormatter()

    if args.path:
        format_anything(fmt, args.path)
    else:
        path = input("\n请输入待处理文件path(支持直接拖拽): ")
        while True:
            path = path.strip().strip('"')
            if os.path.exists(path):
                format_anything(fmt, path)
                fmt.overwrite()
            else:
                print("Error: File [{}] NOT found." % path)

            path = input("继续输入path，按[Q]退出: ")
            if path.lower() == "q":
                break


# -*- coding: utf-8 -*-
from django import template
from permissions import shortcuts

register = template.Library()

@register.filter
def has_role_permission(membership, permission_string):
    """
    :param permission_string: 'app_label.model:permission'.
    """
    model, permission = permission_string.split(':')
    return shortcuts.has_role_permission(membership, model, permission)

@register.simple_tag(takes_context=True)
def has_permission(context, model, permission, obj=None):
    """
    :param model: 'app_label.model'
    """
    membership = context['current_membership']
    return (shortcuts.has_role_permission(membership, model, permission) or
            shortcuts.has_object_permission(membership, obj, permission))