from json import dumps, loads
from uuid import uuid4

from collections import OrderedDict
from klein import Klein

class CustomerStore(object):
    app = Klein()

    def __init__(self):
        self.customers = OrderedDict(dict())  # Trivial to swap out for some simple k/v store or SimpleDB

    @app.route('/', methods=['GET', 'POST'])
    def create_customer(self, request):
        request.setHeader('Content-Type', 'application/json')

        if request.method == 'POST':
            body = loads(request.content.read())
            if body.keys() != ['first_name', 'last_name', 'country']:
                request.setResponseCode(400)
                return dumps({'error': 'ValidationError',
                              'error_message': 'You must include `first_name` and `last_name` keys'})
            pk = '{0} {1}'.format(body['first_name'], body['last_name'])
            if pk in self.customers:
                request.setResponseCode(400)
                return dumps({'error': 'UniqueKeyError',
                              'error_message': 'First name + last_name combination must be unique'})
            body['id'] = uuid4().get_hex()
            self.customers[pk] = body
            return dumps({'created': body})
        else:
            if not request.args or 'limit' in request.args:
                if not self.customers:
                    request.setResponseCode(404)
                    return dumps({'error': 'NotFound',
                                  'error_message': 'No customers in collection'})
                return dumps(self.customers[:request.args.get('limit', 2)])  # Show only two records by default
            elif request.args.keys() != ['first_name', 'last_name']:
                request.setResponseCode(400)
                return dumps({'error': 'ValidationError',
                              'error_message': 'You must include `first_name` and `last_name` keys'})
            pk = '{0} {1}'.format(request.args['first_name'][0], request.args['last_name'][0])
            if pk not in self.customers:
                request.setResponseCode(404)
                return dumps({'error': 'NotFound',
                              'error_message': 'First name + last_name combination not found in collection'})

            return dumps(self.customers[pk])

    @app.route('/<string:name>', methods=['PUT'])
    def save_customer(self, request, name):
        request.setHeader('Content-Type', 'application/json')
        body = loads(request.content.read())
        # You can also edit the pk here, which might not be a good idea:
        if {'first_name', 'last_name', 'id'}.issubset(set(body.keys())):  # Allow you to edit `id` here
            request.setResponseCode(400)
            return dumps({'error': 'ValidationError',
                          'error_message': 'You must include `first_name`, `last_name`, `country` and/or `id` key(s)'})
        if name not in self.customers:
            request.setResponseCode(404)
            return dumps({'error': 'NotFound',
                          'error_message': '"{0}" not found in customers collection'.format(name)})

        self.customers[name].update(body)
        return dumps(self.customers[name])

    @app.route('/<string:name>', methods=['GET'])
    def retrieve_customer(self, request, name):
        request.setHeader('Content-Type', 'application/json')
        if name not in self.customers:
            request.setResponseCode(404)
            return dumps({'error': 'NotFound',
                          'error_message': '"{0}" not found in customers collection'.format(name)})
        return dumps(self.customers[name])

    @app.route('/<string:name>', methods=['DELETE'])
    def delete_customer(self, request, name):
        request.setHeader('Content-Type', 'application/json')
        if name not in self.customers:
            request.setResponseCode(404)
            return dumps({'error': 'NotFound',
                          'error_message': '"{0}" not found in customers collection'.format(name)})
        return dumps({'deleted': self.customers.pop(name)})

if __name__ == '__main__':
    store = CustomerStore()
    store.app.run('0.0.0.0', 8080)


import stripe
from yoga.settings import SITE_NAME
from django.conf import settings
from .stripe_users import StripeCustomer
import pendulum

from userTransactions.models import UserTransactionItem
from siteTally.models import siteTransaction
from subscription.models import subscription_product, subscription

stripe.api_key = settings.STRIPE_SECRET_KEY

class StripeUserSubscription:
    #creator: required creator(User) db object which the user is subscribing to
    #subscriber: optional subscriber(User) db object of the subscriber
    #local subscriptionProductId obj, see findCreateSubProductId
    def __init__(self, creator=None, subscriber=None,
                 localSubscriptionProduct=None,
                 subscriptionObj = None, st_subscriptionId=None, st_intentId=None):

        self.creator = creator
        self.subscriber = subscriber
        self.localSubscriptionProduct = localSubscriptionProduct
        self.stSubscriptionId = st_subscriptionId
        self.localSubscriptionObj = subscriptionObj
        self.stIntentId = st_intentId

    def findCreateSubProductId(self):
        try:
            productObj = subscription_product.objects.filter(user=self.creator).select_related('creator', 'subscriber')[0]
            self.localSubscriptionProduct = productObj
            return self

        except:
            try:
                #create stripe product object
                if self.creator.subscription_units > 50:
                    product = stripe.Product.create(
                        description = f'userId: {self.creator.pk} | type:creator subscription',
                        name = f"userId: {self.creator.pk}",
                        shippable = False,
                        metadata = {
                            'type': 'subscription',
                            'site':settings.SITE_NAME,
                            'user_id':self.creator.pk
                        }
                        )
                else:
                    raise Exception("user subscription units must be greater than 50")

                try:
                    #create stripe price object for stripe product
                    price = stripe.Price.create(
                        product = product['id'],
                        unit_amount = self.creator.subscription_units,
                        currency = 'usd',
                        recurring = {
                            'interval':'month',
                            'interval_count':1
                            },
                        metadata = {
                        'site':settings.SITE_NAME,
                        'user_id':self.creator.pk,
                        }
                    )
                    #create local db record for product and price id's
                    newProduct = subscription_product.objects.create(
                                st_productId = product['id'],
                                st_priceId = price['id'],
                                user = self.creator
                                )
                    self.localSubscriptionProduct = newProduct
                    return self
                except:
                    print('error creating new price or db product')
                    return False
            except:
                print('error creating st product')
                return False

    def updateStripeProductInfo(self):
        #under construction
        productId = self.creator.st_productId
        if not productId:
            raise Exception('local product id object required')
        try:
            newPrice = stripe.Product.modify(productId)
            return True
        except:
            return False

    #updates stripe price of subscription
    #price has to be updated first
    # returns True if successful, False otherwise
    def updateStripeProductPrice(self):
        try:
            priceId = self.localSubscriptionProduct.st_priceId
        except:
            raise Exception("need stripe price id")
        try:
            units = self.creator.subscription_units
            newPrice = stripe.Price.modify(priceId, metadata = {'unit_amount':units})
            return True
        except:
            return False

    def createUserSubscription(self):
        if not self.subscriber:
            raise Exception('subscriber obj required')
        try:
            new_sub = stripe.Subscription.create(
                customer = self.subscriber.customerId,
                items = [{'price':self.localSubscriptionProduct.st_priceId}],
                payment_behavior='default_incomplete',
                expand=['latest_invoice.payment_intent'],
                metadata={
                    'creater':self.creator.pk,
                    'subscriber':self.subscriber.pk,
                    'site':SITE_NAME,
                }
            )
            self.st_subscription = new_sub
            return self
        except Exception as e:
            print(e)
            return False

    def create_transaction_records(stripe_subscription, self):
        if not stripe_subscription or not self.creator or not self.subscriber:
            raise Exception('need payment intent id, user obj and post obj')

        CustomerId = StripeCustomer(self.subscriber).findCreateCustomerId()
        invoice = stripe_subscription['latest_invoice']

        if invoice['status'] == 'succeeded':
            try:
                price = int(invoice['amount'])
                UserTransactionItem.objects.create(
                    user = self.subscriber,
                    units = price,
                    is_payment = False,
                    is_purchase = True,
                    is_refund = False,
                    subscription = invoice['metadata']['subscription']
                )

                #create record for creator
                UserTransactionItem.objects.create(
                    user = self.creator,
                    units = price,
                    is_payment = True,
                    is_purchase = False,
                    is_refund = False,
                    subscription = invoice['metadata']['subscription']
                )
            except:
                return False

            try:
                siteTransaction.objects.create(
                    st_transaction_id = invoice['id'],
                    customerId = CustomerId.stripeCustomer,
                    units = price,
                    is_payment = True,
                    is_refund = False,
                    subscription = invoice['metadata']['subscription']
                )
                return True
            except:
                return False

    def finalizeBuySubscription(self):
        if not self.stSubscriptionId or not self.stIntentId:
            raise Exception('stripe subscription id and stripe payment intent id required')
        try:
            intent = stripe.PaymentIntent.retrieve(self.stIntentId)
        except:
            return False
        try:
            st_subscription = stripe.Subscription.modify(self.stSubscriptionId,
                                                      default_payment_method = intent['payment_method'])
        except:
            return False

        begin_date = pendulum.now('utc').to_iso8601_string()
        end_date = pendulum.now('utc').add(months=1).to_iso8601_string()

        try:
            localSub = subscription.objects.get(creator = self.creator, subscriber = self.subscriber)
            localSub.begin_date = begin_date
            localSub.end_date =  end_date
            localSub.st_subId= self.stSubscriptionId
            localSub.is_active = True
            localSub.save()
            self.localSubscriptionObj = localSub
            return self
        except:
            try:
                localSub = subscription.objects.create(creator = self.creator,
                                                    subscriber = self.subscriber,
                                                    begin_date = begin_date,
                                                    end_date = end_date,
                                                    st_subId = self.stSubscriptionId)
                self.localSubscriptionObj = localSub
                return self
            except:
                print('unable to create local subscription object')
                return False

    def cancelSubscriptionSchedule(self):
        if not self.creator or not self.subscriber:
            raise Exception('creator and subscriber required')
        try:
            sub = subscription.objects.get(creator=self.creator, subscriber=self.subscriber)
        except:
            print('unable to find subscription')
            return False

        try:
            st_sub = stripe.Subscription.modify(sub.st_subId, cancel_at_period_end=True)
            print(st_sub)

        except:
            print('unable to find stripe subscription object')
            return False

        try:
            sub.is_renewed = False
            sub.save()
            return sub
        except:
            return False

    # for use with webhook when subscription cancellation has been received
    def cancelSubscriptionWebhook(self):
        if not self.creator or not self.subscriber:
            raise Exception('creator and subscriber required')
        try:
            sub = subscription.objects(creator=self.creator, subscriber=self.subscriber)
        except:
            print('unable to find subscription')
            return False

        try:
            sub.is_active = False
            sub.save()
            return sub
        except:
            return False

    def create_transaction_webhook(self, invoice):
        if invoice['billing_reason'] == 'subscription_cycle':

            sub_id = invoice['subscription']

            st_sub = stripe.Subscription.retrieve(sub_id)
            local_sub = subscription.objects.filter(creator = st_sub['metadata']['creator'], subsciber = st_sub['metadata']['subscriber']).select_related(
            'creator', 'subscriber')[0]

            self.creator = local_sub.creator
            self.subscriber = local_sub.subscriber
            self.localSubscriptionObj = local_sub

            CustomerId = StripeCustomer(self.subscriber).findCreateCustomerId()

            try:
                price = int(invoice['amount'])
                UserTransactionItem.objects.create(
                    user = self.subscriber,
                    units = price,
                    is_payment = False,
                    is_purchase = True,
                    is_refund = False,
                    subscription = self.localSubscriptionObj
                )

                #create record for creator
                UserTransactionItem.objects.create(
                    user = self.creator,
                    units = price,
                    is_payment = True,
                    is_purchase = False,
                    is_refund = False,
                    subscription = self.localSubscriptionObj
                )
            except:
                return False

            try:
                siteTransaction.objects.create(
                    st_transaction_id = invoice['id'],
                    customerId = CustomerId.stripeCustomer,
                    units = price,
                    is_payment = True,
                    is_refund = False,
                    subscription = self.localSubscriptionObj
                )
                return True
            except:
                return False


import graphene
import json
from collections import namedtuple
from elasticsearch import Elasticsearch
import urllib.parse

client = Elasticsearch()

def _json_object_hook(d):
    return namedtuple('X', d.keys())(*d.values())

def json2obj(data):
    as_python = json.loads(data, object_hook=_json_object_hook)
    return as_python

class Form(graphene.ObjectType):
    form_id = graphene.String()
    orth = graphene.String()
    gram = graphene.String()
    usgs = graphene.List(graphene.String)

class Sense(graphene.ObjectType):
    sense_id = graphene.String()
    definition = graphene.String()
    usgs = graphene.List(graphene.String)

class DictEntry(graphene.ObjectType):
    id = graphene.String()
    forms = graphene.List(Form)
    senses = graphene.List(Sense)

def select_from_elastic_response(elastic_raw):
    from_elastic = []
    for e in elastic_raw:
        elastic_result = {}
        elastic_result['id'] = e['_id']
        elastic_result['forms'] = e['_source']['forms']
        elastic_result['senses'] = e['_source']['senses']
        from_elastic.append(elastic_result)
    return from_elastic

def get_from_elastic(dict_id, query, query_type, field):
    query = urllib.parse.unquote(query)
    query_type = urllib.parse.unquote(query_type)
    field = urllib.parse.unquote(field)

    res = ''

    if field == 'form':
        res = client.search(index=dict_id,
                            body={
                                "query": {
                                    "nested": {
                                        "path": "forms",
                                        "query": {
                                            query_type: {
                                                "forms.orth": query
                                            }
                                        }
                                    }
                                }})
    if field == 'sense':
        res = client.search(index=dict_id,
                            body={
                                "query": {
                                    "nested": {
                                        "path": "senses",
                                        "query": {
                                            query_type: {
                                                "senses.definition": query
                                            }
                                        }
                                    }
                                }})

    return res

class DictQuery(graphene.ObjectType):
    entries = graphene.List(DictEntry, dict_id=graphene.String(), query=graphene.String(), query_type=graphene.String(),
                            field=graphene.String())

    def resolve_entries(self, info, query, query_type, field):
        res = get_from_elastic('beo', query=query, query_type=query_type, field=field)
        parsed_results = select_from_elastic_response(res['hits']['hits'])
        return json2obj(json.dumps(parsed_results))


import streamlit as st
from graph import box,histogram,pie
from dataframe import df

def score():
    s_names = ('S1','S2','S3','S4','S5','S6','S7','S8','S9','S10','GPA')
    s = st.radio('Điểm từng session',s_names,horizontal=True)
    if s == 'S1':
        box(df,'CLASS-GROUP','S1','GENDER')
        histogram(df,'S1',None,'PYTHON-CLASS')

    if s == 'S2':
        box(df,'CLASS-GROUP','S2','GENDER')
        histogram(df,'S2',None,'PYTHON-CLASS')

    if s == 'S3':
        box(df,'CLASS-GROUP','S3','GENDER')
        histogram(df,'S3',None,'PYTHON-CLASS')
    if s == 'S4':
        box(df,'CLASS-GROUP','S4','GENDER')
        histogram(df,'S4',None,'PYTHON-CLASS')

    if s == 'S5':
        box(df,'CLASS-GROUP','S5','GENDER')
        histogram(df,'S5',None,'PYTHON-CLASS')

    if s == 'S6':
        box(df,'CLASS-GROUP','S6','GENDER')
        histogram(df,'S6',None,'PYTHON-CLASS')

    if s == 'S7':
        box(df,'CLASS-GROUP','S7','GENDER')
        histogram(df,'S7',None,'PYTHON-CLASS')

    if s == 'S8':
        box(df,'CLASS-GROUP','S8','GENDER')
        histogram(df,'S8',None,'PYTHON-CLASS')

    if s == 'S9':
        box(df,'CLASS-GROUP','S9','GENDER')
        histogram(df,'S9',None,'PYTHON-CLASS')

    if s == 'S10':
        box(df,'CLASS-GROUP','S10','GENDER')
        histogram(df,'S10',None,'PYTHON-CLASS')

    if s == 'GPA':
        box(df,'CLASS-GROUP','GPA',None)
        st.success('Các lớp khối tự nhiên học tốt hơn lớp khối xã hội')
        st.success('Lớp Chuyên Tin học giỏi nhất')
        box(df,'PYTHON-CLASS','GPA',None)
        st.success('Lớp 114 học tốt hơn lớp 115')
        histogram(df,'GPA',None,'GENDER')
        st.success('Nam sinh học tốt hơn nữ sinh')
        pie(df[df['GPA'] >= 6],'CLASS-GROUP','Tỉ lệ đậu lớp MC')
def graph():
    a_names = ('PHÂN BỐ HS NAM - NỮ','PHÂN BỐ HS LỚP SÁNG - CHIỀU','PHÂN BỐ HS CHUYÊN - THƯỜNG','PHÂN BỐ HS CÁC KHỐI')
    a = st.radio('Loại biểu đồ',a_names,horizontal=True)
    if a=='PHÂN BỐ HS NAM - NỮ':
        pie(df, df["GENDER"],"PHÂN BỐ HS NAM - NỮ")
        st.write('Nam hứng thú với AI hơn')
    if a=='PHÂN BỐ HS LỚP SÁNG - CHIỀU':
        pie(df, df["PYTHON-CLASS"],"PHÂN BỐ HS LỚP SÁNG CHIỀU")
        st.write('Học sinh lớp sáng chiều phân bố khá đồng đều')
    if a=='PHÂN BỐ HS CHUYÊN - THƯỜNG':
        pie(df, df["CLASS-GROUP"],"PHÂN BỐ HS CHUYÊN - THƯỜNG")
        st.write('Học sinh lớp chuyên có hứng thú với AI hơn')
    if a=='PHÂN BỐ HS CÁC KHỐI':
        pie(df, df["GRADE"],"PHÂN BỐ HS CÁC KHỐI")
        st.write("Học sinh lớp 12 ít có số lượng đăng kí học AI ít nhất do bận ôn thi THPTQG")


import torch
import torchvision
import torchvision.transforms as transforms

import matplotlib.pyplot as plt
import numpy as np

import torch.nn as nn
import torch.nn.functional as F

import torch.optim as optim

"""
    PyTorch has created a package called torchvision for vision tasks that has data loaders for datasets such as ImageNet, CIFAR10, MNIST, etc. It also provides tools for transforming the data or writing custom datasets and data loaders.

    CIFAR10 contains data for 10 classes/categories of objects.

    Train on GPU
    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
    # move model to GPU
    cifar10_classifier = cifar10_classifier.to(device)
    # move first sample to GPU as an example
    image_0, label_0 = image_0.to(device), label_0.to(device)
"""

# global
classes = (
    "plane",
    "car",
    "bird",
    "cat",
    "deer",
    "dog",
    "frog",
    "horse",
    "ship",
    "truck",
)

def createDataset():
    # use the following to normalize the data and convert to tensors
    transform = transforms.Compose(
        [transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]
    )

    # create training dataset and data loader
    trainset = torchvision.datasets.CIFAR10(
        root="./data", train=True, download=True, transform=transform
    )
    trainloader = torch.utils.data.DataLoader(
        trainset, batch_size=4, shuffle=True, num_workers=0
    )

    # create test dataset and data loader to evaluate the performance of the trained model
    testset = torchvision.datasets.CIFAR10(
        root="./data", train=False, download=True, transform=transform
    )
    testloader = torch.utils.data.DataLoader(
        testset, batch_size=4, shuffle=False, num_workers=0
    )

    # printing some stats about the data
    print(
        f"Training set consists of {len(trainset)} samples & test set consists of {len(testset)}."
    )
    image_0, label_0 = trainset[0]
    c, w, h = image_0.shape
    print(
        f"The first normalized sample of the training data of shape {(c, w, h)} with label {label_0}:\n"
    )
    # print(image_0)
    # imshow(image_0, label_0)
    return [trainset, trainloader, testset, testloader]

"""
Because we have normalized the image, we need to unnormalize it first before plotting
"""

def imshow(img, label):
    img = img / 2 + 0.5  # unnormalize
    npimg = img.numpy()
    plt.imshow(np.transpose(npimg, (1, 2, 0)))
    plt.title(classes[label])
    plt.show()

"""
    Class to create the ML model (classifier)
"""

class Classifier(nn.Module):
    def __init__(self):
        super().__init__()
        self.feature_net = nn.Sequential(
            nn.Conv2d(in_channels=3, out_channels=6, kernel_size=5),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2, stride=2),
            nn.Conv2d(in_channels=6, out_channels=16, kernel_size=5),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2, stride=2),
        )
        self.classifier = nn.Sequential(
            nn.Linear(16 * 5 * 5, 120),
            nn.ReLU(),
            nn.Linear(120, 84),
            nn.ReLU(),
            nn.ReLU(),
        )

    def forward(self, x):
        features = self.feature_net(x)
        features = features.view(-1, 16 * 5 * 5)
        return self.classifier(features)

def main():
    data = createDataset()
    cifar10_classifier = Classifier()

    # Loss function
    criterion = nn.CrossEntropyLoss()  # standard choice

    # Optimizer
    optimizer = optim.SGD(cifar10_classifier.parameters(), lr=0.001, momentum=0.9)

    train(
        model=cifar10_classifier,
        optimizer=optimizer,
        criterion=criterion,
        trainloader=data[1],
    )
    test(testloader=data[3])

    per_class_accuracy(testloader=data[3])

"""
    There are a lot of tools for keeping track of loss functions
    Tensorboard is one of such tools
    https://pytorch.org/tutorials/intermediate/tensorboard_tutorial.html
"""

def train(
    model, optimizer, criterion, trainloader, n_epoch=3, save_to="./trained_cifar10.pth"
):
    # loop over the dataset multiple times
    for epoch in range(n_epoch):

        running_loss = 0.0
        num_iters = len(trainloader)
        for i, data in enumerate(trainloader, 0):
            # get the inputs; data is a list of [inputs, labels]
            inputs, labels = data

            # zero the parameter gradients
            optimizer.zero_grad()

            # forward + backward + optimize
            outputs = model(inputs)
            loss = criterion(outputs, labels)
            loss.backward()
            optimizer.step()

            # print statistics
            running_loss += loss.item()
            if i % 2000 == 1999:  # print every 2000 mini-batches
                print(
                    f"Loss at epoch [{epoch}/{n_epoch}], iteration [{i + 1}/{num_iters}]: {running_loss / 2000}"
                )
                running_loss = 0.0

    print(f"Training is done after {n_epoch} epochs!")

    torch.save(model.state_dict(), save_to)
    print(f"Trained model is saved to {save_to}.")

def test(testloader, load_pretrained_weights="./trained_cifar10.pth"):
    model = Classifier()
    model.load_state_dict(torch.load(load_pretrained_weights))

    correct = 0
    total = 0
    with torch.no_grad():
        for images, labels in testloader:

            outputs = model(images)
            _, predicted = torch.max(outputs.data, 1)
            total += labels.size(0)
            correct += (predicted == labels).sum().item()

    print(
        f"Accuracy of the network on the 10000 test images: {100 * correct / total} %"
    )

batch_size = 4

def per_class_accuracy(testloader, load_pretrained_weights="./trained_cifar10.pth"):
    model = Classifier()
    model.load_state_dict(torch.load(load_pretrained_weights))

    class_correct = np.zeros(10)
    class_total = np.zeros(10)

    with torch.no_grad():
        for images, labels in testloader:
            outputs = model(images)
            _, predicted = torch.max(outputs, 1)

            correct = (predicted == labels).squeeze()
            for i in range(batch_size):
                label = labels[i]
                class_correct[label] += correct[i].item()
                class_total[label] += 1

    for i in range(10):
        print(f"Accuracy of {classes[i]}: {100 * class_correct[i] / class_total[i]} %")

main()


import numpy as np
import torch
import pickle
from ogb.nodeproppred.dataset_pyg import PygNodePropPredDataset
import scipy.sparse as sp
from torch_geometric.utils import to_undirected
from torch_sparse import SparseTensor

def preprocess_features(features):
    """Row-normalize feature matrix and convert to tuple representation"""
    rowsum = np.array(features.sum(1))
    rowsum = (rowsum==0)*1+rowsum
    r_inv = np.power(rowsum, -1).flatten()
    r_inv[np.isinf(r_inv)] = 0.
    r_mat_inv = sp.diags(r_inv)
    features = r_mat_inv.dot(features)
    return features

def sys_normalized_adjacency_i(adj):
   adj = sp.coo_matrix(adj)
   adj = adj + sp.eye(adj.shape[0])
   row_sum = np.array(adj.sum(1))
   row_sum=(row_sum==0)*1+row_sum
   d_inv_sqrt = np.power(row_sum, -0.5).flatten()
   d_inv_sqrt[np.isinf(d_inv_sqrt)] = 0.
   d_mat_inv_sqrt = sp.diags(d_inv_sqrt)
   return d_mat_inv_sqrt.dot(adj).dot(d_mat_inv_sqrt).tocoo()

def sys_normalized_adjacency(adj):
   adj = sp.coo_matrix(adj)
   #adj = adj + sp.eye(adj.shape[0])
   row_sum = np.array(adj.sum(1))
   row_sum=(row_sum==0)*1+row_sum
   d_inv_sqrt = np.power(row_sum, -0.5).flatten()
   d_inv_sqrt[np.isinf(d_inv_sqrt)] = 0.
   d_mat_inv_sqrt = sp.diags(d_inv_sqrt)
   return d_mat_inv_sqrt.dot(adj).dot(d_mat_inv_sqrt).tocoo()

def sparse_mx_to_torch_sparse_tensor(sparse_mx):
    """Convert a scipy sparse matrix to a torch sparse tensor."""
    sparse_mx = sparse_mx.tocoo().astype(np.float32)
    indices = torch.from_numpy(
        np.vstack((sparse_mx.row, sparse_mx.col)).astype(np.int64))
    values = torch.from_numpy(sparse_mx.data)
    shape = torch.Size(sparse_mx.shape)
    return torch.sparse.FloatTensor(indices, values, shape)

def sparse_mx_to_torch_sparse_tensor(sparse_mx):
    """Convert a scipy sparse matrix to a torch sparse tensor."""
    sparse_mx = sparse_mx.tocoo().astype(np.float32)
    indices = torch.from_numpy(
        np.vstack((sparse_mx.row, sparse_mx.col)).astype(np.int64))
    values = torch.from_numpy(sparse_mx.data)
    shape = torch.Size(sparse_mx.shape)
    return torch.sparse.FloatTensor(indices, values, shape)

#Load dataset
dataset = PygNodePropPredDataset('ogbn-papers100M')
data = dataset[0]
split_idx = dataset.get_idx_split()

#get split indices
train_idx, valid_idx, test_idx = split_idx['train'], split_idx['valid'], split_idx['test']

edge_index = data.edge_index
N = data.num_nodes

#Load edges and create adjacency
row,col = edge_index
adj = SparseTensor(row=row,col=col,sparse_sizes=(N,N))
adj = adj.to_scipy(layout='csr')
print("Getting undirected matrix...")
adj = adj + adj.transpose()
print("Saving unnormalized adjacency matrix")
sp.save_npz('100M_undirected_adj.npz',adj)

#adj = sp.load_npz('100M_undirected_adj.npz')

feat = data.x.numpy()

feat = torch.from_numpy(feat).float()
print("Normalizing matrix A...")
adj_mat = sys_normalized_adjacency(adj)
sp.save_npz('normalized_adj.npz',adj_mat)
adj_mat = sparse_mx_to_torch_sparse_tensor(adj_mat)
print("Normalizing matrix A_i...")
adj_mat_i =  sys_normalized_adjacency_i(adj)
sp.save_npz('normalized_adj_i.npz',adj_mat_i)
adj_mat_i = sparse_mx_to_torch_sparse_tensor(adj_mat_i)

#creating 3-hop features

list_mat_train = []
list_mat_valid = []
list_mat_test = []
#Adding just features X
list_mat_train.append(feat[train_idx,:])
list_mat_valid.append(feat[valid_idx,:])
list_mat_test.append(feat[test_idx,:])

agg_feat = feat
agg_feat_i = feat

#Calculate features for 3 hops
for i in range(4):
    agg_feat = torch.spmm(adj_mat,agg_feat)
    agg_feat_i = torch.spmm(adj_mat_i,agg_feat_i)
    list_mat_train.append(agg_feat[train_idx,:])
    list_mat_valid.append(agg_feat[valid_idx,:])
    list_mat_test.append(agg_feat[test_idx,:])
    list_mat_train.append(agg_feat_i[train_idx,:])
    list_mat_valid.append(agg_feat_i[valid_idx,:])
    list_mat_test.append(agg_feat_i[test_idx,:])

with open('training.pickle',"wb") as fopen:
    pickle.dump(list_mat_train,fopen)

with open('validation.pickle',"wb") as fopen:
    pickle.dump(list_mat_valid,fopen)

with open('test.pickle',"wb") as fopen:
    pickle.dump(list_mat_test,fopen)

#save labels
labels = data.y.data
print(f"Type: {type(labels)},{labels.shape}")

labels_train = labels[train_idx].reshape(-1).long()
labels_valid = labels[valid_idx].reshape(-1).long()
labels_test = labels[test_idx].reshape(-1).long()

with open('labels.pickle',"wb") as fopen:
    pickle.dump([labels_train,labels_valid,labels_test],fopen)


import os

from django.core.management.base import BaseCommand

from couchdbkit.exceptions import ResourceNotFound

from dimagi.utils.couch.database import get_db

# possible expansion: allow this to accept doc ids on the command line
# for use like `cat doc_ids.txt | ./manage.py get_doc_domains `xargs echo`
class Command(BaseCommand):
    help = "Takes a file with one doc id per line and outputs their domains"

    def add_arguments(self, parser):
        parser.add_argument(
            '--full',
            action='store_true',
            dest='full',
            default=False,
            help="Output a full list of doc ids, followed by their domain",
        )

    def handle(self, filename, **options):
        self.full = options.get('full')
        if not os.path.exists(filename):
            return "File %s not found" % filename
        with open(filename) as file:
            doc_ids = file.readlines()
        self.domains = set()
        self.db = get_db()
        for id in doc_ids:
            self.handle_doc(id.strip())

    def handle_doc(self, id):
        try:
            doc = self.db.get(id)
        except ResourceNotFound:
            doc = {}
        domain = doc.get('domain', None)
        if self.full:
            doc_type = doc.get('doc_type', None)
            print("{0},{1},{2}".format(id, domain, doc_type))
        elif domain and domain not in self.domains:
            self.domains.add(domain)
            print(domain)


import os
from pathlib import Path

import pandas as pd

from src.dcr_to_petri_readable import Dcr2PetriTransport
from pm4py.objects.dcr.importer import importer as dcr_importer
from pm4py.objects.petri_net.importer import importer as pnml_importer

from concurrent.futures import ThreadPoolExecutor,wait

def run_analysis_on_pnmls(datasets_folder):

    pnml_res = []

    for root, dirs, files in os.walk(datasets_folder):
        for file in files:
            if file.endswith(".dcrxml"):
                pnml_res.append(os.path.join(root, file))

    analysis_results = {}
    for pnml in pnml_res:
        path_without_extension = os.path.splitext(pnml)[0]
        name_with_extension = os.path.basename(pnml)
        name = Path(pnml).stem
        version = name.split('_')[-1]
        # analysis
        tapn = pnml_importer.apply(pnml)
        analysis_results[name] = {}
        analysis_results[name]['version'] = version
        analysis_results[name]['#places'] = len(tapn.places)
        analysis_results[name]['#transitions'] = len(tapn.transitions)
        analysis_results[name]['#arcs'] = len(tapn.arcs)
    df = pd.DataFrame(analysis_results).T
    df.to_csv('models/analysis/optimization_analysis.csv')

def run_dcrxml_files(folder_with_dcr_xmls,override=False):
    dcrxml_res = []

    for root, dirs, files in os.walk(folder_with_dcr_xmls):
        for file in files:
            if file.endswith(".dcrxml"):
                dcrxml_res.append(os.path.join(root, file))
    dcrxml_res2 = []
    for dcrxml in dcrxml_res:
        if dcrxml.count('/')>4:
            dcrxml_res2.append(dcrxml)

    with ThreadPoolExecutor(16) as tpe:
            # futures = [tpe.submit(print, dcrxml) for dcrxml in dcrxml_res]
            futures = [tpe.submit(run_one_dcrxml_all_optimizations_file, dcrxml,override,2*60) for dcrxml in dcrxml_res]
    # with ThreadPoolExecutor(16) as tpe:
    #         # futures = [tpe.submit(print, dcrxml) for dcrxml in dcrxml_res]
    #         futures = [tpe.submit(run_one_dcrxml_all_optimizations_file, dcrxml,override,2*60) for dcrxml in dcrxml_res2]
    #         # for future in futures:
    #             # res = future.result(timeout=10*60)
    #             # print(f'[i] res: {res}')
    # exceptions = ['/home/vco/Datasets/12688511/CoSeLoG WABO 2.dcrxml',
    #               '/home/vco/Datasets/12681647/XES-files/event_log_choice_2_10000.dcrxml',
    #               '/home/vco/Datasets/12681647/XES-files/event_log_sequence_100_10000.dcrxml'
    #               ]
    # for dcrxml in dcrxml_res2:
    #     if dcrxml not in exceptions:
    #         run_one_dcrxml_all_optimizations_file(dcrxml, False, 2*60)

def run_one_dcrxml_all_optimizations_file(input_dcrxml_path, override=False, reachability_timeout=None):
    '''

    :param input_dcrxml_path: full path with extension
    :param reachability_timeout: timeout in seconds to search the reachability graph.
    :return:
    '''
    path_without_extension = os.path.splitext(input_dcrxml_path)[0]
    name_with_extension = os.path.basename(input_dcrxml_path)
    name = Path(input_dcrxml_path).stem

    dcrxml_files = [
        [input_dcrxml_path, f'{path_without_extension}_unoptimized.pnml', False, False, True,'unoptimized'],
        [input_dcrxml_path, f'{path_without_extension}_dcranalysis.pnml', True, False, False,'dcranalysis'],
        [input_dcrxml_path, f'{path_without_extension}_pnreachability.pnml', False, True, False,'pnreachability'],
        [input_dcrxml_path, f'{path_without_extension}_fulloptimization.pnml', True, True, False,'fulloptimization'],
    ]
    for mapping_call in dcrxml_files:
        if os.path.isfile(mapping_call[1]) and not override:
            print(f'[i] File {Path(mapping_call[1]).stem} already exists.')
            pass
        else:
            d2p = Dcr2PetriTransport(preoptimize=mapping_call[2], postoptimize=mapping_call[3],
                                     map_unexecutable_events=mapping_call[4],debug=False)
            d2p.print_steps = True
            d2p.reachability_timeout = 60*10 #10 minutes
            if reachability_timeout:
                d2p.reachability_timeout = reachability_timeout
            print(f'[i] Import {mapping_call[0]}')
            dcr = dcr_importer.apply(mapping_call[0])
            print(f'[i] Converting {mapping_call[0]} as {mapping_call[5]}')
            tapn, m = d2p.dcr2tapn(dcr, tapn_path=mapping_call[1])
            print(f'[i] Done for: {mapping_call[0]}!')

if __name__ == '__main__':
    # run_one_dcrxml_all_optimizations_file('/home/vco/Datasets/Hospital_log.dcrxml')
    # run_one_dcrxml_all_optimizations_file('/home/vco/Datasets/12683249/Road_Traffic_Fine_Management_Process.dcrxml',override=True)
    #run_one_dcrxml_all_optimizations_file('/home/vco/Projects/dcr-to-tapn/models/results/Road_Traffic_Fine_Management_Process.dcrxml', override=True)
    # run_one_dcrxml_all_optimizations_file('/home/vco/Projects/dcr-to-tapn/models/debug/Road_Traffic_Fine_Management_Process_test.dcrxml', override=True)
    # run_dcrxml_files('/home/vco/Datasets/')

    run_dcrxml_files('models/results/',override=True)
    print('[i] Done!')


# Bu uygulamada argparse kütüphanesinin kullanımını gösteriyoruz
# programa gelen argümanları daha iyi kontrol etmek için kullanılıyor.

import argparse

#parser nesnesi oluşturuluyor
parser = argparse.ArgumentParser()

#parser nesnesine kullanıcıdan alınacak bir argüman tanıtılıyor
#ilk parametre argümanın alacağı değişken ismi
#ikinci parametre argümanın yanında çıkacak olan yardım mesajı

parser.add_argument("ilk_arguman",help="ilk arguman yardım");

#argümanlar yakalanıyor
args = parser.parse_args()

#ilk argümanın değeri ekrana çıkartılıyor
print(args.ilk_arguman)


from shapely.geometry import Polygon

from tests import csv_point_input_location, union_polygon_input_location
from tests.test_base import TestBase

class TestConstructors(TestBase):

    def test_st_envelope_aggr(self):
        point_csv_df = self.spark.read.format("csv").\
            option("delimiter", ",").\
            option("header", "false").\
            load(csv_point_input_location)

        point_csv_df.createOrReplaceTempView("pointtable")
        point_df = self.spark.sql("select ST_Point(cast(pointtable._c0 as Decimal(24,20)), cast(pointtable._c1 as Decimal(24,20))) as arealandmark from pointtable")
        point_df.createOrReplaceTempView("pointdf")
        boundary = self.spark.sql("select ST_Envelope_Aggr(pointdf.arealandmark) from pointdf")

        coordinates = [
            (1.1, 101.1),
            (1.1, 1100.1),
            (1000.1, 1100.1),
            (1000.1, 101.1),
            (1.1, 101.1)
        ]

        polygon = Polygon(coordinates)

        assert boundary.take(1)[0][0] == polygon

    def test_st_union_aggr(self):
        polygon_csv_df = self.spark.read.format("csv").\
            option("delimiter", ",").\
            option("header", "false").\
            load(union_polygon_input_location)

        polygon_csv_df.createOrReplaceTempView("polygontable")
        polygon_csv_df.show()
        polygon_df = self.spark.sql("select ST_PolygonFromEnvelope(cast(polygontable._c0 as Decimal(24,20)),cast(polygontable._c1 as Decimal(24,20)), cast(polygontable._c2 as Decimal(24,20)), cast(polygontable._c3 as Decimal(24,20))) as polygonshape from polygontable")
        polygon_df.createOrReplaceTempView("polygondf")
        polygon_df.show()
        union = self.spark.sql("select ST_Union_Aggr(polygondf.polygonshape) from polygondf")

        assert union.take(1)[0][0].area == 10100


from ast import literal_eval
from base64 import b64decode
from logging import getLogger
from time import time
from typing import Union, Optional
from .models import UserProfile
from .exceptions import MojangError
from utils.http import session
# THIS IS A MODIFIED VERSION OF summer's wrapper
# This one is async
from ast import literal_eval
from base64 import b64decode
from logging import getLogger
from time import time
from typing import Union, Optional

from utils.http import session
from .exceptions import MojangError
from .models import UserProfile

log = getLogger(__name__)

class MojangAPI:
    # NOTE: Names are held for exactly 37 days after being released.
    # Update this value (by converting days to seconds) if the duration is ever changed.
    _NAME_HOLD_DURATION = 3196800000

    @classmethod
    async def get_uuid(cls, username: str, timestamp: Optional[int] = None) -> Union[str, None]:
        """Convert a Minecraft name to a UUID.

        Args:
            username:  The Minecraft username to be converted.
            timestamp (optional): Get the username's UUID at a specified UNIX timestamp.
                You can also get the username's first UUID by passing 0.
                However, this only works if the name was changed at least once, or if the account is legacy.

        Returns:
            str: The UUID. Otherwise, None if the username does not exist.
        """

        if timestamp is None:
            timestamp_now = int(time() * 1000.0)
            timestamp = int((timestamp_now - cls._NAME_HOLD_DURATION) / 1000)

        resp = await session.get(f"https://api.mojang.com/users/profiles/minecraft/{username}?at={timestamp}")
        if resp.ok:
            try:
                json = await resp.json()
                return json["id"]
            except Exception:
                return None
        return None

    @staticmethod
    async def get_uuids(names: list) -> dict:
        """Convert up to 10 usernames to UUIDs in a single network request.

        Args:
            names: The Minecraft username(s) to be converted.
                If more than 10 are included, only the first 10 will be parsed.

        Returns:
            dict: username:uuid pairs of the converted usernames. Names are also case-corrected.
                If a username does not exist, it will not be included in the returned dictionary.
        """
        if len(names) > 10:
            names = names[:10]
        async with session.post("https://api.mojang.com/profiles/minecraft", data=names) as ses:
            data = await ses.json()

        if not isinstance(data, list):
            if data.get("error"):
                raise ValueError(data["errorMessage"])
            else:
                raise MojangError(data)

        sorted_names = dict()
        for name_data in data:
            sorted_names[name_data["name"]] = name_data["id"]

        return sorted_names

    @staticmethod
    async def get_username(uuid: str) -> Union[str, None]:
        """Convert a UUID to a username.

        Args:
            uuid: The Minecraft UUID to be converted to a username.

        Returns:
            str: UUID if username exists. None otherwise.
        """
        resp = await session.get(f"https://sessionserver.mojang.com/session/minecraft/profile/{uuid}")
        if resp.ok:
            json = await resp.json()
            return json["name"]
        return None

    @classmethod
    async def get_drop_timestamp(cls, username: str) -> Union[int, None]:
        """Get the timestamp of when a username drops

        Args:
            username: Minecraft name to get drop date of

        Returns:
            int: The drop timestamp. Otherwise, None if the username is not being released/dropped.
        """
        uuid = cls.get_uuid(username)
        if not uuid:
            raise ValueError("Username is invalid. Failed to convert username to UUID")
        resp = await session.get(f"https://api.mojang.com/user/profiles/{uuid}/names")
        name_changes = [name_change for name_change in reversed(await resp.json())]
        for i, name_info in enumerate(name_changes):
            if name_info["name"].lower() == username.lower():
                try:
                    name_changed_timestamp = name_changes[i - 1]["changedToAt"]
                    drop_timestamp = (name_changed_timestamp + cls._NAME_HOLD_DURATION) / 1000
                    return int(drop_timestamp)
                except KeyError:
                    return None

    @staticmethod
    async def get_profile(uuid: str) -> Union[UserProfile, None]:
        """Returns a `UserProfile` object

        `UserProfile` Attributes:
            id (str): UUID of the profile
            name (str): Name of the profile
            cape_url (str or None): URL to the profile's cape
            skin_url (str or None): URL to the profile's skin
            skin_model (str): Skin model of the profile
            is_legacy_profile (bool): Check if the profile is legacy
            timestamp (int): Timestamp of when the profile was retrieved
        """
        resp = await session.get(f"https://sessionserver.mojang.com/session/minecraft/profile/{uuid}")
        try:
            js = await resp.json()
            value = js["properties"][0]["value"]
        except KeyError:
            return None
        user_profile = literal_eval(b64decode(value).decode())
        return UserProfile(user_profile)
    @staticmethod
    async def get_name_history(uuid: str) -> list:
        """Get a user's name history

        Args:
            uuid: The user's UUID.

        Returns:
            list: A list of dictionaries, each of which contains a name:changed_to_at pair.
                If changed_to_at is set to 0, it is because it is the profile's first name.
        """
        ses = await session.get(f"https://api.mojang.com/user/profiles/{uuid}/names")
        name_history = await ses.json()

        name_data = list()
        for data in name_history:
            name_data_dict = dict()
            name_data_dict["name"] = data["name"]
            if data.get("changedToAt"):
                name_data_dict["changed_to_at"] = data["changedToAt"]
            else:
                name_data_dict["changed_to_at"] = 0
            name_data.append(name_data_dict)
        return name_data

    @staticmethod
    async def get_api_status() -> dict:
        """Get the API / network status of various Mojang services

        Returns:
            dict: Returns dictionary with status of various Mojang services.
                Possible values are green (no issues), yellow (some issues), red (service unavailable).
        """
        re = await session.get("https://status.mojang.com/check")
        data = await re.json()

        servers = dict()
        for server_data in data:
            for k, v in server_data.items():
                servers[k] = v
        return servers

    @staticmethod
    async def get_blocked_servers() -> list:
        """Returns a list of SHA1 hashes of current blacklisted servers that do not follow EULA.
        These servers have to abide by the EULA or be shut down forever. The hashes are not cracked.
        """
        return await session.get("https://sessionserver.mojang.com/blockedservers").text.splitlines()

    @staticmethod
    async def get_sale_statistics(item_sold_minecraft: bool = True,
                            prepaid_card_redeemed_minecraft: bool = True,
                            item_sold_cobalt: bool = False,
                            item_sold_scrolls: bool = False,
                            prepaid_card_redeemed_cobalt: bool = False,
                            item_sold_dungeons: bool = False
                            ):
        """Get statistics on the sales of Minecraft.
        You will receive a single object corresponding to the sum of sales of the requested type(s)
        At least one type of sale must be set to True.

        Returns:
            dict: the sales metrics. Possible keys include `total`, `last24h` and `sale_velocity_per_seconds`
        """
        options = [k for k, v in locals().items() if v]

        if not options:
            raise MojangError("Invalid parameters supplied. Include at least one metric key.")

        sepo = await session.post("https://api.mojang.com/orders/statistics", data={"metricKeys": options})
        data = await sepo.json()
        metrics = dict()
        metrics["total"] = data["total"]
        metrics["last24h"] = data["last24h"]
        metrics["sale_velocity_per_seconds"] = data["saleVelocityPerSeconds"]
        return metrics


import json
import time
import math
import requests
import os
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'lastwill.settings')
import django
django.setup()

from rest_framework.exceptions import ValidationError
from lastwill.contracts.submodels.tron import convert_address_to_hex
from lastwill.consts import TRON_REPLENISH_THRESHOLD
from lastwill.settings import TRON_REPLENISH_ACCOUNT, TRON_REPLENISH_CHECK_ACCOUNT, NETWORKS

def check_account(account_params, tron_url):
    data = {"address": account_params['address']}
    result = requests.post(tron_url + '/wallet/getaccountresource', data=json.dumps(data))
    account_info = json.loads(result.content.decode())

    if 'NetLimit' in account_info:
        if 'NetUsed' in account_info:
            net_available = account_info['NetLimit'] - account_info['NetUsed']
        else:
            net_available = account_info['NetLimit']
    else:
        net_available = 0

    if 'EnergyLimit' in account_info:
        if 'EnergyUsed' in account_info:
            energy_available = account_info['EnergyLimit'] - account_info['EnergyUsed']
        else:
            energy_available = account_info['EnergyLimit']
    else:
        energy_available = 0

    return {
        'net_delta': net_available,
        'energy_delta': energy_available
    }

def freeze_tron_resource(account_params, tron_url, amount, resource_name):
    data = {
        "owner_address": account_params['address'],
        "frozen_balance": amount * TRON_REPLENISH_THRESHOLD['MIN_TRX'],
        "frozen_duration": 3,
        "resource": resource_name
    }
    result = requests.post(tron_url + '/wallet/freezebalance', data=json.dumps(data))
    freeze_tx = json.loads(result.content.decode())
    print('transaction created')
    print(freeze_tx)
    freeze_tx = {'transaction': freeze_tx,
                 'privateKey':  account_params['private_key']}
    trx1 = json.dumps(freeze_tx)
    print('transaction sign')
    sign_res = requests.post(tron_url + '/wallet/gettransactionsign', data=trx1)
    trx_signed = json.loads(sign_res.content.decode())
    trx2 = json.dumps(trx_signed)
    print(trx2)
    for i in range(5):
        print('attempt=', i)
        result = requests.post(tron_url + '/wallet/broadcasttransaction', data=trx2)
        print(result.content.decode())
        answer = json.loads(result.content.decode())
        print('answer=', answer, flush=True)
        if answer['result']:
            params = {'value': trx_signed['txID']}
            result = requests.post(tron_url + '/wallet/gettransactionbyid', data=json.dumps(params))
            ret = json.loads(result.content.decode())
            if ret:
                print('tx_hash=', trx_signed['txID'], flush=True)
                return
        time.sleep(5)
    else:
                raise ValidationError({'result': 1}, code=400)

def check_and_freeze(account_params, tron_url):
    account_resources = check_account(account_params, tron_url)
    try:
        if account_resources['net_delta'] <= TRON_REPLENISH_THRESHOLD['NET']:
            print('Started automatic replenish of NET for {addr}'.format(addr=account_params["address"]))
            freeze_tron_resource(account_params, tron_url, 1, "BANDWIDTH")
            print('Finished automatic replenish')
        else:
            print('account {addr} NET is above required limit on'.format(addr=account_params["address"]),
                  account_resources['net_delta'] - TRON_REPLENISH_THRESHOLD['NET']
                  )
        if account_resources['energy_delta'] <= TRON_REPLENISH_THRESHOLD['ENERGY']:
            print('Started automatic replenish of ENERGY for {addr}'.format(addr=account_params["address"]))
            trx_energy_amount = math.ceil(TRON_REPLENISH_THRESHOLD['ENERGY'] / TRON_REPLENISH_THRESHOLD['MIN_TRX'])
            freeze_tron_resource(account_params, tron_url, trx_energy_amount, "ENERGY")
            print('Finished automatic replenish')
        else:
            print('account {addr} ENERGY is above required limit on'.format(addr=account_params["address"]),
                  account_resources['energy_delta'] - TRON_REPLENISH_THRESHOLD['ENERGY']
                  )
    except Exception as e:
        print(e)
        print('TRON autoreplenish for {addr} failed'.format(addr=account_params["address"]))

def convert_trx_resources(network):
    tron_url = 'http://%s:%s' % (str(NETWORKS[network]['host']), str(NETWORKS[network]['port']))
    # account_parameters = {
    #     "address": '41' + convert_address_to_hex(NETWORKS[network]['address'])[2:],
    #     "private_key": NETWORKS[network]['private_key']
    # }
    account_main = {
        "address": '41' + convert_address_to_hex(TRON_REPLENISH_ACCOUNT['address'])[2:],
        "private_key": TRON_REPLENISH_ACCOUNT['private_key']
    }
    account_check = {
        "address": '41' + convert_address_to_hex(TRON_REPLENISH_CHECK_ACCOUNT['address'])[2:],
        "private_key": TRON_REPLENISH_CHECK_ACCOUNT['private_key']
    }

    check_and_freeze(account_main, tron_url)
    check_and_freeze(account_check, tron_url)


import requests
import json
import telebot

# Chave do bot no Telegram
bot_key = 'BOT KEY AQUI'

# Chave da API da OpenAI
openai_key = 'OPENAI KEY AQUI'

# URL da API
api_url = 'https://api.openai.com/v1/chat/completions'

# Função para enviar mensagem para a API
def send_message(messages, message):
    headers = {
        'Content-Type': 'application/json',
        'Authorization': f'Bearer {openai_key}'
    }

    data = {
        'model': 'gpt-3.5-turbo',
        'messages': messages + [{"role": "user", "content": message}]
    }

    response = requests.post(api_url, headers=headers, data=json.dumps(data))

    if response.status_code == 200:
        return json.loads(response.text)['choices'][0]['message']['content']
    else:
        return None

# Inicializa o bot
bot = telebot.TeleBot(bot_key)

# Armazena as mensagens da conversa
conversation = []

@bot.message_handler(commands=['start'])
def start(message):
    bot.send_message(message.chat.id, "Olá! Eu sou o ChatPEO. Me envie uma mensagem e eu tentarei respondê-lo da melhor forma possível.")

# Função para responder as mensagens
@bot.message_handler(func=lambda message: True)
def echo_all(message):
    # Adiciona a mensagem atual à lista
    conversation.append({"role": "user", "content": message.text})

    # Envia a mensagem para a API
    response = send_message(conversation, message.text)

    if response is not None:
        # Adiciona a mensagem da API à lista
        conversation.append({"role": "assistant", "content": response})

        # Envia a resposta do bot
        bot.reply_to(message, response)
    else:
        bot.reply_to(message, 'Desculpe, não consegui entender.')

# Inicia o bot
bot.polling()


from urllib.request import Request, urlopen
from bs4 import BeautifulSoup as soup
import re
import random as r

Foods = []	# This array will store all our Food objects

# This is our Food class, we will use this to store all the required data from
# a recipe as its own object.
class Food:
	def __init__(self, name, calorie, summary, img_url, bluePoint, greenPoint, purplePoint, recipeKeys):

		self.name = name
		self.calorie = calorie
		self.summary = summary
		self.img_url = img_url
		self.bluePoint = bluePoint
		self.greenPoint = greenPoint
		self.purplePoint = purplePoint
		self.recipeKeys = recipeKeys

def slice_summary(summary):	#slice_summary is used to slice the summaries into
	whitespaceCount = 0		# a more readable form with multiple lines.
	i = 0
	for char in summary:

		if(char == " "):
			whitespaceCount = whitespaceCount + 1
		if(whitespaceCount == 7):
			summary = summary[:i] + "\n" + summary[i:]
			whitespaceCount = 0
			i = i + 1

		i = i + 1

	return summary

def parseData(url):
	client = urlopen(url)
	html = client.read()
	client.close
	page_soup = soup(html, "html.parser")

	posts = []
	# Getting all posts from site
	posts.extend(page_soup.findAll("article",{"class":"post teaser-post even"}))
	posts.extend(page_soup.findAll("article",{"class":"post teaser-post odd"}))

	# Creating a Food object out of every post
	for post in posts:
		# Getting the name of the recipe
		name = post.find("h2",{"class":"title"}).get_text()

		# Getting the calorie amount for the recipe, if we fail then that means
		# that the post in question is not a recipe, so we just skip it
		# and move on. This way we get rid of posts like
		# "7 DAY HEALTHY MEAL PLAN (NOV 16-22)" that are not recipes
		try:
			calorie = float(post.find("span",{"class":"icon-star"}).get_text())
			calorie = int(round(calorie))
		except(AttributeError):
			continue

		# Getting the summary for the recipe
		summary = post.find("p",{"class":"excerpt"}).get_text()
		summary = slice_summary(summary)

		# Getting the image for the recipe
		# Notice how we only get the url for later use
		# and wont download the image right away.
		image = post.find("img")
		img_url = (image["src"])

		# Getting the blue, green and purple point values, if there are none
		# assign -1.
		try:
			greenPoint = int(post.find("span",{"class":"smart-points green"}).get_text())
			bluePoint = int(post.find("span",{"class":"smart-points blue"}).get_text())
			purplePoint = int(post.find("span",{"class":"smart-points purple"}).get_text())

		except(AttributeError):
			greenPoint = -1
			bluePoint = -1
			purplePoint = -1

		# Getting the recipekeys for each recipe
		tmp = []
		icons = post.find("div",{"class":"post-meta"})
		recipeKey = icons.findAll("img")

		for i in recipeKey:
			tmp.append(i["alt"])

		# Casting the list as a dictionary to remove duplicate recipekeys
		recipeKeys = list(dict.fromkeys(tmp))

		Foods.append(Food(name, calorie, summary, img_url, bluePoint, greenPoint, purplePoint, recipeKeys))

def scrape():
		# Scraping the first 30 pages
	for i in range(1,31):
		print("{0}/30  pages scraped".format(i))
		# Utilizing a user-agent so that our requests will go through
		url = Request("https://skinnytaste.com/page/{0}".format(i), headers={'User-Agent': 'Mozilla/5.0'})
		parseData(url)
	print("Done!")

def get_list(minimum,maximum):
	arr = []
	for i in Foods:
		if(i.calorie >= minimum and i.calorie <= maximum):
			arr.append(i)
		if(len(arr) == 5):
			return arr
	return arr

def get_data():
	scrape()
	return Foods


import typing as T

import numpy as np

import megengine as mge
import megengine.functional as F
from megengine import Parameter

from ..logger import get_logger
from .init import ones_, zeros_
from .module import Module

class GroupNorm(Module):
    r"""Applies Group Normalization over a mini-batch of inputs
    Refer to `Group Normalization <https://arxiv.org/abs/1803.08494>`__

    .. math::
            y = \frac{x - \mathrm{E}[x]}{ \sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta

    The mean and standard-deviation are calculated separately over the each group.
    :math:`\\gamma` and :math:`\\beta` are learnable affine transform parameters of
    attr:`num_channels` if :attr:`affine` is ``True``.

    Args:
        num_groups (int): number of groups that divided from channels.
        num_channels (int): number of channels expected in input
        eps: a value added to the denominator for numerical stability. Default: 1e-5
        affine: this module has learnable affine parameters (weight, bias) when affine is set to be True.

    Shape:
        - Input: :math:`(N, C, H, W)` (now only support NCHW format tensor)
        - Output: :math:`(N, C, H, W)` (same shape as input)

    Examples:
        >>> import numpy as np
        >>> inp = Tensor(np.arange(2 * 3 * 4 * 4).astype(np.float32).reshape(2, 3, 4, 4))
        >>> m = M.GroupNorm(3, 3)
        >>> out = m(inp)
        >>> out.numpy().shape
        (2, 3, 4, 4)
    """

    def __init__(self, num_groups, num_channels, eps=1e-5, affine=True, **kwargs):
        super().__init__(**kwargs)
        assert num_channels % num_groups == 0
        self.num_groups = num_groups
        self.num_channels = num_channels
        self.eps = eps
        self.affine = affine
        if self.affine:
            self.weight = Parameter(np.ones(num_channels, dtype=np.float32))
            self.bias = Parameter(np.zeros(num_channels, dtype=np.float32))
        else:
            self.weight = None
            self.bias = None
        self.reset_parameters()

    def reset_parameters(self):
        if self.affine:
            ones_(self.weight)
            zeros_(self.bias)

    def forward(self, x):
        x = F.nn.group_norm(
            x, self.num_groups, self.affine, self.weight, self.bias, self.eps
        )
        return x

    def _module_info_string(self) -> str:
        s = (
            "groups={num_groups}, channels={num_channels}, "
            "eps={eps}, affine={affine}"
        )
        return s.format(**self.__dict__)

class InstanceNorm(Module):
    r"""Applies Instance Normalization over a mini-batch of inputs
    Refer to `Instance Normalization <https://arxiv.org/abs/1607.08022>`__

    .. math::
            y = \frac{x - \mathrm{E}[x]}{ \sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta

    The mean and standard-deviation are calculated per-dimension separately for each object in a mini-batch.
    :math:`\\gamma` and :math:`\\beta` are learnable affine transform parameters of
    attr:`num_channels` if :attr:`affine` is ``True``.
    Note that InstanceNorm equals using GroupNorm with num_groups = num_channels.

    Args:
        num_channels (int): number of channels expected in input
        eps: a value added to the denominator for numerical stability. Default: 1e-5
        affine: this module has learnable affine parameters (weight, bias) when affine is set to be True.

    Shape:
        - Input: :math:`(N, C, H, W)` (now only support NCHW format tensor)
        - Output: :math:`(N, C, H, W)` (same shape as input)

    Examples:
        >>> import numpy as np
        >>> inp = Tensor(np.arange(2 * 3 * 4 * 4).astype(np.float32).reshape(2, 3, 4, 4))
        >>> m = M.InstanceNorm(3)
        >>> out = m(inp)
        >>> out.numpy().shape
        (2, 3, 4, 4)
    """

    def __init__(self, num_channels, eps=1e-05, affine=True, **kwargs):
        super().__init__(**kwargs)
        self.num_channels = num_channels
        self.eps = eps
        self.affine = affine
        if self.affine:
            self.weight = Parameter(np.ones(num_channels, dtype="float32"))
            self.bias = Parameter(np.zeros(num_channels, dtype="float32"))
        else:
            self.weight = None
            self.bias = None
        self.reset_parameters()

    def reset_parameters(self):
        if self.affine:
            ones_(self.weight)
            zeros_(self.bias)

    def forward(self, x):
        x = F.nn.instance_norm(x, self.affine, self.weight, self.bias, self.eps)
        return x

    def _module_info_string(self) -> str:
        s = "channels={num_channels}, eps={eps}, affine={affine}"
        return s.format(**self.__dict__)

class LayerNorm(Module):
    r"""Applies Layer Normalization over a mini-batch of inputs
    Refer to `Layer Normalization <https://arxiv.org/pdf/1607.06450v1.pdf>`_

    .. math::
            y = \frac{x - \mathrm{E}[x]}{ \sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta

    The mean and standard-deviation are calculated separately over the last
    certain number dimensions which have to be of the shape specified by
    :attr:`normalized_shape`.
    :math:`\\gamma` and :math:`\\beta` are learnable affine transform parameters of
    :attr:`normalized_shape` if :attr:`affine` is ``True``.
    The standard-deviation is calculated via the biased estimator.

    .. note::

        Unlike Batch Normalization and Instance Normalization, which applies
        scalar scale and bias for each entire channel/plane,
        Layer Normalization applies per-element scale and bias.

    Args:
        normalized_shape(int or tuple): input shape from an expected input of size
            size :math:`[*, normalized\_shape[0], normalized\_shape[1], ..., normalized\_shape[-1]]`.
            If it is a single integer, this module will normalize over the last dimension
            which is expected to be of that specific size.
        eps: a value added to the denominator for numerical stability. Default: 1e-5
        affine: this module has learnable affine parameters (weight, bias) when affine is set to be True.

    Shape:
        - Input: :math:`(N, *)` (2-D, 3-D, 4-D or 5-D tensor)
        - Output: :math:`(N, *)` (same shape as input)

    Examples:
        >>> import numpy as np
        >>> inp = Tensor(np.arange(2 * 3 * 4 * 4).astype(np.float32).reshape(2, 3, 4, 4))
        >>> m = M.LayerNorm((4, 4))
        >>> out = m(inp)
        >>> out.numpy().shape
        (2, 3, 4, 4)
    """

    def __init__(self, normalized_shape, eps=1e-05, affine=True, **kwargs):
        super().__init__(**kwargs)
        if isinstance(normalized_shape, int):
            normalized_shape = (normalized_shape,)
        self.normalized_shape = tuple(normalized_shape)
        self.eps = eps
        self.affine = affine
        if self.affine:
            self.weight = Parameter(np.ones(self.normalized_shape, dtype="float32"))
            self.bias = Parameter(np.zeros(self.normalized_shape, dtype="float32"))
        else:
            self.weight = None
            self.bias = None

        self.reset_parameters()

    def reset_parameters(self):
        if self.affine:
            ones_(self.weight)
            zeros_(self.bias)

    def forward(self, x):
        x = F.nn.layer_norm(
            x, self.normalized_shape, self.affine, self.weight, self.bias, self.eps
        )
        return x

    def _module_info_string(self) -> str:
        s = "normalized_shape={normalized_shape}, eps={eps}, affine={affine}"
        return s.format(**self.__dict__)

class GeneralNorm(Module):
    r"""Applies General Normalization over a mini-batch of inputs

    .. math::
            y = \frac{x - \mathrm{E}[x]}{ \sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta

    The mean and standard-deviation are calculated separately according to the axis
    given by :attr:`normalized_axis`.
    :math:`\\gamma` and :math:`\\beta` are learnable affine transform parameters
    if :attr:`affine` is ``True``.
    The standard-deviation is calculated via the biased estimator.

    Args:
        normalized_shape(int, list or tuple): the shape of input needs to be normalized, normalized_shape must be specified when affine is true. When affine=true, we will directly use this shape to initialize weight/bias. Please ensure that the order is correct. Default: None
        normalized_axis(int, list or tuple): the axis of input needs to be normalized, one-to-one correspondence between normalized_axis and normalized_shape. Default: -1
        eps: a value added to the denominator for numerical stability. Default: 1e-5
        affine: this module has learnable affine parameters (weight, bias) when affine is set to be True.

    Shape:
        - Input: :math:`(N, *)` (2-D, 3-D, 4-D or 5-D tensor)
        - Output: :math:`(N, *)` (same shape as input)

    Examples:
        >>> import numpy as np
        >>> inp = Tensor(np.arange(2 * 3 * 4 * 4).astype(np.float32).reshape(2, 3, 4, 4))
        >>> m = M.GeneralNorm((2, 3), (0, 1))
        >>> out = m(inp)
        >>> out.numpy().shape
        (2, 3, 4, 4)
        >>> m = M.GeneralNorm((3, 4), (1, -2)) # Please be careful.
        >>> out = m(inp)
        >>> out.numpy().shape
        (2, 3, 4, 4)
        >>> m = M.GeneralNorm((2, 4, 3), (0, 2, 1)) # Incorrect initialization, the order of normalized_axis is incorrect, should be adjusted to m = M.GeneralNorm((2, 3, 4), (0, 1, 2)).
        >>> m = M.GeneralNorm((2, 4, 3), (0, -2, 1)) # Incorrect initialization, the order of normalized_axis is incorrect, should be adjusted to m = M.GeneralNorm((2, 3, 4), (0, 1, -2)).
        >>> m = M.GeneralNorm((3, 4), (3, -1)) # Incorrect initialization, because axis=-1 and axis=3 are the same axis, namely axis=3.
    """

    def __init__(
        self, normalized_shape=None, normalized_axis=0, eps=1e-05, affine=True, **kwargs
    ):
        super().__init__(**kwargs)

        self.eps = eps
        self.affine = affine

        if self.affine:
            assert (
                normalized_shape is not None
            ), "normalized_shape must be specified when affine is true"
            assert (
                normalized_axis is not None
            ), "normalized_axis must be specified when affine is true"
            if not isinstance(normalized_axis, T.Sequence):
                normalized_axis = [normalized_axis]
            if not isinstance(normalized_shape, T.Sequence):
                normalized_axis = [normalized_shape]
            assert isinstance(normalized_axis, (list, tuple))
            assert isinstance(normalized_shape, (list, tuple))

            assert len(normalized_axis) == len(
                normalized_shape
            ), "The size of normalized_axis and normalized_shape are different"
            assert len(set(normalized_axis)) == len(
                normalized_axis
            ), "there are duplicate axis in list normalized_axis"

            self.weight = Parameter(np.ones(normalized_shape, dtype="float32"))
            self.bias = Parameter(np.zeros(normalized_shape, dtype="float32"))
        else:
            self.weight = None
            self.bias = None

        self.normalized_shape = normalized_shape
        self.normalized_axis = normalized_axis
        self.reset_parameters()

    def reset_parameters(self):
        if self.affine:
            ones_(self.weight)
            zeros_(self.bias)

    def forward(self, x):
        self.normalized_axis = [
            num + x.ndim if num < 0 else num for num in self.normalized_axis
        ]
        assert self.normalized_axis == sorted(
            self.normalized_axis
        ), "The order of normalized_axis is incorrect, should be {}, but got {}. Please specify the values of axis in the correct order in normalized_axis".format(
            sorted(self.normalized_axis), self.normalized_axis
        )
        inp_shape = x.shape
        for i in range(len(self.normalized_axis)):
            assert (
                inp_shape[self.normalized_axis[i]] == self.normalized_shape[i]
            ), "inp.shape={}, normalized_axis={}, normalized_shape={}, inp.shape[normalized_axis[{}]]({}) != normalized_shape[{}]({})".format(
                x.shape,
                self.normalized_axis,
                self.normalized_shape,
                i,
                inp_shape[self.normalized_axis[i]],
                i,
                self.normalized_shape[i],
            )

        x = F.nn.general_norm(
            x, self.normalized_axis, self.affine, self.weight, self.bias, self.eps
        )
        return x

    def _module_info_string(self) -> str:
        s = "normalized_shape={normalized_shape}, normalized_axis={normalized_axis}, eps={eps}, affine={affine}"
        return s.format(**self.__dict__)


import os
import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
from torchvision import datasets
from utils.cli import get_argument
from utils.model import NN
from utils.make_dataset import LoadData, get_json

def training(
    model,
    criterion,
    optimizer,
    featureloader,
    labelloader,
    epochs=15,
    print_every=40
):
    epoch_loss = 0
    steps = 0

    for epoch in range(epochs):
        for features, labels in zip(featureloader, labelloader):
            steps += 1
            features = features.type(torch.FloatTensor)
            labels = labels.type(torch.FloatTensor)
            labels.resize_(labels.shape[0], 1)

            output = model.forward(features)
            loss = criterion(output, labels)

            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

            epoch_loss += loss.item()

            if steps % print_every == 0:
                print('Epoch: {}/{}..'.format(epoch, epochs),
                      'Training loss: {:.3f}'.format(epoch_loss/print_every))

                epoch_loss = 0

def save_model(
    model,
    epochs,
    optimizer,
    scaler,
    filename='checkpoint.pth',
    save_dir='reports'
):
    checkpoint = {
        'model': model,
        'model_state': model.state_dict(),
        'epochs': epochs,
        'optim': optimizer,
        'optim_state': optimizer.state_dict(),
        'scaler': scaler
    }
    torch.save(checkpoint, os.path.join(save_dir, filename))
    print('[INFO] model is saved!')

if __name__ == '__main__':
    argument = get_argument()
    feature_name = get_json('references/col_to_feat.json')
    train = LoadData(argument.data, names=feature_name, sep='\s+')
    scaled_train, scaler = train.standardize()
    # convert dataset into tensor
    featureset = torch.from_numpy(scaled_train[:, 2:])
    labelset = torch.from_numpy(train.target)
    # define data loader
    featureloader = torch.utils.data.DataLoader(featureset, batch_size=32)
    labelloader = torch.utils.data.DataLoader(labelset, batch_size=32)
    # define the model
    model = NN(hidden_sizes=argument.hidden_units, drop_p=argument.drop_p)
    # define loss function and optimizer
    criterion = nn.MSELoss()
    optimizer = optim.RMSprop(model.parameters(), lr=argument.learning_rate)
    # train the model
    training(
        model, criterion, optimizer,
        featureloader, labelloader,
        epochs=argument.epochs
    )
    # save the trained model
    save_model(
        model, argument.epochs, optimizer, scaler,
        filename=argument.save_name, save_dir=argument.save_dir
    )


# -*- coding: utf-8 -*-

import numpy
from cv2 import cv2

class Image:
    def __init__(self, image):
        self.image = cv2.imread(image, cv2.IMREAD_UNCHANGED)

    @property
    def width(self):
        return self.image.shape[1]

    @property
    def height(self):
        return self.image.shape[0]

class MatchImg(object):
    def __init__(self, source, template, threshod=0.95):
        """
        匹配一个图片，是否是另一个图片的局部图。source是大图，template是小图。即判断小图是否是大图的一部分。
        :param source:
        :param template:
        :param threshod: 匹配程度，值越大，匹配程度要求就越高，最好不要太小
        """
        self.source_img = source
        self.template_img = template
        self.threshod = threshod

    def set_confidence(self, confidence):
        self.threshod = confidence

    def match_template(self, method=cv2.TM_CCOEFF_NORMED):
        """
        返回小图左上角的点，在大图中的坐标。
        :param method:
        :return: list[tuple(x,y),...]
        """
        try:
            result = cv2.matchTemplate(self.source_img.image, self.template_img.image, method)
            locations = numpy.where(result >= self.threshod)
            res = list(zip(locations[1], locations[0]))  # 返回的是匹配到的template左上角那个坐标点在image中的位置，可能有多个值
            return res
        except cv2.error as e:
            print(e)

    def get_template_position(self):
        """
        获取小图在大图中，左上角和右下角的坐标
        :return: List[list[x,y,x,y],...]
        """
        res = self.match_template()
        new_pos = []
        for r in res:
            r = list(r)
            r.append(r[0] + self.template_img.width)
            r.append(r[1] + self.template_img.height)
            new_pos.append(r)
        return new_pos

    def get_img_center(self):
        """
        获取大图中，每个小图中心点所在的坐标
        :return:
        """
        pos = self.match_template()
        points = []
        for p in pos:
            x, y = p[0] + int(self.template_img.width / 2), p[1] + int(self.template_img.height / 2)
            points.append((x, y))
        return points

def load_image_file(path):
    path = path
    image = Image(path)
    return image

if __name__ == "__main__":
    img1 = load_image_file(r'E:\PyAutoProject\img\img.png')
    img2 = load_image_file(r'E:\PyAutoProject\img\img.png')

    process = MatchImg(img1, img2, 0.96)
    points = process.get_img_center()
    print(points)


# -*- coding: utf-8 -*-
"""
Created on Sun Feb 23 09:36:24 2020

@author: Richie Bao-caDesign设计(cadesign.cn).Chicago
"""
import rasterio,os,gc,h5py,datetime,scipy
from rasterio.windows import Window
from tqdm import tqdm
import numpy as np
from numba import jit, cuda,njit
import datetime
import numba
from joblib import Memory
import tables
from numba import jit,cuda,njit
from tempfile import mkdtemp
import os.path as path
import scipy.ndimage

#读取栅格，并查看属性值，返回需要的属性
def rasterProperties(rasterFp):
    raster=rasterio.open(rasterFp)
    print("type:",type(raster))
    print("transform:",raster.transform)
    print("[width,height]:", raster.width, raster.height)
    print("number of bands:",raster.count)
    print("bounds:",raster.bounds)
    print("driver:", raster.driver)
    print("no data values:",raster.nodatavals)
    print("_"*50)
    # print("meta:",raster.meta)
    # print("_"*50)
    # print("profile:",raster.profile)
    return raster.width, raster.height

# Yield successive n-sized
# chunks from l.
#递归分组列表数据
def divide_chunks(l,n):
    # looping till length l
    for i in range(0, len(l), n):
        yield l[i:i + n]
#建立用于rasterio库分批读取一个较大Raster数据的windows列表（如果内存溢出，而要处理较大的单独raster数据时）
#关于rasterio的window数据格式可以查看其官方网站
def rasterio_windows(totalWidth,totalHeight,subWidth,subHeight):
    w_n=list(divide_chunks(list(range(totalWidth)), subWidth))
    h_n=list(divide_chunks(list(range(totalHeight)), subHeight))
    wins=[Window(w[0],h[0],len(w),len(h)) for h in h_n for w in w_n]
    # print(wins)
    print("raster windows amount:",len(wins))
    return wins

# @njit(parallel=True)
# @cuda.jit(debug=True)
def ArrayCoordi(rasterArray):
    relativeCellCoords=np.indices(rasterArray.shape)
    relativeCellCoords2D=np.stack(relativeCellCoords,axis=2).reshape(-1,2)
    del relativeCellCoords
    # print(relativeCellCoords2D_O)
    return relativeCellCoords2D

#计算首尾坐标，用到三角函数
# @profile #内存监测所标识的函数
# @cuda.jit(debug=True)
# @njit(parallel=True)
#计算首尾坐标，用到三角函数
# @profile #内存监测所标识的函数
def equalCircleDivide(originalCoordiArray,radious,lineProfileAmount):
    # print(originalCoordiArray.shape)
    angle_s=360/lineProfileAmount
    angle_array=np.array([i*angle_s for i in range(lineProfileAmount)],dtype=np.float32)
    #.astype(np.float16)
    opposite=np.sin(np.radians(angle_array),dtype=np.float16)*radious
    opposite=opposite.astype(np.float32)
    # print(opposite.dtype)
    yCoordi=np.add(opposite,originalCoordiArray[:,1].reshape(-1,1),dtype=np.float32)
    del opposite
    # print(yCoordi.dtype)

    adjacent=np.cos(np.radians(angle_array),dtype=np.float16)*radious
    xCoordi=np.add(adjacent,originalCoordiArray[:,0].reshape(-1,1),dtype=np.float32)
    del adjacent,angle_array
    # print(xCoordi.dtype)

    CoordiArray=np.stack((xCoordi,yCoordi),axis=-1)
    # print(CoordiArray.dtype)
    # del xCoordi,yCoordi
    return CoordiArray

#计算障碍角度和高度,以及SVF值。参考文献：城区复杂下垫面天空视域因子参数化方法——以北京鸟巢周边地区为例
# @profile #内存监测所标识的函数
# @cuda.jit(debug=True)
# @njit(parallel=True)
def SVF(radious,equalDistance,ziList,j):
    segment=radious/equalDistance
    distanceList=np.array([i*segment for i in range(equalDistance+1)],dtype=np.float32)
    distanceList=distanceList[1:]

    ziListSub=ziList[:,:,1:]
    ziListOrigin=ziList[:,:,0]

    sinValues=np.true_divide(np.subtract(ziListSub,np.expand_dims(ziListOrigin, axis=2)),np.sqrt(np.add(np.power(distanceList,2),np.power(np.subtract(ziListSub,np.expand_dims(ziListOrigin, axis=2)),2))))

    sinMaxValue=np.amax(sinValues,axis=2)
    del sinValues
    SVFValue=1-np.true_divide(np.sum(sinMaxValue, axis=1),sinMaxValue.shape[-1])

    # SVFValueFn=os.path.join(saveFp,"SVFValue_%d"%(j))
    # np.save(SVFValueFn, SVFValue)
    # del SVFValue
    # print("SVF Value saved:%d"%i)
    # with open(os.path.join(saveFp,"SVFValue_filenames.txt"), "a") as text_file:
    #     text_file.write("%s\n"%SVFValueFn)

    return SVFValue.astype(np.float32)

#数组延直线截面提取单元值
# @profile #内存监测所标识的函数
# @cuda.jit(debug=True)
# @njit(parallel=True)
def lineProfile(zValue,originalPoint,endPoint):
    num=equalDistance+1

    x0=originalPoint[:,0].reshape(-1,1)
    x1=endPoint[:,:,0]
    # print(x0.dtype)
    # print(x1.dtype)
    x=np.linspace(x0, x1, num,dtype=np.int) #可以不用修改数组类型。出于内存优化考虑,会加快后续np.stack计算速度
    del x0,x1
    # fp[:]=np.linspace(x0, x1, num,dtype=np.int)[:]

    y0=originalPoint[:,1].reshape(-1,1)
    y1=endPoint[:,:,1]
    y=np.linspace(y0, y1, num,dtype=np.int)
    del y0,y1

    xStack=np.stack(x,axis=-1)
    # xStackSplit=np.array_split(xStack,saveN) #为减缓单次数组计算量，切分数组为saveN个单个数组，切分量越大，单个数组越小，计算所占用内存越小
    # del xStack #清空该变量，释放内存.关于numpy array占据内存的大小，查找确认
    # print("xStack finished!")

    yStack=np.stack(y,axis=-1)
    # yStackSplit=np.array_split(yStack,saveN)
    # del yStack
    del x,y

    zi = scipy.ndimage.map_coordinates(zValue,[xStack,yStack],cval=0,mode="nearest",order=0) #根据数组索引值，提取实际值
    del xStack,yStack

    # #记录每次存储的数据路径为.txt文件，最后通过该文件读取所有数据，组合与写入raster格式文件
    # with open(os.path.join(saveFp,"SVFValue_filenames.txt"), "a") as text_file:
    #     text_file.write("%s\n"%SVFValueFn)
    return zi

#read raster blocks 分批读取较大栅格数据，并计算
def SVF_Wins(rasterFp,dataSave_fp, windowsList):
    i=0
    for win in tqdm(windowsList):
        with rasterio.open(rasterFp,"r+") as src:
            src.nodata=-1
            w = src.read(1, window=win)
            # print("_"*50)
            # print(w.shape)
            profile=src.profile
            win_transform=src.window_transform(win)
        # print("_"*50)
        # print(w.shape)
        # print(ok)
        '''计算部分'''
        # coordiArray=equalCircleDivide(T,radious,lineProfileAmount)
        relativeCellCoords2D=ArrayCoordi(w)
        # print(relativeCellCoords2D)
        # print("+"*50)
        a_T = datetime.datetime.now()
        coordiArray=equalCircleDivide(relativeCellCoords2D,radious,lineProfileAmount)
        b_T= datetime.datetime.now()
        print("qualCircleDivide time span:", b_T-a_T)
        # print(coordiArray)
        gc.collect()

        # print(coordiArray.shape)
        # print(relativeCellCoords2D.shape)
        c_T=datetime.datetime.now()
        zi=lineProfile(w,relativeCellCoords2D,coordiArray)
        d_T=datetime.datetime.now()
        print("lineProfile time span:",d_T-c_T)
        gc.collect()
        del coordiArray

        e_T=datetime.datetime.now()
        SVFValue=SVF(radious,equalDistance,zi,i)
        f_T=datetime.datetime.now()
        print("SVF time span:",d_T-c_T)
        # print(SVFValue.shape)
        gc.collect()
        del zi

        profile.update(
            width=win.width,
            height=win.height,
            count=1,
            transform=win_transform,
            compress='lzw',
            dtype=rasterio.float32
            )
        # with rasterio.open(os.path.join(saveRasterByBlock_fp,"testingBlock.tif"), 'w', driver='GTiff' width=512, height=256, count=1,dtype=w.dtype,**profile) as dst:

        with rasterio.open(os.path.join(saveFp,"SVF3_%d.tif"%i), 'w', **profile) as dst:
            dst.write(SVFValue.reshape(w.shape), window=Window(0,0,win.width,win.height), indexes=1)

        del SVFValue

        g_T=datetime.datetime.now()
        print("total time span:",g_T-a_T)

        i+=1
        #在正式计算前，通过break，仅对部分数据编写与调整代码
        # if i ==1:
        #     break

if __name__=="__main__":
    tables.file._open_files.close_all() #关闭文件。该程序可忽略
    gc.collect() #释放缓存，IPython console 命令：%reset -f
    # rasterFp=r"D:\data\lidar\pdal_data\d.tif" #待计算SVF的栅格
    # rasterFp=r"D:\data\data_01_Chicago\pdal_data\chicago_loop_tile_0_0.tif"
    # rasterFp=r"F:\data_02_Chicago\parkNetwork\height_DSM-DTMInter.tif"
    rasterFp=r"F:\data_02_Chicago\ArcGisPro\parkNetwork\height_DSMDTMInter_3_a.tif"
    saveFp=r"F:\data_02_Chicago\parkNetwork\SVF"
    rasterFn=r"F:\data_02_Chicago\parkNetwork\SVF"

    # mem_filename = path.join(saveFp, 'newfile.dat')
    # if os.path.exists(mem_filename):
    #     os.remove(mem_filename)
    # else:
    #     print("Can not delete the file as it doesn't exists,built new one!")
    # fp = np.memmap(mem_filename, dtype='float16', mode='w+', shape=(51, 25000000, 36))
    # print(fp.filename == path.abspath(mem_filename))

    totalWidth,totalHeight=rasterProperties(rasterFp)
    #根据不同的栅格大小，自行调整subWidth和subHeight，满足内存要求
    subWidth=3000
    subHeight=3000
    rasterio_wins=rasterio_windows(totalWidth,totalHeight,subWidth,subHeight)
    # print(ok)

    # hdf5Fn=os.path.join(saveFp,"cacheSVF.hdf5")
    # if os.path.exists(hdf5Fn):
    #     os.remove(hdf5Fn)
    # else:
    #     print("Can not delete the file as it doesn't exists,built new one!")
    # f=h5py.File(hdf5Fn, "a")

    #参数配置
    rasterResolution=3 #计算栅格的分辨率
    radious=100*rasterResolution #扫描半径
    lineProfileAmount=8#扫描截面数量36
    equalDistance=30 #每条扫描线的等分数量  50
    # saveN=12 #配置每块计算时，numpy数组一次性计算量，数值越大，单次数组计算量越小，单次计算占用内存越小。numpy 数组大小在1，000，000量时，内存16G，测试达到最大，如果超过该量值，则需要增加该参数值，即降低单次数组的大小。

    SVF=SVF_Wins(rasterFp, saveFp,rasterio_wins[:100])


import spacy

nlp = spacy.load("en_core_web_lg")

def parse_questions(nlp, text):
    doc = nlp(text)

    return doc

class QuestionMatcherSpacyLgEnglishOnly:

    def match_questions(self, dfs):

        matches = {}

        transforms = []
        for df in dfs:
            # ignore language
            df["parsed"] = df.question.apply(lambda q: parse_questions(nlp, q))

            transforms.append(df["parsed"])

        for i in range(len(transforms)):
            for j in range(i + 1, len(transforms)):
                for ii in range(len(transforms[i])):
                    for jj in range(len(transforms[j])):
                        pairwise_similarity = transforms[i].iloc[ii].similarity(transforms[j].iloc[jj])

                        # if pairwise_similarity > 0.5:
                        matches[(i, ii, j, jj)] = pairwise_similarity

        return matches


from preprocessing.sms import SMSDataPreprocessingManager
from sklearn.metrics import confusion_matrix, classification_report, accuracy_score
from model.model import *
import argparse

if __name__ == "__main__":

    parser = argparse.ArgumentParser(description="set arguments. ")
    parser.add_argument("--data_path", default="./data/sample/spam.csv", type=str, help="Traing and Validation data")
    parser.add_argument("--model_path", default="./models/model/model.pkl", type=str, help="save model path")

    args = parser.parse_args()

    # 데이터 불러오기
    train_df = SMSDataPreprocessingManager.read_entire_data(path=args.data_path)

    # 데이터 preprocessing
    X_data_tfidf, X_train_tfidf, X_test_tfidf, y_train, y_test = SMSDataPreprocessingManager.data_preprocess(train_df)

    # training model
    lgbm_model = training_model(X_train_tfidf, X_test_tfidf, y_train, y_test)

    # save model
    save_model(lgbm_model, args.model_path)

    # evaluate model
    y_pred = evaluate_model(lgbm_model, X_test_tfidf)

    print(classification_report(y_test, y_pred, target_names=['ham', 'spam']))