import re
import unittest

import pytest
from cognite.client.data_classes import ContextualizationJob
from cognite.client.exceptions import ModelFailedException

from cognite.experimental import CogniteClient
from cognite.experimental.data_classes import PNIDDetectionList, PNIDDetectResults
from tests.utils import jsgz_load

COGNITE_CLIENT = CogniteClient()
PNIDAPI = COGNITE_CLIENT.pnid_parsing

@pytest.fixture
def mock_detect(rsps):
    response_body = {"jobId": 789, "status": "Queued"}
    rsps.add(
        rsps.POST,
        PNIDAPI._get_base_url_with_base_path() + PNIDAPI._RESOURCE_PATH + "/detect",
        status=200,
        json=response_body,
    )
    yield rsps

@pytest.fixture
def mock_extract_pattern(rsps):
    response_body = {"jobId": 456, "status": "Queued"}
    rsps.add(
        rsps.POST,
        PNIDAPI._get_base_url_with_base_path() + PNIDAPI._RESOURCE_PATH + "/extractpattern",
        status=200,
        json=response_body,
    )
    yield rsps

@pytest.fixture
def mock_convert(rsps):
    response_body = {"jobId": 345, "status": "Queued"}
    rsps.add(
        rsps.POST,
        PNIDAPI._get_base_url_with_base_path() + PNIDAPI._RESOURCE_PATH + "/convert",
        status=200,
        json=response_body,
    )
    yield rsps

@pytest.fixture
def mock_status_detect_ok(rsps):
    response_body = {
        "jobId": 123,
        "status": "Completed",
        "items": [
            {"text": "a", "boundingBox": {"xMin": 0, "xMax": 1, "yMin": 0, "yMax": 1}, "entities": [{"name": "a"}]}
        ],
        "fileId": 123432423,
        "fileExternalId": "123432423",
    }
    rsps.add(
        rsps.GET,
        re.compile(PNIDAPI._get_base_url_with_base_path() + PNIDAPI._RESOURCE_PATH + "/detect" + "/\\d+"),
        status=200,
        json=response_body,
    )
    yield rsps

@pytest.fixture
def mock_status_pattern_ok(rsps):
    response_body = {
        "jobId": 456,
        "status": "Completed",
        "items": [],
        "fileId": 123432423,
        "fileExternalId": "123432423",
    }
    rsps.add(
        rsps.GET,
        re.compile(PNIDAPI._get_base_url_with_base_path() + PNIDAPI._RESOURCE_PATH + "/extractpattern" + "/\\d+"),
        status=200,
        json=response_body,
    )
    yield rsps

@pytest.fixture
def mock_status_convert_ok(rsps):
    response_body = {
        "jobId": 123,
        "status": "Completed",
        "svgUrl": "svg.url.com",
        "pngUrl": "png.url.com",
        "fileId": 123432423,
        "fileExternalId": "123432423",
    }
    rsps.add(
        rsps.GET,
        re.compile(PNIDAPI._get_base_url_with_base_path() + PNIDAPI._RESOURCE_PATH + "/convert" + "/\\d+"),
        status=200,
        json=response_body,
    )
    yield rsps

@pytest.fixture
def mock_status_failed(rsps):
    response_body = {"jobId": 123, "status": "Failed", "errorMessage": "error message"}
    rsps.add(
        rsps.GET,
        re.compile(PNIDAPI._get_base_url_with_base_path() + PNIDAPI._RESOURCE_PATH + "/\\d+"),
        status=200,
        json=response_body,
    )
    yield rsps

class TestPNIDParsing:
    def test_detect_entities_str(self, mock_detect, mock_status_detect_ok):
        entities = ["a", "b"]
        file_id = 123432423
        job = PNIDAPI.detect(
            file_id=file_id, entities=entities, name_mapping={"a": "c"}, partial_match=False, min_tokens=3
        )
        assert isinstance(job, ContextualizationJob)
        assert "items" in job.result
        assert 789 == job.job_id
        assert "Completed" == job.status

        n_detect_calls = 0
        n_status_calls = 0
        for call in mock_detect.calls[1:]:
            if "detect" in call.request.url and call.request.method == "POST":
                n_detect_calls += 1
                assert {
                    "entities": entities,
                    "fileId": file_id,
                    "nameMapping": {"a": "c"},
                    "partialMatch": False,
                    "minTokens": 3,
                    "searchField": "name",
                } == jsgz_load(call.request.body)
            else:
                n_status_calls += 1
                assert "/789" in call.request.url
        assert 1 == n_detect_calls
        assert 1 == n_status_calls

    def test_detect_entities_dict(self, mock_detect, mock_status_detect_ok):
        entities = [{"name": "a"}, {"name": "b"}]
        file_id = 123432423
        job = PNIDAPI.detect(
            file_id=file_id, entities=entities, name_mapping={"a": "c"}, partial_match=False, min_tokens=3
        )
        assert isinstance(job, ContextualizationJob)
        assert "items" in job.result
        assert 789 == job.job_id
        assert "Completed" == job.status

        assert 1 == len(job.matches)
        assert [{"name": "a"}] == job.matches[0].entities
        assert "a" == job.matches[0].text

        n_detect_calls = 0
        n_status_calls = 0
        for call in mock_detect.calls:
            if "detect" in call.request.url and call.request.method == "POST":
                n_detect_calls += 1
                assert {
                    "entities": [{"name": "a"}, {"name": "b"}],
                    "fileId": file_id,
                    "nameMapping": {"a": "c"},
                    "partialMatch": False,
                    "minTokens": 3,
                    "searchField": "name",
                } == jsgz_load(call.request.body)
            else:
                n_status_calls += 1
                assert "/789" in call.request.url
        assert 1 == n_detect_calls
        assert 1 == n_status_calls

    def test_extract_pattern(self, mock_extract_pattern, mock_status_pattern_ok):
        patterns = ["ab{1,2}"]
        file_id = 123432423
        job = PNIDAPI.extract_pattern(file_id=file_id, patterns=patterns)
        assert isinstance(job, ContextualizationJob)
        assert "Queued" == job.status
        assert "items" in job.result
        assert "Completed" == job.status
        assert 456 == job.job_id

        n_extract_pattern_calls = 0
        n_status_calls = 0
        for call in mock_extract_pattern.calls:
            if "extractpattern" in call.request.url and call.request.method == "POST":
                n_extract_pattern_calls += 1
                assert {"patterns": patterns, "fileId": file_id} == jsgz_load(call.request.body)
            else:
                n_status_calls += 1
                assert "/456" in call.request.url
        assert 1 == n_extract_pattern_calls
        assert 1 == n_status_calls

    def test_convert(self, mock_convert, mock_status_convert_ok):
        items = [
            {
                "text": "21-PT-1019",
                "boundingBox": {
                    "xMax": 0.5895183277794608,
                    "xMin": 0.573159648591336,
                    "yMax": 0.3737254901960784,
                    "yMin": 0.3611764705882352,
                },
            }
        ]
        file_id = 123432423
        job = PNIDAPI.convert(file_id=file_id, items=items, grayscale=True)
        assert isinstance(job, ContextualizationJob)
        assert "Queued" == job.status
        assert "svgUrl" in job.result
        assert "Completed" == job.status
        assert 345 == job.job_id

        n_convert_calls = 0
        n_status_calls = 0
        for call in mock_convert.calls:
            if "convert" in call.request.url and call.request.method == "POST":
                n_convert_calls += 1
                assert {
                    "fileId": file_id,
                    "items": items,
                    "grayscale": True,
                } == jsgz_load(call.request.body)
            else:
                n_status_calls += 1
                assert "/345" in call.request.url
        assert 1 == n_convert_calls
        assert 1 == n_status_calls

    def test_file_external_id(self, mock_detect, mock_status_detect_ok):
        entities = [{"name": "a"}, {"name": "b"}]
        file_external_id = "123432423"
        job = PNIDAPI.detect(
            file_external_id=file_external_id,
            entities=entities,
            name_mapping={"a": "c"},
            partial_match=False,
            min_tokens=3,
        )
        assert isinstance(job, PNIDDetectResults)
        assert isinstance(job._repr_html_(), str)
        assert "fileId" in job.result
        assert "fileExternalId" in job.result
        assert file_external_id == job.file_external_id
        assert isinstance(job.matches, PNIDDetectionList)
        assert "Completed" == job.status


# coding: utf-8

# ## Build a simple trading strategy

# In[3]:

import pandas as pd
import matplotlib.pyplot as plt
get_ipython().magic('matplotlib inline')

# ### 1. Munging the stock data and add two columns - MA10 and MA50

# In[4]:

import pandas as pd
import matplotlib.pyplot as plt
get_ipython().magic('matplotlib inline')

#import MS's stock data, add two columns - MA10 and MA50
#use dropna to remove any "Not a Number" data
ms = pd.DataFrame.from_csv('../data/microsoft.csv')
ms['MA10'] = ms['Close'].rolling(10).mean()
ms['MA50'] = ms['Close'].rolling(50).mean()
ms = ms.dropna()
ms.head()

# ### 2. Add "Shares" column to make decisions base on the strategy

# In[7]:

#Add a new column "Shares", if MA10>MA50, denote as 1 (long one share of stock), otherwise, denote as 0 (do nothing)

ms['Shares'] = [1 if ms.loc[ei, 'MA10']>ms.loc[ei, 'MA50'] else 0 for ei in ms.index]

# In[8]:

#Add a new column "Profit" using List Comprehension, for any rows in fb, if Shares=1, the profit is calculated as the close price of
#tomorrow - the close price of today. Otherwise the profit is 0.

#Plot a graph to show the Profit/Loss

ms['Close1'] = ms['Close'].shift(-1)
ms['Profit'] = [ms.loc[ei, 'Close1'] - ms.loc[ei, 'Close'] if ms.loc[ei, 'Shares']==1 else 0 for ei in ms.index]
ms['Profit'].plot()
plt.axhline(y=0, color='red')

# ### 3. Use .cumsum() to display our model's performance if we follow the strategy

# In[10]:

#Use .cumsum() to calculate the accumulated wealth over the period

ms['wealth'] = ms['Profit'].cumsum()
ms.tail()

# In[11]:

#plot the wealth to show the growth of profit over the period

ms['wealth'].plot()
plt.title('Total money you win is {}'.format(ms.loc[ms.index[-2], 'wealth']))

# ## You can create your own simple trading strategy by copying the codes above and modify the codes accordingly using the data of Microsoft (microsoft.csv).


from datetime import datetime
from django.db.models import FilteredRelation, Q, F
from .utils import MemberHistoryLine
from .models import MemberTransaction, Account, AccountType, Aid, Meeting

def get_accounts(*, used_for):
    if used_for.upper() == "MEMBER":
        used_for_list = [Account.MEMBER, Account.BOTH]
    elif used_for.upper() == "ORG":
        used_for_list = [Account.ORG, Account.BOTH]
    else:
        raise ValueError("Wrong argument value. Only Member and Org are allowed!")
    return [
        account.title
        for account in (
            Account.objects.filter(used_for__in=used_for_list).only("title").all()
        )
    ]

class MemberFinance(object):
    def __init__(self, email):
        qs = (
            MemberTransaction.objects.select_related(
                "meeting",
                "member",
                "member__user",
                "account",
            )
            .filter(member__user__email=email)
            .only(
                "meeting__date",
                "account__title",
                "amount",
                "member__id",
                "member__user",
            )
            .all()
        )

        self._member_accounts = [
            AccountType.ASSISTANCE,
            AccountType.SAVINGS,
            AccountType.SCHOLAR_SAVINGS,
            AccountType.PROJECT,
            AccountType.SANCTION,
        ]

        # Logged In Member
        self._member = qs[0].member
        # Transactions
        self._transactions = qs
        # Member's aids
        self._aids = self._member.aids.all() if qs else []

        self._date_format = "%m/%m/%Y"

        # History Table
        history = list(
            Meeting.objects.annotate(
                A=FilteredRelation(
                    "absences", condition=Q(absences__member=self._member.id)
                ),
                S=FilteredRelation(
                    "sanctions", condition=Q(sanctions__member=self._member.id)
                ),
            )
            .filter(date__lte=datetime.today())
            .values(
                "date",
                "A__absence_reason",
                "A__justified",
                "A__sanctioned",
                "S__sanction_reason",
                "S__amount",
                "S__sanction_type",
                "S__title",
                "S__executed_or_paid",
            )
            .all()
        )

        self._history = {}

        for line in history:
            date = line["date"].strftime(self._date_format)
            values = {
                key.split("__")[1]: (value or None)
                for key, value in line.items()
                if key != "date"
            }
            self._history[date] = values

    @property
    def account_balances(self):
        balances = {title: 0.0 for title in self._member_accounts}
        for transaction in self._transactions:
            balances[transaction.account.title] += float(transaction.amount)
        balances["absences"] = len(
            [1 for _, line in self._history.items() if line["absence_reason"]]
        )
        balances["assistance_aid"] = len(
            [1 for aid in self._aids if aid.aid_type == Aid.ASSISTANCE]
        )
        balances["happiness_aid"] = len(
            [1 for aid in self._aids if aid.aid_type == Aid.HAPPINESS]
        )
        return balances

    @property
    def history(self):
        date_format = "%m/%m/%Y"
        base_dict = {title: 0 for title in self._member_accounts}
        base_dict.update(
            {
                "absence": False,
                "absence_reason": "",
                "justified": "",
                "sanctioned": "",
                "sanction_reason": "",
                "amount": "",
                "sanction_type": "",
                "title": "",
                "executed_or_paid": "",
            }
        )
        for transaction in self._transactions:
            date = transaction.meeting.date.strftime(self._date_format)
            if date not in self._history:
                # Normally, this must never happen
                self._history[date] = base_dict.copy()
            self._history[date][transaction.account.title] = float(transaction.amount)

        return (
            self._history.items()
        )  # [MemberHistoryLine(date=date, **values) for date, values in his.items()]

"""
from ade4.selectors import MemberFinance as MF
m = MF(email='johnsmith@f4.com')
q = m.queryset
m.account_balances

for date, vals in d.items():
    print(date)
    for c, v in vals.items():
        print('\t',c,': ',v)

me = Meeting.objects.filter(date__lte=datetime(2022, 6, 21)).all().values('date', 'absences').extra(where=[f"member_id={member.id} OR member_id IS NULL"]).all()
me = Meeting.objects.filter(date__lte=datetime.today()).all().values('date', 'absences').extra(where=[f"member_id={member.id} OR member_id IS NULL"]).all()

SELECT "ade4_meeting"."date", "ade4_absence"."reason", "ade4_absence"."member_id"
FROM "ade4_meeting" LEFT OUTER JOIN "ade4_absence" ON
    ("ade4_meeting"."id" = "ade4_absence"."meeting_id")
WHERE ("ade4_meeting"."date" < 2022-06-21 AND (member_id=1 OR member_id IS NULL))
ORDER BY "ade4_meeting"."date" DESC

me = Meeting.objects.annotate(
pizzas_vegetarian=FilteredRelation(
'absences__member_id', condition=Q(absences__member_id=member.id),)
).filter(date__lte=datetime.today()).all().values('date', 'absences').all()

me = Meeting.objects.annotate(
    A=FilteredRelation('absences', condition=Q(absences__member=member.id))
).filter(date__lte=datetime.today()).all().values('date', 'A__reason').all()

me = Meeting.objects.annotate(
    A=FilteredRelation('absences', condition=Q(absences__member=member.id)),
    S=FilteredRelation('sanctions', condition=Q(sanctions__member=member.id))
).filter(date__lte=datetime.today()).all().values('date', 'A__reason', 'S__amount').all()

me = Meeting.objects.annotate(
    A=FilteredRelation('absences', condition=Q(absences__member=member.id)),
    S=FilteredRelation('sanctions', condition=Q(sanctions__member=member.id))
).filter(date__lte=datetime.today()).values('date', 'A__reason', 'S__amount').annotate(
    reason=F('A__reason'), amount=F('S__amount')
)

"""


import numpy as np
from PIL import Image
from keras.utils import Sequence
from skimage import transform as sktr
import random

class DataGenerator(Sequence):
    '''
    This is a data generator which output a batch of (X, y) data to feed into keras model.fit_generator method.
    Following inputs are required.

        dic_Im_names: defines the unique image names for the training/validation/test sets
                        with 'train', 'val', and 'test' as the keys, respectively.
        filepath: path of the resized image of size = img_sz. Default value is 'preprocessed/',
        datatype: indicates the group of the data and takes values of 'train', 'val' or 'test'. Default value is 'train'
        batch_size: batch size. Default set to 16.
        image_sz: size of the input image and mask.Default set to 512
        Im_channels: number of channels of the input image. Default set to 3.
        msk_channels: number of channels of the mask. Default set to 1.
        shuffle: shuffle the data for each individual epoch. Default is True.
        aug_on: implement random augmentation (rotation and horizontal flip) for image and mask. default is False.
    '''
    def __init__(self, dic_Im_names,
                 filepath='preprocessed/',
                 datatype='train',
                 batch_size=16,
                 image_sz=512,
                 Im_channels=3,
                 msk_channels=1,
                 shuffle=True,
                 aug_on=False):
        '''
        init method taking inputs and defines instance varables.
        '''
        self.dic_Im_names = dic_Im_names
        # Select a proper pool (train/val/test) of car names based on the conditions given.
        if datatype == 'train':
            self.Im_names = dic_Im_names['train']
        elif datatype == 'val':
            self.Im_names = dic_Im_names['val']
        elif datatype == 'test':
            self.Im_names = dic_Im_names['test']
        else:
            raise ValueError('Invalid datatype is provided!')

        self.filepath = filepath
        self.batch_size = batch_size
        self.image_sz = image_sz
        self.Im_channels = Im_channels
        self.msk_channels = msk_channels
        self.shuffle = shuffle
        self.aug_on = aug_on
        self.on_epoch_end()

    def __getitem__(self, index):
        '''
        Main function to generate a batch of (X, y) training samples.
        Index defines the current batch index in a epoch.
        '''
        # batch_indexs is a list containing all the indexs of instances in a epoch.
        batch_indexs = self.idxs[index*self.batch_size:(index+1)*self.batch_size]
        # batch_Im_names is a list containing all the image names in the selected batch.
        batch_Im_names = [self.Im_names[i] for i in batch_indexs]
        # X and y are empty array reserved to fed in the instance values (X[i], y[i]).
        X = np.empty((self.batch_size, self.image_sz, self.image_sz, self.Im_channels))
        y = np.empty((self.batch_size, self.image_sz, self.image_sz))

        for i, Im_name in enumerate(batch_Im_names):
            X[i] = np.array(Image.open(self.filepath + 'train/' + Im_name))
            y[i] = np.array(Image.open(self.filepath + 'train_masks/' + Im_name.split('.')[0] + '_mask.gif'))
            if self.aug_on:
                X[i], y[i] = self.random_rotation_flip(X[i], y[i])
        X = X/255.0 # Normalization
        # Original y.shape: (self.batch_size, np.shape(y)[1], np.shape(y)[2]). Add one channel dimention to fit in the U-net.
        y = y.reshape(self.batch_size, np.shape(y)[1], np.shape(y)[2], self.msk_channels)
        return X, y

    def __len__(self):
        '''
        Return the total number of batches in one epoch.
        '''
        return int(np.floor(len(self.Im_names)/self.batch_size))

    def on_epoch_end(self):
        '''
        Assign indexs to each instance in a epoch.
        Randomly shuffle the index on epoch end, if self.shuffle == True.
        '''
        self.idxs = np.arange(len(self.Im_names))
        if self.shuffle:
            np.random.shuffle(self.idxs)

    def random_rotation_flip(self, image, msk):
        '''
        Rotate the image with a random degree smaller than 25.
        '''
        random_degree = random.uniform(-25, 25)
        Image_agmt = sktr.rotate(image, random_degree, mode='symmetric')
        msk_agmt =sktr.rotate(msk, random_degree, mode='symmetric')
        # 50% chance of flipping the image.
        a = random.uniform(0, 1)
        if a > 0.5:
            return Image_agmt[:, ::-1, :], msk_agmt[:, ::-1]
        else:
            return Image_agmt, msk_agmt


from pprint import pprint

from twisted.web.server import Site
from twisted.web import server
from twisted.web.resource import Resource
from twisted.internet import reactor, defer, threads
from twisted.web.static import File
from string import Template

import cPickle
import time
import os
import socket

def wait(seconds, result=None):
    d = defer.Deferred()
    reactor.callLater(seconds, d.callback, result)
    return d

class Load(Resource):

    def final(self, message, request):
        request.finish()

    #@defer.inlineCallbacks

    def crap(self, request, name, num):
        x = ''
        pprint(str(num))
        for n in range(num):
            with open('c:\\' + name + '_' + str(n) + '.vbo', 'rb') as f:
                x = cPickle.load(f)
                request.write(x)
                pprint("sending cloud #" + str(n))
                #yield wait(0.0)

    def render_GET(self, request):
        isLeaf = True
        cloudName = request.received_headers['cloudName']

        # check that file cloudName exists
        # nyi

        # find out how many vbo's there are
        with open('c:\\' + cloudName) as f:
            numVBO = int(f.readline())

        if numVBO > 0:
            # extract filename
            fileName = cloudName[0:cloudName.rfind('.')]
            d = threads.deferToThread(self.crap, request, fileName, numVBO)
            d.addCallback(lambda _: request.finish())

            return server.NOT_DONE_YET

        #x += cPickle.load(f)
        #print(type(x))

        #equest.finish()
        #return server.NOT_DONE_YET
        #return x

        #return -1;

class UploadVBO(Resource):
    def render_GET(self, request):
        return ''

    def render_POST(self, request):
        cloudName = request.received_headers['x-cloudName']
        cloudSeq = request.received_headers['x-cloudSequence']
        cloudArrayByteLength = request.received_headers['X-cloudArrayByteLength']
        cloudNumPoints = request.received_headers['X-cloudNumPoints']
        pprint(cloudName + ' ' + cloudSeq + ' ' + cloudArrayByteLength + ' ' + cloudNumPoints)

        #pprint(request.__dict__)
        #pprint(request.content.read())

        with open('c:\\' + cloudName + '_' + cloudSeq + '.vbo', 'wb') as f:
            d = request.content.read()
            cPickle.dump(d, f, cPickle.HIGHEST_PROTOCOL)

        return ''

## this completes the upload process by writing .pointcloud file
## right now there's on TotalSequence but more will come
class CreatePointCloudFile(Resource):
    def render_GET(self, request):
        return ''

    def render_POST(self, request):
        cloudName = request.received_headers['x-cloudName']
        cloudTotalSeq = request.received_headers['X-cloudTotalSequence']
        pprint("writing " + cloudName + ".pointcloud, # of VBOs = " + cloudTotalSeq)

        # write .pointcloud file
        with open('c:\\' + cloudName + '.pointcloud', 'w') as f:
            f.write(str(cloudTotalSeq))

        return 1

class QueryPickPoint(Resource):
    def render_GET(self, request):
       isLeaf = True
       RayOriginX = request.received_headers['rayoriginx']
       RayOriginY = request.received_headers['rayoriginy']
       RayOriginZ = request.received_headers['rayoriginz']
       RayDirX = request.received_headers['raydirx']
       RayDirY = request.received_headers['raydiry']
       RayDirZ = request.received_headers['raydirz']

       HOST = '127.0.0.1'
       PORT = 9000
       s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
       s.connect((HOST, PORT))
       #s.sendall('1,0.849,-2.379,0.445,0.019,0.569,-0.822')

       strQuery = '1,%s,%s,%s,%s,%s,%s' % (RayOriginX,RayOriginY,RayOriginZ,RayDirX,RayDirY,RayDirZ)
       pprint(strQuery)
       s.sendall(strQuery)

       data = s.recv(1024)
       s.close()

       responseData = repr(data).replace('\'','') # need to get rid of single quotes returned from octree backend
       pprint(responseData)
       request.write(responseData)

       request.finish()
       return server.NOT_DONE_YET

class PointCloudListing(Resource):
    isLeaf = True

    def getChild(self, name, request):
        if name == '':
            return self
        return resource.Resource.geChild(self, name, request)

    def render_GET(self, request):
        allFiles = os.listdir("c:\\")
        cloudFiles = [f for f in allFiles if f.find(".pointcloud") > 0]

        s = []
        for f in cloudFiles:
            s.append("" + f + "
")

        template = open("index.template", "r")
        data = template.read().replace("<--pointclouds-->", '\n'.join(s))

        return data

class PointCloud(Resource):
    isLeaf = True

    def render_GET(self, request):
        template = open("view.html", "r")
        cloudName = request.postpath[0]

        t = Template(template.read())
        html = ''

        if cloudName == '':
            html = t.substitute(entry='return;')
        else:
            html = t.substitute(entry='startServer("' + cloudName + '");')

        return html

#root = Resource()
root = File("C:\\Users\\mer\\Documents\\GitHub\\bobcat")

root.putChild('upload', UploadVBO())
root.putChild('load', Load())
root.putChild('findpickpoint', QueryPickPoint())
root.putChild('finalize', CreatePointCloudFile())
root.putChild('contentlist', PointCloudListing())
root.putChild('pointcloud', PointCloud())

factory = Site(root)
reactor.listenTCP(8080, factory)
reactor.run()


#!/usr/bin/env python
# coding: utf-8

# # GPU Light gradient boosting trained on TF-IDF reduced 50 dimensions
#
# 1. Same emotion dataset from [NLP-dataset](https://github.com/huseinzol05/NLP-Dataset)
# 2. Same splitting 80% training, 20% testing, may vary depends on randomness
# 3. Same regex substitution '[^\"\'A-Za-z0-9 ]+'
#
# ## Example
#
# Based on Term-frequency Inverse document frequency
#
# After that we apply SVD to reduce the dimensions, n_components = 50

# In[8]:

import re
import time

import numpy as np
import sklearn.datasets
from sklearn import metrics, pipeline
from sklearn.cross_validation import train_test_split
from sklearn.decomposition import TruncatedSVD
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.model_selection import StratifiedKFold

import lightgbm as lgb

# In[2]:

def clearstring(string):
    string = re.sub("[^\"'A-Za-z0-9 ]+", "", string)
    string = string.split(" ")
    string = filter(None, string)
    string = [y.strip() for y in string]
    string = " ".join(string)
    return string

# because of sklean.datasets read a document as a single element
# so we want to split based on new line
def separate_dataset(trainset):
    datastring = []
    datatarget = []
    for i in range(len(trainset.data)):
        data_ = trainset.data[i].split("\n")
        # python3, if python2, just remove list()
        data_ = list(filter(None, data_))
        for n in range(len(data_)):
            data_[n] = clearstring(data_[n])
        datastring += data_
        for n in range(len(data_)):
            datatarget.append(trainset.target[i])
    return datastring, datatarget

# In[3]:

trainset_data = sklearn.datasets.load_files(container_path="data", encoding="UTF-8")
trainset_data.data, trainset_data.target = separate_dataset(trainset_data)

# In[4]:

train_X, test_X, train_Y, test_Y = train_test_split(
    trainset_data.data, trainset_data.target, test_size=0.2
)

# In[5]:

decompose = pipeline.Pipeline(
    [("count", TfidfVectorizer()), ("svd", TruncatedSVD(n_components=50))]
).fit(trainset_data.data)

# In[6]:

params_lgb = {
    "max_depth": 27,
    "learning_rate": 0.03,
    "verbose": 50,
    "early_stopping_round": 200,
    "metric": "multi_logloss",
    "objective": "multiclass",
    "num_classes": len(trainset_data.target_names),
    "device": "gpu",
    "gpu_platform_id": 0,
    "gpu_device_id": 0,
}

# In[10]:

train_X = decompose.transform(train_X)
test_X = decompose.transform(test_X)

# In[11]:

d_train = lgb.Dataset(train_X, train_Y)
d_valid = lgb.Dataset(test_X, test_Y)
watchlist = [d_train, d_valid]
t = time.time()
clf = lgb.train(params_lgb, d_train, 100000, watchlist, early_stopping_rounds=200, verbose_eval=100)
print(round(time.time() - t, 3), "Seconds to train lgb")

# In[12]:

print(
    metrics.classification_report(
        test_Y, np.argmax(clf.predict(test_X), axis=1), target_names=trainset_data.target_names
    )
)

# In[13]:

clf.save_model("lgb-tfidf-svd50.model")

# In[ ]:


from tensorflow import python as tf
import numpy as np
import os
import matplotlib.pyplot as plt
from random import shuffle
from label_vehicles import label_vehicle
import pandas as pd
import glob
import cv2
from preprocessing import preprocessing
from rotate import rotate
from rotate_horizontal import rotate_horizontal_and_resize
from preparation import create_mask_of_tag
from load_model import pretty_cm, transform_image, evaluation_indices, report

def main(path):
    print("Working on it, please have patience")
    src_dir = 'data2/' + path
    df = pd.read_csv('data2/val_csv.csv', sep=',')
    br = 0

    for filename in glob.glob(os.path.join(src_dir, '*.jpg')):
        im = cv2.imread(filename)
        name = filename.replace(src_dir, '')
        img_name = name.replace('.jpg', '')
        img_name = img_name.replace('\\', '')
        preprocessed = preprocessing(im, img_name, df)
        create_mask_of_tag(preprocessed, img_name, df)
        br += 1
        print(br)
    src_dir = 'data2/masked_val/small vehicle'
    dst_dir = 'data2/masked_rotated_h_val/'
    for filename in glob.glob(os.path.join(src_dir, '*.jpg')):
        im = cv2.imread(filename)
        name = filename.replace(src_dir, '')
        tag_name = name.replace('.jpg', '')
        tag_name = tag_name.replace('\\', '')
        rot = rotate(im, tag_name, df)
        rot_h = rotate_horizontal_and_resize(rot, (30, 75))
        tag_id_new = int(tag_name)
        selection = df[df.tag_id == tag_id_new]

        for idx, row in selection.iterrows():
            if str(row.general_class) == "small vehicle":
                add_dst = 'small vehicle/'
                if str(row.sub_class) == "sedan":
                    cv2.imwrite(dst_dir + add_dst + 'sedan/' + str(row.tag_id) + '.jpg', rot_h)
                elif str(row.sub_class) == "hatchback":
                    cv2.imwrite(dst_dir + add_dst + 'hatchback/' + str(row.tag_id) + '.jpg', rot_h)
                elif str(row.sub_class) == "minivan":
                    cv2.imwrite(dst_dir + add_dst + 'minivan/' + str(row.tag_id) + '.jpg', rot_h)
                elif str(row.sub_class) == "van":
                    cv2.imwrite(dst_dir + add_dst + 'van/' + str(row.tag_id) + '.jpg', rot_h)
                elif str(row.sub_class) == "jeep":
                    cv2.imwrite(dst_dir + add_dst + 'jeep/' + str(row.tag_id) + '.jpg', rot_h)
                elif str(row.sub_class) == "pickup":
                    cv2.imwrite(dst_dir + add_dst + 'pickup/' + str(row.tag_id) + '.jpg', rot_h)
        br += 1
        print(br)

    # preparing data for predictions
    size = (30, 75)
    X_eval = list()
    y_eval = list()
    X_tag = list()

    # hatchback
    files = os.listdir('data2/masked_rotated_h_val/small vehicle/hatchback/')
    files.sort()

    for i in range(0, len(files)):
        X_eval.append(transform_image('data2/masked_rotated_h_val/small vehicle/hatchback/' + files[i], size))
        y_eval.append(0)
        tag = files[i].replace('.jpg', '')
        X_tag.append(int(tag))
    # jeep
    files = os.listdir('data2/masked_rotated_h_val/small vehicle/jeep/')
    files.sort()

    for i in range(0, len(files)):
        X_eval.append(transform_image('data2/masked_rotated_h_val/small vehicle/jeep/' + files[i], size))
        y_eval.append(1)
        tag = files[i].replace('.jpg', '')
        X_tag.append(int(tag))
    # minivan
    files = os.listdir('data2/masked_rotated_h_val/small vehicle/minivan/')
    files.sort()

    for i in range(0, len(files)):
        X_eval.append(transform_image('data2/masked_rotated_h_val/small vehicle/minivan/' + files[i], size))
        y_eval.append(2)
        tag = files[i].replace('.jpg', '')
        X_tag.append(int(tag))
    # pickup
    files = os.listdir('data2/masked_rotated_h_val/small vehicle/pickup/')
    files.sort()

    for i in range(0, len(files)):
        X_eval.append(transform_image('data2/masked_rotated_h_val/small vehicle/pickup/' + files[i], size))
        y_eval.append(3)
        tag = files[i].replace('.jpg', '')
        X_tag.append(int(tag))
    # sedan
    files = os.listdir('data2/masked_rotated_h_val/small vehicle/sedan/')
    files.sort()

    for i in range(0, len(files)):
        X_eval.append(transform_image('data2/masked_rotated_h_val/small vehicle/sedan/' + files[i], size))
        y_eval.append(4)
        tag = files[i].replace('.jpg', '')
        X_tag.append(int(tag))
    # van
    files = os.listdir('data2/masked_rotated_h_val/small vehicle/van/')
    files.sort()

    for i in range(0, len(files)):
        X_eval.append(transform_image('data2/masked_rotated_h_val/small vehicle/van/' + files[i], size))
        y_eval.append(5)
        tag = files[i].replace('.jpg', '')
        X_tag.append(int(tag))
    # stacking the arrays
    X_eval = np.vstack(X_eval)

    labels_index = {0: "hatchback", 1: "jeep", 2: "minivan", 3: "pickup", 4: "sedan", 5: "van"}

    # load the model
    cnn_classifier = tf.keras.models.load_model('data2/vehicle_classification_model_dropout.h5')

    cnn_pred = cnn_classifier.predict_classes(X_eval, batch_size=32)

    pretty_cm(cnn_pred, y_eval, labels_index)
    correctly_classified_indices, misclassified_indices = evaluation_indices(cnn_pred, y_eval)

    plt.figure(figsize=(36, 6))
    shuffle(correctly_classified_indices)
    plt.show()

    for plot_index, good_index in enumerate(correctly_classified_indices[0:5]):
        plt.subplot(1, 5, plot_index + 1)
        plt.imshow(X_eval[good_index])
        plt.title('Predicted: {}, Actual: {}'.format(labels_index[cnn_pred[good_index]],
                                                     labels_index[y_eval[good_index]]), fontsize=5)
    plt.show()

    print(X_tag)
    print(list(cnn_pred))
    print(y_eval)
    predicted_dict = dict(zip(X_tag, list(cnn_pred)))
    actual_dict = dict(zip(X_tag, y_eval))

    print(len(X_tag))
    print(len(list(cnn_pred)))
    print(len(y_eval))

    df = pd.read_csv('data2/val_csv.csv', sep=',')
    path = 'data2/val'
    src_dir = path
    dst_dir = 'data2/labeled_predicted_val'
    for filename in glob.glob(os.path.join(src_dir, '*.jpg')):
        im = cv2.imread(filename)
        name = filename.replace(src_dir, '')
        img_name = name.replace('.jpg', '')
        img_name = img_name.replace('\\', '')
        labeled_img_p = label_vehicle(im.copy(), img_name, df, predicted_dict)
        cv2.imwrite(dst_dir + name, labeled_img_p)
        labeled_img_a = label_vehicle(im.copy(), img_name, df, actual_dict)
        cv2.imwrite('data2/labeled_actual_val' + name, labeled_img_a)

    report(y_eval, list(cnn_pred), labels_index)
    print("Done")

if __name__ == "__main__":
    main('val')


from django import forms
from .models import Order

class OrderCreateForm(forms.ModelForm):
	DIVISION_CHOICES = (
		('North', 'North'),
		('South', 'South'),
		('Central', 'Central'),
	)

	DISCRICT_CHOICES = (
		('Delhi', 'Delhi'),
		('Bangalore', 'Bangalore'),
		('Mumbai', 'Mumbai'),
	)

	PAYMENT_METHOD_CHOICES = (
		('Visa', 'Visa'),
		('MasterCard','MasterCard')
	)

	division = forms.ChoiceField(choices=DIVISION_CHOICES)
	district =  forms.ChoiceField(choices=DISCRICT_CHOICES)
	payment_method = forms.ChoiceField(choices=PAYMENT_METHOD_CHOICES, widget=forms.RadioSelect())

	class Meta:
		model = Order
		fields = ['name', 'email', 'phone', 'address', 'division', 'district', 'zip_code', 'payment_method', 'account_no', 'transaction_id']


__all__ = [
    # Utilities
    'CapabilityTemplate',
    'add',
    # Base services
    'bird',
    'dhclient',
    'dhcpd',
    'firstboot',
    'gateway',
    'jail',
    'linux',
    'named',
    'openssh',
    'pf',
    'pflate',
    'pflog',
    'resolvconf',
    'sshd',
    'trust',
    'zfs'
]

import frieze
import importlib
import io
import os
import sys
import tarfile

import mako.template
import pkg_resources as pkg

from openarc import staticproperty

class CapabilityTemplate(object):

    # Resource expectations.
    cores  = None
    memory = None

    # Mounts to be fed into the capability's container
    mounts = []

    # Ports to be redirected from outside
    ports = []

    # Config files that need to be set in the container
    config = []

    # The site in which we would like this capability to be run. None for any site.
    affinity = None

    # Application can exist inside a jail
    jailable = True

    # Package to install. If set to None, nothing is installed. If set and
    # custom_pkg=False, a naive call to pkg install <package> is issued. If set
    # and custom_pkg=True, pkg add <path/to/txz> is issued.
    package = None

    # Knobs used by job system to adjust the operation of this capability. Knobs
    # not in this list cannot be set on the capability
    knobs = []

    # Resource directory (for internal bookkeeping)
    resource_path = 'frieze.capability.resources'

    def __init__(self):
        self.set_knobs = {}
        self.cmd_count = 0

    def next_cmd_cfg_name(self):
        cfg_name = f'{self.name}_cmd_{self.cmd_count}'
        self.cmd_count += 1
        return cfg_name

    def generate_cfg_files(self, host, __exclude__=[]):
        rv = {}
        __exclude__.append('__init__.py')
        __exclude__.append('__pycache__')
        try:
            cmd_cnt = 0
            cap_cfgs = pkg.resource_listdir(self.resource_path, self.name)
            for cfg in [cfg for cfg in cap_cfgs if cfg not in __exclude__]:
                cfg_raw = pkg.resource_string(f'{self.resource_path}.{self.name}', cfg)
                if cfg_raw[0:2]=='#!':
                    cfg_name = self.next_cmd_cfg_name()
                else:
                    cfg_1l = bytearray()
                    for b in cfg_raw:
                        if b=='\n'.encode()[0]:
                            break
                        cfg_1l.append(b)
                    cfg_name = bytes(cfg_1l[2:]).decode().split(' ')[0]

                try:
                    rv[cfg_name] = mako.template.Template(cfg_raw.decode()).render(host=host)
                except UnicodeDecodeError:
                    rv[cfg_name] = cfg_raw
                except Exception as e:
                    print(f"Error processing template [{cfg_name}] for capability [{self.name}]")
                    raise
        except FileNotFoundError:
            pass
        return rv

    @staticproperty
    def name(cls):
        return cls.__name__

    def setknob(self, knob, value):
        knobs = getattr(self, 'set_knobs', {})
        if not knobs:
            setattr(self, 'set_knobs', knobs)
        if knob in self.knobs:
            self.set_knobs[knob] = value
        else:
            raise OAError("[%s] is not a permitted knob for %s" % (knob, self.name))
        return self

    @property
    def setknobs_exist(self):
        return len(self.set_knobs)>0

    def startcmd(self, os, fib, *args):
        """Return command to be used when starting this capability without
        the system job management infrastructure"""
        from ..osinfo import OSFamily
        extra_prms = ' '.join(args)
        return {
            OSFamily.FreeBSD: "setfib %s service %s restart %s" % (fib.value, self.name, extra_prms)
        }[os.family]

def add(searchdir):
    """Add user-defined capabilities in searchdir. Pop up one directory and
    use import_modules"""
    canonical_path = os.path.expanduser(searchdir)
    sys.path.append(canonical_path)

    capmod = importlib.import_module('frieze.capability')
    modules = os.listdir(canonical_path)
    for module_name in modules:
        # Push module information into frieze.capability, and fix resource_path
        try:
            module = importlib.import_module(module_name)
        except ModuleNotFoundError:
            continue

        for m in dir(module):
            elem = getattr(module, m)
            if type(elem)==type and isinstance(elem(), CapabilityTemplate):
                setattr(capmod, elem.__name__, elem)
                setattr(elem, 'resource_path', f'{module_name}.resources')

    importlib.invalidate_caches()

## Service definitions

class bird(CapabilityTemplate):
    package = 'bird'
    jailable = False

class dhcpd(CapabilityTemplate):
    package = 'isc-dhcp44-server'
    jailable = False
    knobs = [
        'ifaces'
    ]

class dhclient(CapabilityTemplate):
    jailable = False

class firstboot(CapabilityTemplate):

    def generate_cfg_files(self, host, bootstrap=False):

        if bootstrap:
            exclude = []
        else:
            exclude = [
                'bootstrap.sh',
                'configinit.payload',
                'resetimage.sh',
            ]

        return super().generate_cfg_files(host, __exclude__=exclude)

    def generate_bootstrap_tarball(self):
        """Generate tarball of commands to be executed by frieze_configinit, which
        executes this payload on first boot"""
        rv = {
            frieze.capability.ConfigFile.PRE_COMMAND_LIST : [
                'sysrc -f /etc/rc.conf firstboot_pkgs_list="bash nano ca_root_nss openssh-portable"',
                'sysrc -f /etc/rc.conf sshd_enable="NO"',
                'sysrc -f /etc/rc.conf.local openssh_enable="YES"',
                'mkdir -p /usr/local/etc/ssh',
                'printf "TrustedUserCAKeys /usr/local/etc/ssh/ca.pub\nPasswordAuthentication no\nChallengeResponseAuthentication no\nPermitRootLogin prohibit-password\n" > /usr/local/etc/ssh/sshd_config'
            ]
        }

        return\
            frieze.capability.ConfigInit(rv)\
            .generate()\
            .decode()

class gateway(CapabilityTemplate): pass

class jail(CapabilityTemplate):

    def generate_cfg_files(self, host):
        """Generate regular files, and then add in fstabs for individual jails """
        rv = super().generate_cfg_files(host, __exclude__=['jail-fstab', 'jail-zfs-skeleton', 'jail-configinit'])

        pkg_name = 'frieze.capability.resources.%s' % self.name
        jail_zfs_template    = pkg.resource_string(pkg_name, 'jail-zfs-skeleton').decode()
        jail_fstab_template  = pkg.resource_string(pkg_name, 'jail-fstab').decode()
        jail_cfinit_template = pkg.resource_string(pkg_name, 'jail-configinit').decode()

        for container in host.containers:
            fstab_file = f'/usr/local/jails/{container.sysname}.fstab'
            rv[self.next_cmd_cfg_name()] = mako.template.Template(jail_zfs_template).render(container=container, host=host)
            rv[fstab_file] = mako.template.Template(jail_fstab_template).render(container=container, host=host)

            # Create config payload
            cfinit_file = f'{container.jaildir}/root/cfinit'
            rv[cfinit_file] = mako.template.Template(jail_cfinit_template).render(container=container, host=host)

        return rv

class linux(CapabilityTemplate): pass

class named(CapabilityTemplate):
    package = 'bind912'

    def generate_cfg_files(self, host):
        rv = {}
        zones = {}

        # Load some resources
        pkg_name = 'frieze.capability.resources.%s' % self.name
        zone_template        = pkg.resource_string(pkg_name, 'zone.db').decode()
        revzone_template     = pkg.resource_string(pkg_name, 'revzone.db').decode()
        named_local_template = pkg.resource_string(pkg_name, 'named.conf.local').decode()

        # Track reverse zones
        networks = dict()

        # Generate forward lookup files
        for site in host.site.domain.site:
            filename = "/usr/local/etc/namedb/dynamic/%s.db" % site.zone
            zones[site.zone] = filename
            rv[filename] = mako.template.Template(zone_template).render(zonecontainer=site, host=host)

            # Break forward lookups into class C's and assign revzones
            for dhost in site.host:
                try:
                    networks[dhost.revzone]
                except KeyError:
                    networks[dhost.revzone] = []
                networks[dhost.revzone].append({
                    'fqdn'   : dhost.fqdn,
                    'ip_ext' : dhost.ip4().split('.')[-1]
                })

        for deployment in host.site.domain.deployment:
            filename = "/usr/local/etc/namedb/dynamic/%s.db" % deployment.zone
            zones[deployment.zone] = filename
            rv[filename] = mako.template.Template(zone_template).render(zonecontainer=deployment, host=host)

            for container in deployment.containers:
                try:
                    networks[deployment.revzone]
                except KeyError:
                    networks[deployment.revzone] = []

                networks[deployment.revzone].append({
                    'fqdn'   : container.fqdn,
                    'ip_ext' : '.'.join(reversed(container.ip4().split('.')[2:]))
                })

        for nw, zonehosts in networks.items():
            filename = "/usr/local/etc/namedb/dynamic/%s.db" % nw
            zones[nw] = filename
            rv[filename] = mako.template.Template(revzone_template).render(revzone=nw, zonehosts=zonehosts, host=host)

        # Generate named.conf.local
        rv['/usr/local/etc/namedb/named.conf.local'] =\
            mako.template.Template(named_local_template).render(zones=zones)

        return {**rv, **super().generate_cfg_files(host, __exclude__=['zone.db', 'revzone.db', 'named.conf.local'])}

class pf(CapabilityTemplate): pass

class pflate(CapabilityTemplate): pass

class pflog(CapabilityTemplate): pass

class openssh(CapabilityTemplate):
    package = 'openssh-portable'

class resolvconf(CapabilityTemplate): pass

class sshd(CapabilityTemplate): pass

class trust(CapabilityTemplate):

    def generate_cfg_files(self, host):
        rv = {}

        # Generate trusts
        if type(host)!=frieze.Container:
            return rv

        from .server import TrustType, CertAction, CertFormat

        pkg_name = 'frieze.capability.resources.%s' % self.name
        crt_template  = pkg.resource_string(pkg_name, 'chain.crt')
        priv_template = pkg.resource_string(pkg_name, 'private.pem')

        cap = host.capability
        if cap.capability_alias and cap.capability_alias.is_external:
            print(f'======>Generating trusts for [{cap.fqdn}]')
            aliases = [ca.fqdn for ca in cap.capability_alias]
            cert =\
                cap.deployment.domain\
                .trust(trust_type=TrustType.LETSENCRYPT)\
                .issue_certificate(
                    aliases,
                    CertAction.REMOTE_ACCESS,
                    cert_format=CertFormat.PEM
                )

            rv[cap.truststore(pubkey=True)]  = mako.template.Template(crt_template).render(cap=cap, cert=cert)
            rv[cap.truststore(pubkey=False)] = mako.template.Template(priv_template).render(cap=cap, cert=cert)

        return rv

class zfs(CapabilityTemplate):

    def generate_cfg_files(self, host):

        rv = super().generate_cfg_files(host, __exclude__=['zpool-init'])

        pkg_name = 'frieze.capability.resources.%s' % self.name
        zpool_template = pkg.resource_string(pkg_name, 'zpool-init').decode()

        for bs in host.block_storage:
            try:
                rv[frieze.capability.ConfigFile.PRE_COMMAND_LIST]
            except KeyError:
                rv[frieze.capability.ConfigFile.PRE_COMMAND_LIST] = []
            rv[frieze.capability.ConfigFile.PRE_COMMAND_LIST].append(
                mako.template.Template(zpool_template).render(blockstore=bs, host=host)
            )

        return rv


#!/usr/bin/python3

#Este programa desacumula os jsons, isto é: compara um arquivo com o subsequente e deleta qualquer timestamp menor do que o timestamp máximo encontrado no arquivo anterior, tanto para os dados de navegação quanto para o de telemetria. Deve-se especificar o endereço dos dados acumulados de entrada e o endereço dos dados desacumulados de saída. Deve-se também especificar uma data de início. No primeiro arquivo, começando com "0.", o programa irá deletar qualquer dado anterior à 00:00 da data especificada.

#Exemplo: python3 preprocess.py -i /home/felipeadachi/Dados/cumulating/cumulating2-13.03.19 -o /home/felipeadachi/Dados/non-cumulating/13.03.19 -d 2019,3,13

import json
import os
import collections
import datetime
import argparse
import numpy as np
from attributes import which_topast,past_to_get

input_1_filename = 'nonein.json'
output_filename = 'noneout.json'

parser = argparse.ArgumentParser(description='Preprocess Raw JSON files')

required = parser.add_argument_group('required named arguments')

required.add_argument('-i','--input', help='Location of local data to create database from', required=True)
required.add_argument('-o','--output', help='Output path', required=True)
required.add_argument('-d','--date', help='Date to decumulate', required=True)

args = vars(parser.parse_args())

date=args['date'].split(',')
folder_path_in = args['input']
folder_path_out = args['output']

which_todelta = [['deapth']]

#                                    Y   M  D
date_to_start=int(datetime.datetime(2019,1,19).strftime('%s'))*1000

date_to_start=int(datetime.datetime(int(date[0]),int(date[1]),int(date[2])).strftime('%s'))*1000

#---------------------#---------------------#---------------------#---------------------#---------------------#

def main():

    i=0
    number_of_files = len(os.listdir(folder_path_in))
    list_of_files = [None]*number_of_files
    print(number_of_files)
    for data_file in sorted(os.listdir(folder_path_in)):

        numfile=data_file.partition(".")[0]
        print(numfile)
        list_of_files[int(numfile)]=data_file
        print(numfile+">"+data_file)

    print(list_of_files)

    while i < number_of_files:
        json_current_content= read_json_content(folder_path_in + '/' + list_of_files[i])
        if (i != 0):
            json_preceding_content = read_json_content(folder_path_in + '/' + list_of_files[i-1])
            json_file_content = (json_preceding_content,json_current_content)
            print("max timestamps:")
            json_content_out = content_modifier(*json_file_content)
        else:
            json_content_out= check_start_date(json_current_content)

        json_content_out=motorsep(json_content_out)

        json_content_out=has_elapsed(json_content_out)

        json_content_out = delta(json_content_out)

        #json_content_out = past_commands(json_content_out,which_topast)

        #del json_content_out['0']
        if not os.path.exists(folder_path_out):
            os.makedirs(folder_path_out)
        write_json_content(folder_path_out + '/' + list_of_files[i],json_content_out)
        i += 1

        #json_file_content=read_json_content(folder_path_in+'/'+data_file)
        #del json_file_content['0']

        #write_json_content(folder_path_out+'/'+data_file,json_file_content)

        #json_file_content = None

        #json_file_content = read_json_content(folder_path_in+'/'+data_file)

        #if json_file_content is None:
            #exit(0)
        #print(data_file)
        #output_filename=folder_path_out+'/'+data_file
        #write_json_content(output_filename, json_file_content)

#---------------------#---------------------#---------------------#---------------------#---------------------#

def check_start_date(current_content):

    for i in range(0,2):

        data_list = current_content[str(i)]['data']

        l=0

        while l < len(data_list):

            data = data_list[l]
            cur_timestamp = data['timestamp']

            if cur_timestamp < date_to_start:
                #print("current timestamp:",cur_timestamp,"type:",type(cur_timestamp))
                #print("date_to_start:",date_to_start,"type:",type(date_to_start))

                del data_list[l]

            else:
                l=l+1

    return current_content

#---------------------#---------------------#---------------------#---------------------#---------------------#

def content_modifier(content_1, content_2):

    timestamps_nav = list()
    timestamps_telemetry = list()

    for data_nav in content_1['0']['data']:
        timestamps_nav.append(data_nav['timestamp'])

    max_timestamp_nav_preceding = max(timestamps_nav)
    print(max_timestamp_nav_preceding)
    data_list_nav = content_2['0']['data']

    l = 0

    while l < len(data_list_nav):

        data_nav = data_list_nav[l]
        cur_timestamp_nav = data_nav['timestamp']

        if cur_timestamp_nav <= max_timestamp_nav_preceding:
            del data_list_nav[l]

        else:
            l = l + 1

    for data_telemetry in content_1['1']['data']:
        timestamps_telemetry.append(data_telemetry['timestamp'])

    max_timestamp_telemetry_preceding = max(timestamps_telemetry)
    print(max_timestamp_telemetry_preceding)
    data_list_telemetry = content_2['1']['data']

    l = 0

    while l < len(data_list_telemetry):

        data_telemetry = data_list_telemetry[l]
        cur_timestamp_telemetry = data_telemetry['timestamp']

        if cur_timestamp_telemetry <= max_timestamp_telemetry_preceding:
            del data_list_telemetry[l]

        else:
            l = l + 1

    return content_2

#---------------------#---------------------#---------------------#---------------------#---------------------#

def motorsep(content_1):

    data_list = content_1['0']['data']

    i = 0
    while i < len(data_list):
        #print(i)
        data = data_list[i]
        nomotors=False
        nomtargs=False
        #try:
        #    motor1,motor2,motor3 = (data['motors'].split(","))

        #except KeyError:
        #    nomotors=True

        #if nomotors==False:
        #    data['motor1']=int(motor1)
        #    data['motor2']=int(motor2)
        #    data['motor3']=int(motor3)

        try:
            mtarg1,mtarg2,mtarg3 = data['mtarg'].split(",")
        except KeyError:
            print("key error mtarg:",data['timestamp'])
            nomtargs=True

        if nomtargs==False:
            data['mtarg1']=int(mtarg1)
            data['mtarg2']=int(mtarg2)
            data['mtarg3']=int(mtarg3)

        i += 1

    return content_1

#---------------------#---------------------#---------------------#---------------------#---------------------#

def read_json_content(filename):
    with open(filename, 'r') as f:
        return json.JSONDecoder(object_pairs_hook=collections.OrderedDict).decode(f.read(os.stat(filename).st_size))

#---------------------#---------------------#---------------------#---------------------#---------------------#
def delta(content):

    data_list=content['0']['data']

    i=0

    while (i<len(data_list)):

        data=data_list[i]
        for attribute in which_todelta:
            if i==0 or data['has_elapsed']==1:
                data[attribute[0]+'_delta'] = np.nan
                try:
                    prev=data[attribute[0]]
                except KeyError:
                    print("key error:",attribute)
            else:
                data_prev=data_list[i-1]
                if(attribute==["yaw"]):
                    #print("aqui")
                    #Quando o rov passa de 340 para 10 graus por exemplo, o delta seria calculado como 330 graus, quando na verdade ele deslocou apenas 20 graus
                    if abs(float(data[attribute[0]])-float(data_prev[attribute[0]]))>360-abs(float(data[attribute[0]])-float(data_prev[attribute[0]])):
                        data[attribute[0]+'_delta'] = float("%.2f" % round(360-(abs(float(data[attribute[0]])-float(data_prev[attribute[0]]))),2))
                        #print("aqui:",data[attribute[0]],"timestamp:",data['timestamp'])

                    else:
                        data[attribute[0]+'_delta'] = float("%.2f" % round(float(data[attribute[0]])-float(data_prev[attribute[0]]),2))

                else:
                    try:
                        if data[attribute[0]]==data_prev[attribute[0]]:
                            data[attribute[0]+'_delta'] = data_prev[attribute[0]+'_delta']
                        else:
                            data[attribute[0]+'_delta'] = float("%.2f" % round((float(data[attribute[0]])-float(data_prev[attribute[0]])),2))

                    except KeyError:
                        print("key error:",attribute)
        i+=1

    return content

#---------------------#---------------------#---------------------#---------------------#---------------------#

def has_elapsed(content):

    data_list = content['0']['data']

    i=0

    while (i<len(data_list)):

        data=data_list[i]
        if i==0:
            data['has_elapsed']=1
        if i >= 1:
            if (data['timestamp']-data_list[i-1]['timestamp'])>=500:
                data['has_elapsed']=1
            else:
                data['has_elapsed']=0
        i+=1
    return content

#---------------------#---------------------#---------------------#---------------------#---------------------#

def past_commands(content,which_topast):

    data_list=content['0']['data']

    i=0

    while (i<len(data_list)):

        #get_token retorna o número de vezes permitido para pegar os comandos passados, com base nos flags "has_elapsed" de cada timestep
        token=get_token(i,data_list)
        #print("token for timestep:",i,"is:",token)
        data=data_list[i]
        tkn={}

        for att in which_topast:
                    tkn[att[0]]=token

        for past in past_to_get:

            if i<past:
                for att in which_topast:
                    data[att[0]+'_t'+str(past)]=np.nan

            else:

                for att in which_topast:

                    try:
                        if tkn[att[0]]<=0:
                            data[att[0]+'_t'+str(past)]=np.nan
                        else:
                            data[att[0]+'_t'+str(past)]=data_list[i-past][att[0]]
                            tkn[att[0]]-=1
                            #print("token for:",motor,"in tstep:",i,"is:",tkn[motor[0]])
                    except KeyError:
                        data[att[0]+'_t'+str(past)]=np.nan
                        tkn[att[0]]-=1
        i+=1

    return content

def get_token(i,data_list):
    j=0
    token=0
    while j<max(past_to_get):
        if data_list[i-j]['has_elapsed'] == 0:
            token+=1
        else:
            return token

        j+=1
    return token

#---------------------#---------------------#---------------------#---------------------#---------------------#

def write_json_content(filename, content):
    with open(filename, 'w') as f:
        f.write(json.dumps(content,indent=4))

#---------------------#---------------------#---------------------#---------------------#---------------------#

if(__name__ == '__main__'):
    main();


import openpyxl
from openpyxl.utils import get_column_letter
import os

path = 'C:\\Users\\Arkadiusz\\Desktop\\FV\\Maj'

cur = os.getcwd()                # Check current working directory.
print('Current working directory %s' % cur)  # Print current working directory in console

os.chdir(path)                  # Change the directory
cur2 = os.getcwd()               # Check current working directory
print(cur2)                     # Print current working directory in console

print('-----')                  # Print ------
wb = openpyxl.load_workbook('domeny-tf.xlsx') # Load excell form path
# sheet = wb['domeny-tf']         # Retrieving a sheet from a workbook

sheet = wb.active

# row = sheet.max_row             # Print maximum number of occupied row in console
# print(row)                                  # print row
# print(sheet.max_column)                     # print maximum number of occupied column
# print(get_column_letter(sheet.max_column))  # print maximum column letter of occupied

# for i in range(1,row +1):
#     print(i, sheet.cell(row=i, column=1).value)

# print(tuple(sheet['A1':'C3']))

# for rowOfCellObjects in sheet['A1':'C3']:   # Iteraction over the rows
#     for cellObj in rowOfCellObjects:        # Iteraction over the column
#         print(cellObj.coordinate, cellObj.value)
#     print('---Koniec wiersza---')

list(sheet.columns)  # List sheet column
#
for cellObj in list(sheet.columns)[0]:
    print(cellObj.value)


import random
import string
import requests
import cfscrape
import os
from bs4 import BeautifulSoup

countsPhoto = int(input('Введите количество фотографий: '))

index_element = string.ascii_lowercase + string.digits
# print (index_element)

errors = 0
goods = 0
i = 0

def random_char(y):
     return ''.join(random.choice(index_element) for x in range(y))

def parse_photo():
        print ('Цикл ' + str(i)  + ' пошел')
        index = random_char(6)
        url = 'https://prnt.sc/' +  index
        # print (url)
        scraper = cfscrape.create_scraper()
        r = scraper.get(url)
        soup = BeautifulSoup(r.text)
        # print (soup)
        img_link = soup.find('img', {'class': 'screenshot-image'}).get('src')
        # print (img_link)

        file_name = img_link.split('/')[-1]
        # print (file_name)

        dirName = 'img'
        try:
            os.mkdir(dirName)
        except FileExistsError:
            None

        filepath = os.path.join('./img', file_name)

        img = scraper.get(img_link)
        out = open(filepath, "wb")
        out.write(img.content)
        out.close()

for x in range(countsPhoto):
    i += 1
    try:
        parse_photo()
        goods += 1
        print ('Успешних циклов: ' + str(goods))
    except Exception as e:
        errors += 1
        print ('Проваленых циклов: ' + str(errors))
        None


import sqlite3
from turtle import title

with sqlite3.connect("database1.db") as db:
    cursor = db.cursor()

    # cursor.execute("""CREATE TABLE IF NOT EXISTS staff(
    #     id INTEGER PRIMARY KEY,
    #     FIO VARCHAR(30),
    #     age INTEGER(3),
    #     sex INTEGER NOT NULL DEFAULT 1,
    #     department VARCHAR(100),
    #     title VARCHAR(100)
    # )""")

    # values = [
    #     ("Диденко Д.Н.", "43", "1", "Департамент логистики", "Директор по логистике"),
    #     ("Каруна Т.В.", "35", "2", "Финансово-экономический отдел", "Финансовый директор"),
    #     ("Дворяшина Н.Н.", "33", "2", "Департамент учета и отчетности", "Главный бухгалтер"),
    #     ("Золоторева М.А.", "26", "2", "Отдел по работе с персоналом", "Менеджер по работе с персоналом"),
    #     ("Лучин А.С.", "36", "1", "Департамент логистики", "Директор по транспорту")

    # ]

    # cursor.executemany("INSERT INTO staff(FIO, age, sex, department, title) VALUES(?, ?, ?, ?, ?)", values)
def data_output():
    try:
        db = sqlite3.connect("database1.db")
        cursor = db.cursor()

        for data in cursor.execute("SELECT * FROM staff"): # Этот метод выводит все записи таблицы по строчно
            print(data)
    except sqlite3.Error as e:
        print("Error", e)
    finally:
        cursor.close()
        db.close()

def registration():
    FIO = input("Введите Фамилию и инициалы сотрудника: ")
    age = int(input("Введите возраст сотрудника: "))
    sex = int(input("Введите пол сотрудника 1-муж./2-жен.: "))
    department = input("Введите название отдела/департамента: ")
    title = input("Введите должность сотрудника: ")

    try:
        db = sqlite3.connect("database1.db")
        cursor = db.cursor()

        cursor.execute("SELECT FIO FROM staff WHERE FIO = ?", [FIO])
        if cursor.fetchone() is None:
            values = [FIO, age, sex, department, title]

            cursor.execute("INSERT INTO staff(FIO, age, sex, department, title) VALUES(?, ?, ?, ?, ?)", values)
            db.commit()
            print("Сотрудник зарегистрирован!")
        else:
            print('Такой сотрудник уже существует!')
            registration()
    except sqlite3.Error as e:
        print("Error", e)
    finally:
        cursor.close()
        db.close()

def delete_db(FIO):
    try:
        db = sqlite3.connect("database1.db")
        cursor = db.cursor()

        cursor.execute("DELETE FROM staff WHERE FIO = ?", [FIO])
        db.commit()
        print("Запись удалена")
    except sqlite3.Error as e:
        print("Error", e)
    finally:
        cursor.close()
        db.close()

def update_db(column_name, value, FIO):
    try:
        db = sqlite3.connect("database1.db")
        cursor = db.cursor()

        cursor.execute(f'UPDATE staff SET {column_name} = ? WHERE FIO = ?', (value, FIO))
        db.commit()
        print("Изменения внесены!")
    except sqlite3.Error as e:
        print("Error", e)
    finally:
        cursor.close()
        db.close()


from dateutil import parser
from dateutil.relativedelta import relativedelta
import os
global months

#Equipment script
def get_equipment_cost():
  has_equipment = input("Equipment? (y/n) > ")
  print("")
  if has_equipment.lower() == 'y':
    equip_monthly = float(input("Enter monthly equipment cost: "))
    return equip_monthly * months
  else:
    return 0

#Discount script
def get_discount():
  percent_str = input("What is the discount percent? > ")
  percent = int(percent_str) / 100
  return percent

def program():
  global months
  date1 = input("Date with previous provider (MM/DD/YYYY): ")
  date2 = input("Date with Vivint (MM/DD/YYYY): ")

  date1 = parser.parse(date1)
  date2 = parser.parse(date2)
  global months

  rd = relativedelta(date2, date1)
  months = rd.years * 12 + rd.months

  rd = relativedelta(date2, date1)
  day_diff = rd.days

  print("")
  print(f"There are {months} months between {date1:%m/%d/%Y} and {date2:%m/%d/%Y}")
  print("")
  monthdif = int(input("How many months long is the customers contract > "))
  trumonth = monthdif-months
  if 0 <= day_diff <= 3:
    print("\nThe difference between the dates is less than 3 days. The extra month will be calculated automatically.")
    print(f"\nThe payoff period has increased from the initial {trumonth} months to {trumonth + 1} months after applying the 3-day rule")
    trumonth = trumonth + 1
  else:
    print("\nThree day rule does not apply")
    print(f"\nThere are {trumonth} months left for the customer to pay")

  rmr = float(input("\nWhat is the monthly rate? > "))

  discount = input("\nDiscount? (y/n) > ")
  equip_cost = get_equipment_cost()

  if discount == 'y':
    percent = get_discount()
    total = (rmr * trumonth * percent) + equip_cost
  else:
    total = (rmr * trumonth) + equip_cost
  total = round(total, 2)
  print("")
  print(f"The total buyout is ${total}")

program()

while True:
  print("")
  again = input("Run calculation again? (y/n): ")
  if again.lower() == 'y':
    print("")
    os.system('cls' if os.name == 'nt' else 'clear')
    program()
  else:
    break

print("Done")


from cement import Controller
from cement.utils.version import get_version_banner
from ..core.version import get_version

VERSION_BANNER = """
Add Dependencies and Extensions for Hashicorp Packer %s
%s
""" % (get_version(), get_version_banner())

class Base(Controller):
    class Meta:
        label = 'base'
        description = 'Add Dependencies and Extensions for Hashicorp Packer'
        arguments = [
            (['-v', '--version'],
             {'action': 'version',
              'version': VERSION_BANNER}),
        ]

    def _default(self):
        """Default action if no sub-command is passed."""

        self.app.args.print_help()


import cv2
import base64
import tensorflow as tf
import numpy as np

def prep_points(boxes):
    all_points_boxes = []
    for box in boxes:
        curr_box = []
        box_list =  list(box.items());
        for i in range(4):
            label_x, point_x = box_list[i * 2]
            label_y, point_y = box_list[(i * 2) + 1]
            point = (int(point_x), int(point_y))
            curr_box.append(point)

        all_points_boxes.append(curr_box)

    return (all_points_boxes)

def bb_intersection_over_union(boxA, boxB):
    _, boxA_top_left_y = boxA[0];
    _, boxA_bottom_right_y = boxA[1];

    _, boxB_top_left_y = boxB[0];
    _, boxB_bottom_right_y = boxB[1];

    inter_top_left_y = max(boxA_top_left_y, boxB_top_left_y)
    inter_bottom_right_y = min(boxA_bottom_right_y, boxB_bottom_right_y)
    interArea = max(0, inter_bottom_right_y - inter_top_left_y)

    boxAArea = (boxA_bottom_right_y - boxA_top_left_y)
    boxBArea = (boxB_bottom_right_y - boxB_top_left_y)

    # compute the intersection over union by taking the intersection
    # area and dividing it by the sum of prediction + ground-truth
    # areas - the interesection area
    iou = interArea / float(boxAArea + boxBArea - interArea)

    # return the intersection over union value
    return iou

def foramt_4_points_bb_to_2_points_bb(bounding_box):
    aX, aY = bounding_box[0];
    bX, bY = bounding_box[1];
    cX, cY = bounding_box[2];
    dX, dY = bounding_box[3];

    top_left_x = min(aX, bX, cX, dX)
    top_left_y = min(aY, bY, cY, dY)

    bottom_right_x = max(aX, bX, cX, dX)
    bottom_right_y = max(aY, bY, cY, dY)

    return ([(top_left_x, top_left_y), (bottom_right_x, bottom_right_y)])

def IoU_by_y(dishes_list, prices_list):
    l = list();
    for dish in dishes_list:
        best_iou_value = 0;
        best_iou_price = ();

        dish_box = foramt_4_points_bb_to_2_points_bb(dish)

        for price in prices_list:
            price_box = foramt_4_points_bb_to_2_points_bb(price)

            curr_iou = bb_intersection_over_union(dish_box, price_box)

            if (curr_iou > best_iou_value):
                best_iou_value = curr_iou
                best_iou_price = price

        if (best_iou_value > 0):
            l.append({"price": best_iou_price, "dish": dish})

    return l

def IoU_by_y_with_threshold(dishes_list, prices_list):
    l = list();
    for dish in dishes_list:
        all_prices_in_same_row = list()
        best_iou_value = 0;
        best_iou_price = ();

        dish_box = foramt_4_points_bb_to_2_points_bb(dish)

        for price in prices_list:
            price_box = foramt_4_points_bb_to_2_points_bb(price)

            curr_iou = bb_intersection_over_union(dish_box, price_box)

            if (curr_iou >= 0.5):
                all_prices_in_same_row.append(price)

        if (len(all_prices_in_same_row) > 0):
            all_prices_in_same_row = sorted(all_prices_in_same_row, key=lambda curr_price: curr_price[0][0])

            l.append({"price": all_prices_in_same_row[0], "dish": dish})

    return l

def crop_image(img, bounding_box):
    bb_2_points = foramt_4_points_bb_to_2_points_bb(bounding_box)
    Ax, Ay = bb_2_points[0]
    Bx, By = bb_2_points[1]
    cropped_img = img[Ay:By, Ax:Bx]
    return (cropped_img)

def bb_2_points_intersection_over_union(boxA, boxB):
    # determine the (x, y)-coordinates of the intersection rectangle
    xA = max(boxA[0], boxB[0])
    yA = max(boxA[1], boxB[1])
    xB = min(boxA[2], boxB[2])
    yB = min(boxA[3], boxB[3])

    # compute the area of intersection rectangle
    interArea = max(0, xB - xA + 1) * max(0, yB - yA + 1)

    # compute the area of both the prediction and ground-truth
    # rectangles
    boxAArea = (boxA[2] - boxA[0] + 1) * (boxA[3] - boxA[1] + 1)
    boxBArea = (boxB[2] - boxB[0] + 1) * (boxB[3] - boxB[1] + 1)

    # compute the intersection over union by taking the intersection
    # area and dividing it by the sum of prediction + ground-truth
    # areas - the interesection area
    iou = interArea / float(boxAArea + boxBArea - interArea)

    # return the intersection over union value
    return iou

def find_best_contours(img):
    (original_h, original_w, original_z) = img.shape
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    blur = cv2.GaussianBlur(gray, (5, 5), 1)

    if (original_h <= 35):
        th = cv2.adaptiveThreshold(blur, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY_INV, 5, 2)
    else:
        th = cv2.adaptiveThreshold(blur, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY_INV, 15, 3)

    rect_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (1, original_h))
    th = cv2.morphologyEx(th, cv2.MORPH_CLOSE, rect_kernel)

    contours, hierarchy = cv2.findContours(th, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)
    filtered_contours = [];

    for i, cnt in enumerate(contours):
        is_found_bigger = False
        [x_i, y_i, w_i, h_i] = cv2.boundingRect(cnt)

        for j, _ in enumerate(contours):
            [x_j, y_j, w_j, h_j] = cv2.boundingRect(contours[j])
            iou = bb_2_points_intersection_over_union([x_i, y_i, x_i + w_i, y_i + h_i],
                                                      [x_j, y_j, x_j + w_j, h_j + y_j])

            if (iou > 0):
                if (w_i * h_i < w_j * h_j):
                    is_found_bigger = True
                    break

        if not (is_found_bigger):
            filtered_contours.append(contours[i])

    filtered_contours = sorted(filtered_contours, key=lambda cnt: cv2.boundingRect(cnt)[0])
    return filtered_contours

def predict_price_contours(filtered_contours, img, digits_model, graph):
    (original_h, original_w, original_z) = img.shape
    predicted_price = ""
    for cnt in filtered_contours:
        [x, y, w, h] = cv2.boundingRect(cnt)

        if (h < 0.2 * original_h and w < 0.2 * original_w):
            h = int(0.2 * original_h)
            w = int(0.2 * original_w)

        current_digit = img[y:y + h, x:x + w]
        current_digit = cv2.cvtColor(current_digit, cv2.COLOR_BGR2GRAY)
        row, col = current_digit.shape[:2]
        bottom = current_digit[row - 2:row, 0:col]
        mean = cv2.mean(bottom)[0]

        if (original_h <= 35):
            bordersize = 5
        else:
            bordersize = 10

        th = cv2.resize(current_digit, None, fx=2, fy=2)
        th = cv2.adaptiveThreshold(th, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY_INV, 29, 15)

        th = cv2.resize(th, None, fx=0.5, fy=0.5)
        th = cv2.copyMakeBorder(th, top=bordersize, bottom=bordersize, left=bordersize, right=bordersize,
                                borderType=cv2.BORDER_CONSTANT, value=[0, 0, 0])
        th = cv2.resize(th, dsize=(28, 28), interpolation=cv2.INTER_CUBIC)

        colors, counts = np.unique(th.reshape(-1), axis=0, return_counts=True)

        black_ratio = counts[0] / float(28 * 28)
        if (black_ratio >= 0.99):
            continue

        if (h / float(original_h) < 0.5 or black_ratio > 0.90):
            if (predicted_price == "" or "." in predicted_price):
                continue

            predicted_price = predicted_price + "."
            continue

        th = np.reshape(th, newshape=(28, 28, 1))
        img_for_prediction = np.expand_dims(th, axis=0)

        with graph.as_default():
            pred = digits_model.predict(img_for_prediction)[0]

        maximum = np.max(pred)
        predicted_digit = np.where(pred == maximum)[0][0]
        predicted_price = predicted_price + str(predicted_digit)

    return predicted_price

def convert_points_to_images(img, prices_dishes_matches, digits_model, graph):
    ret = list()
    for x in prices_dishes_matches:
        cropped_dish = crop_image(img, x["dish"])
        cropped_price = crop_image(img, x["price"])
        price_contours = find_best_contours(cropped_price)
        price_string = predict_price_contours(price_contours, cropped_price, digits_model, graph)
        _, binframe = cv2.imencode('.jpg', cropped_dish)
        dish_base64 = base64.b64encode(binframe).decode('UTF-8')

        try:
            price_float = float(price_string)
        except ValueError:
            continue

        ret.append({"dish": dish_base64, "price": price_float})
    return ret;

def extract_items(img, dishes, prices, digits_model, graph):
    dishes_points = prep_points(dishes)
    prices_points = prep_points(prices)

    prices_dishes_matches = IoU_by_y_with_threshold(dishes_points, prices_points)
    prices_dishes_matches = sorted(prices_dishes_matches, key=lambda pair: pair["dish"][0][1])

    final_res = convert_points_to_images(img, prices_dishes_matches, digits_model, graph)
    return final_res


from sortedcontainers import SortedList
class Solution:
    def countSmaller(self, nums: List[int]) -> List[int]:
        # out=[0]*len(nums)
        # for i in range(len(nums)-1):
        #     j= i+1
        #     c=0
        #     while j<len(nums):
        #         if nums[j]<nums[i]:
        #             c+=1
        #         j+=1
        #     out[i] = c
        # return out

        output = []
        arr = SortedList(nums)
        for i in nums:
            idx = arr.index(i)
            output.append(idx)
            arr.remove(i)
        return output


from itertools import product
import json
from PyQt5.QtCore import QVariant

from qgis.core import (QgsPointXY,
                       QgsGeometry,
                       QgsFeature,
                       QgsFields,
                       QgsField)

from ..utils import convert

def get_request_point_features(route_dict, row_by_row):
    """
    Processes input point features depending on the layer to layer relation in directions settings

    :param route_dict: all coordinates and ID field values of start and end point layers
    :type route_dict: dict

    :param row_by_row: Specifies whether row-by-row relation or all-by-all has been used.
    :type row_by_row: str

    :returns: tuple of coordinates and ID field value for each routing feature in route_dict
    :rtype: tuple of QgsPointXY and others
    """

    locations_list = list(product(route_dict['start']['geometries'],
                                  route_dict['end']['geometries']))
    values_list = list(product(route_dict['start']['values'],
                               route_dict['end']['values']))

    # If row-by-row in two-layer mode, then only zip the locations
    if row_by_row == 'Row-by-Row':
        locations_list = list(zip(route_dict['start']['geometries'],
                                  route_dict['end']['geometries']))

        values_list = list(zip(route_dict['start']['values'],
                               route_dict['end']['values']))

    for properties in zip(locations_list, values_list):
        # Skip if first and last location are the same
        if properties[0][0] == properties[0][-1]:
            continue

        coordinates = [QgsPointXY(x, y) for x, y in properties[0]]
        values = properties[1]

        yield (coordinates, values)

def get_fields(from_type=QVariant.String, to_type=QVariant.String, from_name="FROM_ID", to_name="TO_ID", line=False):
    """
    Builds output fields for directions response layer.

    :param from_type: field type for 'FROM_ID' field
    :type from_type: QVariant enum

    :param to_type: field type for 'TO_ID' field
    :type to_type: QVariant enum

    :param from_name: field name for 'FROM_ID' field
    :type from_name: str

    :param to_name: field name for 'TO_ID' field
    :type to_name: field name for 'TO_ID' field

    :param line: Specifies whether the output feature is a line or a point
    :type line: boolean

    :returns: fields object to set attributes of output layer
    :rtype: QgsFields
    """

    fields = QgsFields()
    fields.append(QgsField("DIST_KM", QVariant.Double))
    fields.append(QgsField("DURATION_H", QVariant.Double))
    fields.append(QgsField("PROFILE", QVariant.String))
    fields.append(QgsField("OPTIONS", QVariant.String))
    fields.append(QgsField(from_name, from_type))
    if not line:
        fields.append(QgsField(to_name, to_type))

    return fields

def get_output_feature_directions(response, profile, options=None, from_value=None, to_value=None):
    """
    Build output feature based on response attributes for directions endpoint.

    :param response: API response object
    :type response: dict

    :param profile: Transportation mode being used
    :type profile: str

    :param options: Costing option being used.
    :type options: dict

    :param from_value: value of 'FROM_ID' field
    :type from_value: any

    :param to_value: value of 'TO_ID' field
    :type to_value: any

    :returns: Ouput feature with attributes and geometry set.
    :rtype: QgsFeature
    """
    response_mini = response['trip']
    feat = QgsFeature()
    coordinates, distance, duration = [], 0, 0
    for leg in response_mini['legs']:
            coordinates.extend([
                list(reversed(coord))
                for coord in convert.decode_polyline6(leg['shape'])
            ])
            duration += round(leg['summary']['time'] / 3600, 3)
            distance += round(leg['summary']['length'], 3)

    qgis_coords = [QgsPointXY(x, y) for x, y in coordinates]
    feat.setGeometry(QgsGeometry.fromPolylineXY(qgis_coords))
    feat.setAttributes([distance,
                        duration,
                        profile,
                        json.dumps(options),
                        from_value,
                        to_value
                        ])

    return feat


import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import GridSearchCV
from sklearn.pipeline import Pipeline
import matplotlib.pyplot as plt
from sklearn.preprocessing import StandardScaler

df = pd.read_csv('IMDB.csv')
df = df.sample(frac=1).reset_index(drop=True)
df = df.head(int(len(df)*(20/100)))

reviews = df['review'].values
sentiments = df['sentiment'].values

pipeline = Pipeline([
    ('tfidf', TfidfVectorizer()),
    ('clf', LogisticRegression(max_iter=1000))
])

parameters = {
    'tfidf__max_features': [100, 1000, 10000, 100000],
    'clf__C': [0.1,0.5,1,5]
}

grid_search = GridSearchCV(pipeline, parameters, cv=5, scoring="accuracy")
grid_search.fit(reviews, sentiments)

# Extract the results
results = grid_search.cv_results_
print(results)
mean_scores = results['mean_test_score']

# Reshape the mean scores array to match the parameter combinations
mean_scores = mean_scores.reshape(len(parameters['tfidf__max_features']), len(parameters['clf__C']))

# Plot the average accuracy scores for different max_features values
max_features_values = parameters['tfidf__max_features']
C_values = parameters['clf__C']

for i, max_features in enumerate(max_features_values):
    plt.plot(C_values, mean_scores[i], label=f"max_features={max_features}")

plt.xlabel('C')
plt.ylabel('Average Accuracy Score')
plt.title('C vs Average Accuracy Scores for Different max_features')
plt.legend()
plt.show()

Usage of OvO,OvR,GridSearchCV/Simple NN Pipeline.py


from django.http import HttpResponseRedirect
from django.shortcuts import render_to_response
from django.contrib import messages
from django.utils.translation import ugettext as _
from django.contrib.auth.decorators import login_required
from django.contrib.contenttypes.models import ContentType

from rathings.models import Like, Dislike, dislike_created

@login_required
def like(request, object_id):
    """
    """
    ctype_id = request.GET.get("ctype", None)
    next = request.GET.get("next", None)

    if ctype_id is None:
        messages.add_message(request, messages.ERROR, _("Error interno"))
        return HttpResponseRedirect(request.META.get('HTTP_REFERER'))

    object_ctype = ContentType.objects.get_for_id(ctype_id)
    obj, created = Like.objects.get_or_create(content_type=object_ctype, object_id=object_id, user=request.user)
    if not created:
        messages.add_message(request, messages.INFO, _("Ya te gusta ese comentario"))
    else:
        messages.add_message(request, messages.SUCCESS, _("Ahora te gusta este comentario"))

    if next is not None:
        return HttpResponseRedirect(next)
    return HttpResponseRedirect(request.META.get('HTTP_REFERER'))

@login_required
def dislike(request, object_id):
    """
    """
    ctype_id = request.GET.get("ctype", None)
    next = request.GET.get("next", None)

    if ctype_id is None:
        messages.add_message(request, messages.ERROR, _("Missing ctype_id"))
        return HttpResponseRedirect(request.META.get('HTTP_REFERER'))
    object_ctype = ContentType.objects.get_for_id(ctype_id)
    obj, created = Dislike.objects.get_or_create(content_type=object_ctype, object_id=object_id, user=request.user)
    if not created:
        messages.add_message(request, messages.INFO, _("Ya denunciaste este  {}".format(object_ctype.model)))
    else:
        messages.add_message(request, messages.SUCCESS, _("Acabas de denunciar el {}".format(_(object_ctype.model))))
        dislike_created.send(sender=Dislike, instance=obj)

    if next is not None:
        return HttpResponseRedirect(next)
    return HttpResponseRedirect(request.META.get('HTTP_REFERER'))


# Shared memory 를 만들기 위해서 Manger 모듈 import
from multiprocessing import Manager
# Pro1 프로세스 import
from Step_1.Pro1 import Pro1
from Step_1.Pro2 import Pro2

class body:
    def __init__(self):
        # 1. Shared memory 의 구조를 선언
        self.shared_mem = self.make_memory(label=[])

    def run(self):
        process_list = [Pro1(self.shared_mem), Pro2(self.shared_mem)]       # 리스트에 실행 시키고 싶은 프로세스 넣기
        job_list = []                         # 프로세스 실행후 join 시키기 위해서 리스트 준비

        for __ in process_list:
            __.start()
            job_list.append(__)

        for job in job_list:
            job.join()

    def make_memory(self, label):
        # Tip 및 설명
        # 이 기능의 주된 역할을 Shared memory 의 구조를 작성해서 넘겨준다 (return).
        # 이렇게 하면 Shared memory 의 구조를 변경하거나 관리하기 쉽다.
        # 또한 Shared memory 의 사용 용도에 따라서 구조를 적절하게 사용하는 것이 좋으며
        # dict를 사용하는 편이 데이터 관리에 적합하다.
        return Manager().dict({                    # 기본적으로 dict의 특징을 모두 사용할 수 있다.
            'x축_데이터': [],                # 따라서 dict의 사용법을 잘 알아두는 것이 중요하다.
            'y축_데이터': [],
            1: '숫자도_입력이_가능하다.',
            'Name': 'Shared_mem1',           # 이렇게 이름으로 붙여서 메모리를 관리할 수도 있으며 응용하여 아래와 같이
        })

if __name__ == '__main__':
    Model = body()      # body class를 호출
    Model.run()         # body class의 run함수를 사용하여 프로세스 구동


import os
import json
import requests as req
import pprint as pp
import time
from flask import Flask, request, jsonify
from peewee import SqliteDatabase
from User import User
from helpers import *
import threading
import subscribe
import argparse
from Store import Store

app = Flask(__name__)
db = SqliteDatabase('following.db')

def main():
    parser = argparse.ArgumentParser(description='Notify a Slack channel when Twitch users go live')
    parser.add_argument('-d', '--dev', action='store_true', dest='dev_mode')
    args = parser.parse_args()
    if args.dev_mode:
        print('Starting in dev mode')
        print('Instead of reading from Environment Variables, a local config.ini will be used')
        Store.setup('config.ini')
    else:
        subscribe.check_env_vars()
        Store.setup()

    Store.refresh_access_token()

    subscribe_refresh = threading.Thread(target=subscription_handler, daemon=True)
    subscribe_refresh.start()
    app.run(host='0.0.0.0', port=5000)

@app.route('/twitchbot', methods=['POST'])
def user_update():
    data = request.get_json()['data']
    if data:
        data = data[0]
        if data['type'] == 'live':
            user_id = data['user_id']
            user = User.get(User.user_id == int(user_id))
            curr_time = int(time.time())
            last_live = user.last_live
            user.last_live = curr_time
            user.save()
            print(f'{user.display_name.capitalize()} is live!')
            if last_live is None:
                send_to_slack(data, user)
            else:
                last_live = int(last_live.timestamp())
                if (curr_time - last_live) < 30*60:
                    print(f'{user.display_name.capitalize()} went live recently, not sending message')
                else:
                    send_to_slack(data, user)
    else:
        print('Empty Data - Stream ended')
    return jsonify(success=True)

@app.route('/twitchbot', methods=['GET'])
def accept_challenge():
    print('got GET')
    challenge = request.args.get('hub.challenge')
    topic = request.args.get('hub.topic')
    user_id = topic.split('user_id=')[-1]
    # print(challenge)
    # print(user_id)
    user = User.get(User.user_id == int(user_id))
    user.last_subscribed = int(time.time())
    user.save()
    return challenge, 200, {'ContentType':'text/plain'}

@app.before_request
def before_request():
    db.connect()

@app.after_request
def after_request(response):
    db.close()
    return response

def send_to_slack(data, user):
    stream = make_stream(data)
    message = make_message(user, stream)
    r = req.post(
        Store.slack_webhook,
        headers={ 'Content-type': 'application/json' },
        data=json.dumps(message)
    )
    print(r)

def subscription_handler():
    subscribe.prepare()
    while True:
        print("resubscribing to all")
        for user in User.select():
            subscribe.new_sub(user.user_id)
        time.sleep(Store.subscription_time - 120)

def user_handler():
    while True:
        for user in User.select():
            info = subscribe.get_user_info(user.display_name.lower())
            subscribe.update_or_create(*info)
        update_home()
        time.sleep(5 * 60 * 60)

def update_home():
    while True:
        pass
        # update home with new user data

if __name__ == '__main__':
    main()


'''
Created on Feb 26, 2017

@author: ubuntu
'''

import sqlite3
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
from dateutil import parser
from matplotlib import style
style.use('fivethirtyeight')

conn = sqlite3.connect('tutorial.db')
c = conn.cursor()

def graph_data():
    c.execute('SELECT datestamp, value FROM stuffToPlot')
    data = c.fetchall()

    dates = []
    values = []

    for row in data:
        dates.append(parser.parse(row[0]))
        values.append(row[1])

    plt.plot_date(dates,values,'-')
    plt.show()

graph_data()
c.close
conn.close()


import xlrd
import csv
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import style
style.use("ggplot")
from sklearn.cluster import KMeans
import math
from math import radians, cos, sin, asin, sqrt

#Function to convert longitude/latitude distance to km
def haversine(lon1, lat1, lon2, lat2):
    """
    Calculate the great circle distance between two points
    on the earth (specified in decimal degrees)
    """
    # convert decimal degrees to radians
    lon1, lat1, lon2, lat2 = map(radians, [lon1, lat1, lon2, lat2])
    # haversine formula
    dlon = lon2 - lon1
    dlat = lat2 - lat1
    a = sin(dlat/2)**2 + cos(lat1) * cos(lat2) * sin(dlon/2)**2
    c = 2 * asin(sqrt(a))
    km = 6367 * c
    return km

def antenna_classify ():
    homes_path = 'houseList.xlsx' #Opening List of Houses [Location Code, Addresss, LAT, LONG]
    antennas_path = 'antennaLocations.xlsx' #Opening List of Antennas [Location Code, Addresss, LAT, LONG]

    #Number of Cluster Desired
    number_clusters = 71 #44 has been best

    #Clears previous output
    open('output.csv', 'w').close()

    #Opening Specific sheet in the Excel Workbook
    homes_sheet = xlrd.open_workbook(homes_path).sheet_by_index(0)
    antennas_sheet = xlrd.open_workbook(antennas_path).sheet_by_index(0)
    # print("houseList has " + str(homes_sheet.nrows) + " rows and " + str(homes_sheet.ncols) + " columns.")
    # print("antennaLocations has " + str(antennas_sheet.nrows) + " rows and " + str(antennas_sheet.ncols) + " columns.")

    # for row in range(1, homes_sheet.nrows):
    #         print("code: " + str(homes_sheet.cell_value(row, 0)) + " addr: " + str(homes_sheet.cell_value(row, 1)) + " LAT: " + str(homes_sheet.cell_value(row, 2)) + " LONG: " + str(homes_sheet.cell_value(row, 3)))

    # Array of LAT,LONG coordinates for each house
    # Created for numpy array conversion and KMeans
    house_array = []

    # Populates house_array
    for row in range(1, homes_sheet.nrows):
        array_tuple = [homes_sheet.cell_value(row, 2), homes_sheet.cell_value(row, 3)]
        house_array.append(array_tuple)

    # print(house_array)
    # print(len(house_array))

    # Converts to numpy for kmeans clustering
    # This is where the magic happens
    X = np.array(house_array)
    kmeans = KMeans(n_clusters=number_clusters)
    kmeans.fit(X)

    # Centroids Array corresponding to Antenna Locations
    # Labels Array to indicate which cluster each point (house) belongs to
    centroids = kmeans.cluster_centers_
    labels = kmeans.labels_

    # print(centroids)
    # print(labels)

    # List of Colors for visualization
    colors = ["g.","r.","c.","y.", "k.", "b."] * 500

    # Array of arrays to putting each point in its corresponding cluster
    labeled_points = [[] for i in range(number_clusters)]

    # Populating labeled_points
    for i in range(len(X)):
        # print("coordinate:",X[i], "label:", labels[i])
        labeled_points[labels[i]].append(X[i])

        # #LONG AND LAT BACKWARDS ON EXCEL
        # dist = haversine((centroids[labels[i]][1]),[labels[i]][0],X[i][1],X[i][0])
        # # dist = math.sqrt((centroids[labels[i]][0]-X[i][0])**2 +(centroids[labels[i]][1]-X[i][1])**2)
        # distances_from_centroid[labels[i]].append(dist)

        # Plotting all the clusters
        plt.plot(X[i][0], X[i][1], colors[labels[i]], markersize = 10)

    # Array of distances from each house to its corresponding centroid
    distances_from_centroid = [[] for i in range(number_clusters)]
    # index to step through the different centroids corresponding to different clusters
    index = 0
    # iterating through the all clusters of labeled_points and append corresponding distances from centroids to distances_from_centroid
    for clus in labeled_points:
        for point in clus:
            # dist = sqrt((point[1] - centroids[index][1])**2 + (point[0] - centroids[index][0])**2)
            ### NOTE: RELIES ON ORDER OF CENTROIDS TO MATCH THE ORDER OF CLUSTERS (WHICH IS MOST LIKELY THE CASE)
            dist = haversine(point[1], point[0], centroids[index][1], centroids[index][0]) ###IN KM
            dist_ft = dist * 3280.84
            # print(dist)
            # print(dist_ft)
            distances_from_centroid[index].append(dist_ft)
        index += 1
            # print (point)
            # print(len(clus))
            # print(len(labeled_points[0])+len(labeled_points[1])+len(labeled_points[2])+len(labeled_points[3]))

        #LONG AND LAT BACKWARDS ON EXCEL
        # dist = haversine((centroids[labels[i]][1]),centroids[labels[i]][0],l[i][1],X[i][0])
        # dist = math.sqrt((centroids[labels[i]][0]-X[i][0])**2 +(centroids[labels[i]][1]-X[i][1])**2)
        # distances_from_centroid[labels[i]].append(dist)

    # print(labeled_points)
    # print(len(labeled_points))
    # print(type(labeled_points[0]))
    # print(type(centroids))
    # print(type(X))
    # print(distances_from_centroid)

    # below = 0
    # above = 0
    #
    # for x in distances_from_centroid:
    #     maxim = max(x)
    #     if maxim < 500:
    #         below += 1
    #     elif maxim > 500:
    #         above += 1
        # print(maxim)
    # print(min(distances_from_centroid[0]))
    # print(below)
    # print(above)

    # Array of all antennas found in Excel Spreadsheet
    antenna_array = []
    # Populating antenna_array with [Location Code, Address, LAT, LONG]
    for row in range(1, antennas_sheet.nrows):
        array_tuple = [antennas_sheet.cell_value(row, 0), antennas_sheet.cell_value(row, 1), antennas_sheet.cell_value(row, 2), antennas_sheet.cell_value(row, 3)]
        antenna_array.append(array_tuple)

    # print(len(antenna_array))
    # print(len(centroids))

    # Final Array for actual CSV output
    antennas_final = []
    # NOTE: index_antenna relies
    index_antenna = 0
    # Iterating through each centroid in list of centroids corresponding to each cluster
    # Finds the closest Antenna location to match corrresponding centroid
    for cent in centroids:
        temp_best = []
        for point in antenna_array:
            antenna_type = ""
            dist = sqrt((cent[0] - point[2])**2 + (cent[1] - point[3])**2)
            if max(distances_from_centroid[index_antenna]) <= 100:
                antenna_type = "T-1"
            elif max(distances_from_centroid[index_antenna]) <= 200 and max(distances_from_centroid[index_antenna]) > 100:
                antenna_type = "T-2"
            elif max(distances_from_centroid[index_antenna]) <= 300 and max(distances_from_centroid[index_antenna]) > 200:
                antenna_type = "T-3"
            elif max(distances_from_centroid[index_antenna]) <= 400 and max(distances_from_centroid[index_antenna]) > 300:
                antenna_type = "T-4"
            else:
                antenna_type = "T-5"
            if (temp_best == []):
                temp_best = [point, antenna_type]
            elif (dist) < (sqrt((temp_best[0][2] - cent[0])**2 + (temp_best[0][3] - cent[1])**2)):
                temp_best = [point, antenna_type]

        antennas_final.append(temp_best)
        index_antenna += 1

    # print(antennas_final)
    # print(len(antennas_final))

    # Displays a visualization of all clusters and corresponding centroid
    plt.scatter(centroids[:, 0],centroids[:, 1], marker = "x", s=150, linewidths = 5, zorder = 10)
    plt.show()

    # Outputs the AntennaLocationCode, AntennaType of each final antenna to CSV
    with open("output.csv",'w') as resultFile:
        wr = csv.writer(resultFile, delimiter = ",")
        wr.writerow(['AntennaLocationCode', 'AntennaType'])
        for elem in antennas_final:
            wr.writerow([elem[0][0], elem[1]])

#Runs the function
antenna_classify()