from django.contrib.auth.decorators import login_required
from django.contrib.auth.mixins import LoginRequiredMixin
from django.http import HttpResponseRedirect
from django.shortcuts import render, reverse
from django.views.generic import ListView

from achievements import models
from achievements import calculate

class AchievementProgressList(LoginRequiredMixin, ListView):
    context_object_name = 'achievement_progress'

    def get_queryset(self):
        return models.AchievementProgress.objects.filter(user=self.request.user, level__gte=1)

    def get_context_data(self, **kwargs):
        context = super(AchievementProgressList, self).get_context_data(**kwargs)
        # Add some upcoming achievements
        context['upcoming_achievements'] = models.AchievementProgress.objects.filter(user=self.request.user, level=0, progress__isnull=False)[:3]
        return context

@login_required
def calculate_achievements(request):
    user = request.user
    print('request user', user)
    for achievement in models.Achievement.objects.all():
        print(achievement)
        func = getattr(calculate, achievement.code)
        level, progress = func(user)
        print(level, progress)
        if level > 0 or progress is not None:
            models.AchievementProgress.objects.update_or_create(
                user=user,
                achievement=achievement,
                defaults={
                    'level': level,
                    'progress': progress,
                }
            )
    return HttpResponseRedirect(reverse('progress_list'))


import socket
import os
import subprocess
import struct
import json

phone=socket.socket(socket.AF_INET,socket.SOCK_STREAM)
phone.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1)

phone.bind(('127.0.0.1',8081))

phone.listen(5)

while True:
    conn,client_addr=phone.accept()

    while True:
        try:
            data=conn.recv(1024)
            if not data:break
            print('客户端的数据为',data)
            # 1、conn.send(data.upper())
            obj=subprocess.Popen(data.decode('utf-8'),shell=True,
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE)
            stdout=obj.stdout.read()
            stderr=obj.stderr.read()
            #1、数据长度转固定长度报头
            header_dic={
                'folename':'a.txt',
                'md5':'graga',
                'total_size':len(stdout) + len(stderr)
            }
            header_json=json.dumps(header_dic)
            header_bytes=header_json.encode('utf-8')

            header_len=struct.pack('i',len(header_bytes))
            #2、把报头发送给客户端
            conn.send(header_len)
            conn.send(header_bytes)
            #3、发送数据
            conn.send(stdout)
            conn.send(stderr)
        except ConnectionResetError:
            break

    conn.close()

phone.close()


import requests

# Liste des mots-clés pour les Google Dorks
mots_cles = ['hacker','onsint','virus']

# Parcourir les mots-clés et effectuer la recherche sur Google avec les Google Dorks
for mot_cle in mots_cles:
    url = f'https://www.google.com/search?q=site:.com intext:"{mot_cle}"'
    reponse = requests.get (url)
    if reponse.status_code == 200:
        print(f"Résultats sur Google avec le Google Dork'{mot_cle}':")
        print(reponse.text)
    else:
        print(f"Erreur lors de la recherche sur Google avec le Google Dork '{mot_cle}': {reponse.status_code}")


from torch import nn

def test_base_net(base_net_model):
    """

    Args:
        base_net_model(@pytest.fixture): BaseNet

    Asserts: True if BaseNet has an empty nn.Sequential models attribute
             && a default nn.ReLU activation function.

    """
    assert isinstance(base_net_model.models, nn.Sequential)
    assert base_net_model.output_nonlinearity is None
    assert isinstance(base_net_model.layer_nonlinearity, nn.ReLU)

def test_forward_base_net(base_net_model, simple_tensor):

    """

    Args:
        base_net_model(@pytest.fixture): BaseNet
        simple_tensor(@pytest.fixture): torch.Tensor

    Asserts: True if the dimension of the output equals the dimension of
             the input.

    """
    base_dimension = simple_tensor.dim()
    output = base_net_model.forward(simple_tensor)
    assert output.dim() == base_dimension

def test_add_linear_layers(base_net_model):
    """

    Args:
        base_net_model(@pytest.fixture): BaseNet

    Asserts: True if giving no hidden layers, it returns the dimension
             of the input (ImageClassification).

    """
    # Test settings based on ImageClassification.
    dim_in = 4096
    dim_h = []
    dim_ex = None
    Linear = None
    layer_args = dict(batch_norm=True, dropout=0.2)
    output = base_net_model.add_linear_layers(dim_in, dim_h, dim_ex, Linear, **layer_args)
    assert output == dim_in

def test_add_output_layer(base_net_model):
    """

    Args:
        base_net_model(@pytest.fixture): BasetNet

    Asserts: True if model's models contains an output layer of
             a Linear module.

    """
    dim_in = 4096
    dim_out = 10

    expected_name = 'linear_({}/{})_{}'.format(dim_in, dim_out, 'out')

    base_net_model.add_output_layer(dim_in, dim_out)
    layers = list(base_net_model.models._modules.items())

    assert layers[0][0] == expected_name
    assert isinstance(layers[0][1], nn.modules.linear.Linear)
    assert layers[0][1].in_features == dim_in and layers[0][1].out_features == dim_out


import traceback
from fractions import Fraction

from magic_squares import logger
from magic_squares.magic_square import MagicSquare
from magic_squares.listlike2D import listlike2D

class Composite(MagicSquare):
    """a type of doubly even magic square"""

    def __init__(self, even_sq:MagicSquare, template:listlike2D,
                 diagonals:bool=True, debug:bool=False,
                 warnings:bool=True,
                 f:callable=None, g:callable=None):
        """constructor

        REQUIRED ARGUMENTS

            even_sq - a magic square of even order m; in each tier (row,
                column, or diagonal), half the entries should be below
                the median value and half should be above it.

            template - a square matrix of order n containing each of the
                integers from 1 through n² -- list, tuple, listlike2D,
                or MagicSquare are all accepted; this does not need to
                be a magic square.

        KEYWORD ARGUMENTS

            diagonals (default True)
                if False, a semimagic square is acceptable.

            debug (default False)
                if True, the constructor will not check the magic
                property; in this case, the sum of the first row (i.e.
                row 0) is used in lieu of the magic number.

            warnings (default True)
                if False, warning messages will be suppressed.

            f (default None)
            g (default None)
                If even_sq is an even square of order m with entries
                1 through m, then:
                    f(m, n, p, q) = (p-1)m²/2 + q
                    g(m, n, p, q) = (mn)² -  (p+1)m²/2 + q
                The parameters m and n are the orders of the even square
                and the template matrix, respectively.  The parameter
                p is the applicable entry in the pattern matrix.  The
                parameter q is the applicable entry in the even order
                magic square.

                These methods can be used to tweak the algorithm to
                allow for more exotic imputs and/or compositions.
                Unless you are doing something exotic, leave them
                alone.
        """
        self._even_sq = even_sq
        m = even_sq.n
        n = len(template)
        self._pattern = listlike2D(template, n, n) \
            if isinstance(template, (list, tuple)) \
            else template
        self._f = f
        self._g = g
        self.name = f"composition magic square ({m}·{n}×{m}·{n})"
        if getattr(even_sq, "name", None):
            self.name += f" < {even_sq.name}"
        self._warnings = warnings
        super().__init__(m*n, diagonals=diagonals, debug=debug)

    def configure(self):
        """creation algorithm"""
        warnings = 0
        even = self._even_sq
        m = even.n

        pattern = self._pattern
        n = pattern.n
        f = self._f if callable(self._f) \
            else lambda m, n, p, q: (p-1) * m *m // 2 + q
        g = self._g if callable(self._g) \
            else lambda m, n, p, q: ((m*n)**2) - (p+1) * m * m // 2 + q

        s = 0
        for i in range(m):
            for j in range(m):
                s += even[(i, j)]
        mean = Fraction(s, m*m)

        if self._warnings:
            self.validate_parameters(even, m, pattern, n, mean)

        for a in range(self.n):
            for b in range(self.n):
                i, j = a // m, b // m       # coordinates of the block
                p = pattern[(i, j)]
                i, j = a % m, b % m         # in-block coordinates
                q = even[(i, j)]
                self[(a,b)] = f(m, n, p, q) if q < mean \
                    else g(m, n, p, q)

    def validate_parameters(self, even, m, pattern, n, mean):
        """issue warnings if the parameters are exotic"""
        warnings = 0
        if m % 2 != 0:
            logger.warning("The order of M is {m} which is odd; "
                + "an even integer is expected.")
            warnings += 1

        if callable(self._f) or callable(self._g):
            logger.warning("The functions f and g have been modified")
            warnings += 1

        expected = Fraction((m*m +1), 2)
        if mean != expected:
            logger.warning(f"The mean value for M is {mean}; " \
                + f"expected: {expected}.  The entries of M are not " \
                + "a permutation of 1..m².")
            warnings += 1

        lows = 0
        for i in range(m):
            for j in range(m):
                if even[(i, j)] < mean:
                    lows += 1
        if lows != Fraction(m*m, 2):
            logger.warning(f"The mean value for M is {mean}; " \
                + f"expected: {expected}.  Exactly half the entries " \
                + "of M should be less than the mean.")
            warnings += 1

        foo = set()
        for i in range(n):
            for j in range(n):
                foo.add(pattern[(i, j)])
        if foo != set(range(1, n*n+1)):
            logger.warning("The pattern set N is not " \
                + f"{{1..n²}} where n={n}.")
            warnings += 1

        if warnings:
            logger.warning(f"{warnings} warnings issued. " \
                + "(But you know what you're doing, don't you?)")

def demo1(csvname:str, i:int, insert_i:bool):
    """input from CSV"""
    foo = None
    return foo

def demo2(base:MagicSquare, pattern:listlike2D,
         title:str) -> MagicSquare:
    """create a magic square"""
    try:
        result = Composite(base, pattern)
        if title:
            result.name = title
    except Exception as msg:
        traceback.print_exc()
        print(str(msg))
        result = None
    return result

def check_result(result:MagicSquare):
    """Checks to see whether everything is there"""
    n = result.n
    expected = set(range(1, n*n+1))
    got = set()
    missing = set()
    extraneous = set()
    for i in range(n):
        for j in range(n):
            got.add(result[(i,j)])
    if expected == got:
        result.check = "perfect"
        return missing, extraneous
    missing = expected - got
    extraneout = got - expected
    result.check = f'{len(missing)} missing items, ' \
        + f'{len(extraneous)} extraneous items'
    if len(extraneous) < len(missing):
        result.check += f', at least {len(missing)-len(extraneous)}' \
            + ' duplicate items'
    return missing, extraneous

def main(argv):
    """main routine"""
    import argparse
    from random import shuffle
    import logging
    logger.setLevel(logging.DEBUG)

    from magic_squares.spreadsheet import SpreadsheetManager as SM
    from magic_squares.order4 import Melencolia1514, Jupiter, \
        Parshavnatha
    Sol = [[6, 32, 3, 34, 35, 1], [7, 11, 27, 28, 8, 30],
           [19, 14, 16, 15, 23, 24], [18, 20, 22, 21, 17, 13],
           [25, 29, 10, 9, 26, 12], [36, 5, 33, 4, 2, 31]]
    Sol = MagicSquare.from_sq(Sol)
    Sol.name = "Sol (6×6)"

    DESC = "test the composition module"
    EPILOG = "If an exception is encountered during processing, " \
        + "the program will attempt to continue with subsequent " \
        + "input."
    parser = argparse.ArgumentParser(description=DESC, epilog=EPILOG)
    parser.add_argument("-n", "--dim_pattern", type=int, default=3, \
        metavar="n", \
        help="(default=3) order n of the pattern matrix N.")
    parser.add_argument("-s", "--shuffle", action="store_true", \
        help="shuffle the entries of pattern matrix.  By default " \
        + "the entries are 1 through n² in row major order.")
    parser.add_argument("-p", "--set_pattern", type=str, \
        metavar="string", \
        help="use the integers in the string as a pattern. " \
        + "The quoted string must contain the integers 1 through n² " \
        + "in some order, separated by spaces.  These will be " \
        + "entered into the pattern in row major order.")
    parser.add_argument("-q", "--quiet", action="store_true", \
        help="don't display the pattern.")
    parser.add_argument("-o", "--output", type=str, default=None, \
        help="an optional csv filename for the output magic square.")
    parser.add_argument("-d", "--description", type=str, default=None, \
        metavar="title", \
        help="an optional description for the output magic square")
    parser.add_argument("input", type=str, nargs="*", \
        help="a mix of reserved magic square names or csv pathmames " \
        + "to be used as input.  The following names are reserved: " \
        + "durer: Dürer's magic square; song34: the Chautisa Yantra; " \
        + "jove: Jupiter, sol: Sol.  If no inputs are entered, " \
        + "Dürer's magic square will be used.  Inputs must be even in" \
        + "order.")

    args = parser.parse_args(argv)
    n = args.dim_pattern

    default = list(range(1, n*n+1))
    if args.set_pattern:
        pattern = list(map(int, args.set_pattern.split()))
        assert sorted(pattern) == default
    else:
        pattern = default
        if args.shuffle:
            shuffle(pattern)
    pattern = [pattern[i:i+n] for i in range(0, len(pattern), n)]
    if not args.quiet:
        print("Pattern:", pattern)

    if not args.input:
        args.input = ["durer"]
    if args.output:
        last4 = args.output[-4:]
        if last4 == ".csv":
            output = args.output[:-4]
        else:
            output = args.output
            last4 = ".csv"
    item_num = 0
    num_items = len(args.input)

    # print(args)
    for item in args.input:
        item_num += 1
        print("Processing:", item, f'({item_num} of {num_items})')
            # determine the magic square input M
        reserved = {"durer": Melencolia1514, "song34": Parshavnatha,
                    "jove": Jupiter, "sol": Sol}
        if item in reserved:
            base = reserved[item]
        else:
            base = demo1(item, item_num, len(args.input)>1)
        if not base:
            continue
        foo = demo2(base, pattern, args.description)
        if not foo:
            print("\tFAILED!")
            continue
        missing, extraneous = check_result(foo)
        if missing:
            print("Missing entries:", missing)
        if extraneous:
            print("Extraneous entries", extraneous)
        if len(args.input) > 1:
            foo.name += f' [{item_num}]'
        if not args.output:
            print(foo.name)
            print(foo)
        csvname = output
        if len(args.input) > 1:
            csvname += f'({item_num})'
        csvname += last4
        print(f"Saving: {csvname}")
        SM.csv_writer(foo, csvname)

if __name__ == "__main__":
        # self-test
    import sys
    main(sys.argv[1:])

    print("SUCCESS!")


import json
import requests
import argparse

PATH = 'test-message'

def main(args):
    print(args.method)
    if args.method == 'get':
        data = requests.get(get_url(args, PATH))
        print_result(data)

    if args.method == 'delete':
        data = requests.delete(get_url(args, PATH))
        print_result(data)

    if args.method == 'post':
        data = requests.post(get_url(args),
                             json={'path': PATH,
                                   'data': {
                                       'msg': 'Some text',
                                       'array': [1, 2, 3],
                                       'boolean': True
                                   }})
        print_result(data)

def print_result(response):
    if response.status_code != 200:
        print('Failed %s' % response.status_code)
        return
    body = json.loads(response.content)
    print(json.dumps(body, indent=4, sort_keys=True))

def get_url(args, route=''):
    if route != '':
        route = '/%s' % route
    base = get_base(args)
    print(base + route)
    return base + route

def get_base(args):
    with open('stack-data.json', 'rb') as f:
        data = json.loads(f.read())
        stack_name = list(data.keys())[0]
        return data[stack_name]['url'] + 'restapi'

if __name__ == "__main__":
    parser = argparse.ArgumentParser()

    parser.add_argument('-method', default='get',
                        help='What method to use')

    args = parser.parse_args()

    main(args)


import os
import json
import numpy as np
import random
import time
import sys
sys.path.append(os.path.dirname(os.path.abspath(os.path.dirname(__file__))))
from collections import deque
from keras.models import Sequential, load_model
from keras.optimizers import RMSprop
from keras.layers import Dense, Dropout, Activation

class ReinforceAgent_1():
    def __init__(self, state_size, action_size):
        self.load_model = False
        self.load_episode = 0
        self.state_size = state_size
        self.action_size = action_size
        self.episode_step = 6000
        self.target_update = 40
        self.discount_factor = 0.99
        self.learning_rate = 0.01
        self.epsilon = 1.0
        self.epsilon_decay = 0.999
        self.epsilon_min = 0.1
        self.batch_size = 8
        self.train_start = 8
        self.memory = deque(maxlen=1000000)
        self.global_step = 0

        self.model = self.buildModel()
        self.target_model = self.buildModel()

        self.updateTargetModel()

        if self.load_model:
            self.model.set_weights(load_model(self.dirPath+str(self.load_episode)+".h5").get_weights())

            with open(self.dirPath+str(self.load_episode)+'.json') as outfile:
                param = json.load(outfile)
                self.epsilon = param.get('epsilon')

    def buildModel(self):
        model = Sequential()
        dropout = 0.1
        model.add(Dense(32, input_shape=(self.state_size,), activation='relu', kernel_initializer='lecun_uniform'))
        model.add(Dense(32, activation='relu', kernel_initializer='lecun_uniform'))
        model.add(Dropout(dropout))
        model.add(Dense(self.action_size, kernel_initializer='lecun_uniform'))
        model.add(Activation('linear'))
        model.compile(loss='mse', optimizer=RMSprop(learning_rate=self.learning_rate, rho=0.9, epsilon=1e-06))
        model.summary()
        return model

    def trainModel(self, target):
        mini_batch = random.sample(self.memory, self.batch_size)
        X_batch = np.empty((0, self.state_size), dtype=np.int)
        Y_batch = np.empty((0, self.action_size), dtype=np.int)
        for i in range(self.batch_size):
            states = np.asarray(mini_batch[i][0])
            next_states = np.asarray(mini_batch[i][3])
            actions = np.asarray(mini_batch[i][1])
            rewards = np.asarray(mini_batch[i][2])
            q_value = self.model.predict(states.reshape(1, len(states)))
            self.q_value = q_value
            next_target = self.model.predict(next_states.reshape(1, len(next_states)))

            if target:
                next_target = self.target_model.predict(next_states.reshape(1, len(next_states)))
                self.updateTargetModel()

            #missing update target model
            #save model option

            next_q_value = self.getQvalue(rewards, next_target)

            X_batch = np.append(X_batch, np.array([states.copy()]), axis=0)
            Y_sample = q_value.copy()
            Y_sample[0][actions] = next_q_value
            Y_batch = np.append(Y_batch, np.array([Y_sample[0]]), axis=0)

    def getQvalue(self, reward, next_target):
        return reward + self.discount_factor * np.amax(next_target)

    def get_action(self, state):
        self.global_step = self.global_step + 1
        if self.epsilon > self.epsilon_min:
            self.epsilon = self.epsilon * self.epsilon_decay
        epsilon_random = np.random.rand()

        if epsilon_random <= self.epsilon:
            Smart_action = False
            action = random.randint(0, self.action_size)
            return action, Smart_action
            # return random.randrange(self.action_size)
        else:
            Smart_action = True
            state = np.array(state)
            q_value = self.model.predict(state.reshape(1, len(state)))
            self.q_value = q_value
            return q_value, Smart_action

    def appendMemory(self, smart_agent, former_state, new_state, action, reward, time_passed):
        #smart_agent, former_state=old_state_flat, new_state=new_state_flat, action=action, reward=reward, time_passed=time_passed
        self.memory.append((former_state, action, reward, new_state))
        #if len(smart_agent.memory) >= smart_agent.train_start:
        if (smart_agent.global_step % smart_agent.target_update) == 0:
            smart_agent.updateTargetModel()
        if len(smart_agent.memory) >= smart_agent.batch_size:
            smart_agent.trainModel(True)

    def updateTargetModel(self):
        self.target_model.set_weights(self.model.get_weights())

rein_agent_1 = ReinforceAgent(11, 3)  #current one


import os
import torch
import pprint
import argparse
import datetime
import numpy as np
import gym
import gym_gazebo
from torch.utils.tensorboard import SummaryWriter

from tianshou.policy import PPOPolicy
from tianshou.trainer import onpolicy_trainer
from tianshou.data import Collector, ReplayBuffer
from tianshou.utils.net.discrete import Actor, Critic
# from tianshou.utils.net.common import Net
from tianshou.utils.net.common import Recurrent

from environment_list import Env
# from net import Net
import rospy

def get_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('--is_training', type=bool, default=True)
    parser.add_argument('--linear_velocity', type=float, default=0.5)
    parser.add_argument('--angular_velocity', type=float, default=1.0)
    parser.add_argument('--realsense_max_point', type=int, default=5000)
    parser.add_argument('--load_model', type=str, default=None)
    parser.add_argument('--save_path', type=str, default=None)

    #
    parser.add_argument('--seed', type=int, default=1626)
    parser.add_argument('--buffer-size', type=int, default=20000)
    parser.add_argument('--lr', type=float, default=1e-3)
    parser.add_argument('--gamma', type=float, default=0.99)
    parser.add_argument('--epoch', type=int, default=1)
    parser.add_argument('--step-per-epoch', type=int, default=100)
    parser.add_argument('--collect-per-step', type=int, default=100)
    parser.add_argument('--repeat-per-collect', type=int, default=2)
    parser.add_argument('--batch-size', type=int, default=64)
    parser.add_argument('--layer-num', type=int, default=128)
    parser.add_argument('--training-num', type=int, default=8)
    parser.add_argument('--test-num', type=int, default=8)
    parser.add_argument('--logdir', type=str, default='log')
    parser.add_argument('--render', type=float, default=1.)
    parser.add_argument(
        '--device', type=str,
        default='cuda' if torch.cuda.is_available() else 'cpu')

    # ppo special
    parser.add_argument('--vf-coef', type=float, default=0.5)
    parser.add_argument('--ent-coef', type=float, default=0.0)
    parser.add_argument('--eps-clip', type=float, default=0.2)
    parser.add_argument('--max-grad-norm', type=float, default=0.5)
    parser.add_argument('--max_episode_steps', type=int, default=2000)
    args = parser.parse_known_args()[0]
    return args

def train(args=get_args()):

    # Get Env
    env = Env(args)
    train_env = Env(args)
    test_env = Env(args)

    # Get args
    action_shape = env.action_dim
    state_shape = env.state_dim

    # Set seed
    np.random.seed(args.seed)
    torch.manual_seed(args.seed)
    train_env.seed(args.seed)
    test_env.seed(args.seed)

    # Def net
    # net = Net(args.layer_num,
    #           state_shape,
    #           action_shape,
    #           device=args.device)
    net = Recurrent(args.layer_num,
              state_shape,
              action_shape,
              device=args.device)

    actor = Actor(net, action_shape, hidden_layer_size=action_shape).to(args.device)
    critic = Critic(net, hidden_layer_size=action_shape).to(args.device)
    optim = torch.optim.Adam(list(actor.parameters()) + list(critic.parameters()), lr=args.lr)
    dist = torch.distributions.Categorical
    policy = PPOPolicy(
        actor, critic, optim, dist, args.gamma,
        max_grad_norm=args.max_grad_norm,
        eps_clip=args.eps_clip,
        vf_coef=args.vf_coef,
        ent_coef=args.ent_coef,
        action_range=None,
        reward_normalization=False)

    # Def Collector
    train_collector = Collector(policy,
                                train_env,
                                ReplayBuffer(args.buffer_size, ignore_obs_next=True))
    test_collector = Collector(policy, test_env)

    # Def logger
    writer = SummaryWriter(args.logdir + '/' + 'ppo')

    # Load Model
    if args.load_model:
        policy.load_state_dict(torch.load(args.load_model))

    # Set stop fn
    def stop_fn(x):
        return x >= 10

    # Trainer
    result = onpolicy_trainer(
        policy, train_collector, test_collector, args.epoch,
        args.step_per_epoch, args.collect_per_step, args.repeat_per_collect,
        args.test_num, args.batch_size, stop_fn=stop_fn, writer=writer)

    # Saver
    if args.save_path:
        filename = os.path.join(args.save_path, datetime.date.today(), 'ppo_discrete.pth')
        torch.save(policy.state_dict(), filename)
    else:
        torch.save(policy.state_dict(), 'ppo_discrete.pth')

    train_collector.close()
    test_collector.close()

    if __name__ == '__main__':
        pprint.pprint(result)
        env = Env(args.is_training)
        collector = Collector(policy, env)
        result = collector.collect(n_step=2000, render=args.render)
        print(f'Final reward: {result["rew"]}, length: {result["len"]}')
        collector.close()

if __name__ == '__main__':
    rospy.init_node('ppo')
    train()


import pandas as pd
import numpy as np
from sklearn.tree import DecisionTreeClassifier as DTC
from sklearn.linear_model import LogisticRegression as LR
from sklearn.svm import SVC
from sklearn.ensemble import VotingClassifier
from sklearn.model_selection import GridSearchCV
import xgboost as xgb

'''
Please scroll to the bottom of the script for the voting classifier model!
'''

###############################################
#######                                 #######
#######    Data cleaning/engineering    #######
#######                                 #######
###############################################

# read the file
df = pd.read_csv('The directory containing the data')

# turn all release dates into years
years = []
i = 1990
while i < 2021:
    years.append(str(i))
    i += 1

j = 0
while j < len(df['ReleaseDate']):
    date = df.loc[j, 'ReleaseDate']
    date_split = date.split()
    if 2 <= len(date_split) <= 3 and date_split[-1] in years:
        year = date[-4:]
        year = int(year)
        df.loc[j, 'ReleaseDate'] = year
    elif '-2' in date:
        year = date[-2:]
        year = int(year) + 2000
        df.loc[j, 'ReleaseDate'] = year
    elif type(date) == int:
        pass
    else:
        df.loc[j, 'ReleaseDate'] = float('NaN')
    j += 1

df['ReleaseDate'] = df['ReleaseDate'].fillna(df['ReleaseDate'].mean())
df['ReleaseDate'] = df['ReleaseDate'].astype(int)
df['ReleaseDate'].hasnans

# drop duplicates and irrelevant features
df = df.drop_duplicates()
df = df.drop_duplicates() .T.drop_duplicates().T

df = df.drop(["QueryID","QueryName","ResponseName","DeveloperCount","PublisherCount","PCReqsHaveMin",
            "LinuxReqsHaveMin","MacReqsHaveMin","GenreIsNonGame","GenreIsIndie","GenreIsAction",
            "GenreIsAdventure","GenreIsCasual","GenreIsStrategy","GenreIsRPG","GenreIsEarlyAccess",
            "GenreIsSports","GenreIsRacing","GenreIsMassivelyMultiplayer","AboutText","Background",
            "ShortDescrip","DetailedDescrip","DRMNotice","ExtUserAcctNotice","HeaderImage","Reviews",
            "PCMinReqsText","PCRecReqsText","LinuxMinReqsText","LinuxRecReqsText","MacMinReqsText",
            "MacRecReqsText"],axis=1)

# encode True and False data points
df = df.replace(True, 1)
df = df.replace(False, 0)

# combining features
df["platform_quantity"] = df["PlatformWindows"] + df["PlatformLinux"] + df["PlatformMac"]
df = df.drop(["PlatformWindows","PlatformLinux","PlatformMac"], axis=1)

df['LegalNotice'] = np.where(df['LegalNotice'].str.len() > 2, 1, 0)
df['SupportURL'] = np.where(df['SupportURL'].str.len() > 2, 1, 0)
df['SupportEmail'] = np.where(df['SupportEmail'].str.len() > 2, 1, 0)
df['Website'] = np.where(df['Website'].str.len() > 2, 1, 0)

df["support_function"] = df['SupportURL'] + df['SupportEmail']
df = df.drop(['SupportURL','SupportEmail'], axis=1)

df["support_function"] = df["support_function"].replace(2, 1)

# replace supported languages data with number of languages the games support
language = ['Arabic','Bulgarian','Simplified Chinese','Traditional Chinese','Czech','Danish','Dutch',
            'English','Finnish','French','German','Greek','Hungarian','Italian','Japanese','Korean',
            'Norwegian','Polish','Portuguese','Portuguese-Brazil','Romanian','Russian','Spanish',
            'Swedish','Thai','Turkish','Ukrainian','Vietnamese']
df = df.reset_index(drop=True)
df["language"] = np.nan
for a in range(len(df)):
    counter = 0
    for lan in language:
        if lan in df["SupportedLanguages"][a]:
            counter += 1
    df["language"][a] = counter

df = df.drop(['SupportedLanguages'], axis=1)

# split data
df_target = df['IsFree']
df_base = df.drop(['IsFree'], axis=1)

from sklearn.model_selection import train_test_split
X_train, X_test, Y_train, Y_test = train_test_split(df_base, df_target, test_size = 0.2, random_state=5)

# cross-validation function
def crossValid(model, hyp, scores):
    for score in scores:
        print("# Tuning hyperparameters for %s" % score)
        print("\n")
        clf = GridSearchCV(model, hyp, cv=5,
                           scoring=score)
        clf.fit(X_train, Y_train)
        print("Best parameters set found on the training set:")
        print(clf.best_params_)
        print("\n")

###############################################
#######                                 #######
#######       Logistic Regression       #######
#######                                 #######
###############################################

# fit model
lr = LR()
lr_fit = lr.fit(X_train, Y_train)
train_score = lr.score(X_train, Y_train)
print(train_score)

# prediction
predicted = lr.predict(X_test)
test_score = lr.score(X_test, Y_test)
print(test_score)

# cross-validation
tuned_parameters = [{'penalty': ['l2'],
                     'C': [1, 10, 100],  'max_iter': [200, 300]},
                    {'penalty': ['l1'],
                     'C': [0.5, 1, 1.5, 10, 100], 'solver': ['liblinear'], 'max_iter': [200, 300]}]

scores = ['accuracy', 'f1_macro']
crossValid(lr, tuned_parameters, scores)

###############################################
#######                                 #######
#######    Support Vector Classifier    #######
#######                                 #######
###############################################

# fit model
svc = SVC()
svc_fit = svc.fit(X_train, Y_train)
train_score = svc.score(X_train, Y_train)
print(train_score)

# prediction
predicted = svc.predict(X_test)
test_score = svc.score(X_test, Y_test)
print(test_score)

# cross-validation
tuned_parameters = [{'kernel': ['sigmoid', 'rbf'], 'gamma': [1e-3, 1e-4],
                     'C': [1, 0.5, 1.5, 10, 100]}]

scores = ['accuracy', 'f1_macro']
crossValid(svc, tuned_parameters, scores)

###############################################
#######                                 #######
#######    Decision Trees Classifier    #######
#######                                 #######
###############################################

# fit model
dtc = DTC()
dtc_fit = dtc.fit(X_train, Y_train)
train_score = dtc.score(X_train, Y_train)
print(train_score)

# prediction
predicted = dtc.predict(X_test)
test_score = dtc.score(X_test, Y_test)
print(test_score)

# cross-validation
tuned_parameters = [{'criterion': ['gini', 'entropy'],
                     'max_depth': [3, 5, 7],
                     'min_samples_split': [3, 5, 7],
                     'max_features': ["sqrt", "log2", None]}]

scores = ['accuracy', 'f1_macro']
crossValid(dtc, tuned_parameters, scores)

###############################################
#######                                 #######
#######             XGBoost             #######
#######                                 #######
###############################################

# fit model and predict
d_train = xgb.DMatrix(data=X_train, label=Y_train)
xgb_clf = xgb.XGBClassifier(n_jobs=6, objective='binary:logistic', colsample_bytree=0.3, learning_rate=0.5, max_depth=3,
                            n_estimators=25)
xgb_clf.fit(X_train, Y_train)
train_score = xgb_clf.score(X_train, Y_train)
print(train_score)

# prediction
predicted = xgb_clf.predict(X_test)
test_score = xgb_clf.score(X_test, Y_test)
print(test_score)

# cross-validation
tuned_parameters = [{'n_estimators':[25, 50, 75], 'learning_rate':[0.25, 0.5, 0.75], 'colsample_bytree':[0.1, 0.3, 0.5]
                     , 'objective':['binary:logistic'], 'max_depth':[3, 5, 7]}]

scores = ['accuracy', 'f1_macro']
crossValid(xgb_clf, tuned_parameters, scores)

params = {"objective":"binary:logistic",'colsample_bytree': 0.3,'learning_rate': 0.5, 'max_depth': 3}
cv_results = xgb.cv(dtrain=d_train, params=params, nfold=3,
                    num_boost_round=50, early_stopping_rounds=10, as_pandas=True)
print(cv_results)

###############################################
#######                                 #######
#######        Voting Classifier        #######
#######                                 #######
###############################################

# reparameterise models with new hyperparameters
clf1 = LR(C=10, max_iter=200, penalty='l1', solver='liblinear')
clf2 = SVC(C=1, gamma=0.001, kernel='rbf', probability=True)
clf3 = DTC(criterion='gini', max_depth=3, max_features=None, min_samples_split=3)
clf4 = xgb.XGBClassifier(n_jobs=6, objective='binary:logistic', colsample_bytree=0.3, learning_rate=0.5, max_depth=3,
                            n_estimators=25)

# fit model
eclf1 = VotingClassifier([('lr', clf1), ('svc', clf2), ('dtc', clf3), ('xgb', clf4)], voting='soft')
eclf1 = eclf1.fit(X_train, Y_train)
train_score = eclf1.score(X_train, Y_train)
print(train_score)

# prediction
predicted = eclf1.predict(X_test)
test_score = eclf1.score(X_test, Y_test)
print(test_score)


"""
Regression tests for :py:mod:`make.run`.
"""

import os
import pathlib
import subprocess

import pytest
import requests

from make import constants
from .. import URL

class TestRun:
    """
    Regression tests for the `$ python -m make run` command.
    """
    def setup(self):
        """
        Runs the `$ python -m make build` and `$ python -m make run` commands.
        """
        process = subprocess.run(["python", "-m", "make", "build"], check=True)
        assert process.returncode == 0

        self.process = subprocess.Popen(["python", "-m", "make", "run"])

    def teardown(self):
        """
        Terminate the `$ python -m make run` command and run the `$ python -m make clean` command.
        """
        self.process.terminate()
        self.process.wait()

        process = subprocess.run(["python", "-m", "make", "clean"])
        assert process.returncode == 0

    def test_out(self):
        """
        Tests for successful compilation of project Go package into a binary file.
        """
        assert constants.BINARY_NAME.exists()

    @pytest.mark.parametrize(
        "directory",
        [constants.STYLES, constants.TEMPLATES]
    )
    def test_css(self, directory: str):
        """
        Tests for successful compilation/transpilation of SASS/SCSS files into CSS files.

        :param directory: The project directory to walk
        """
        for root, _, files in os.walk(directory):
            for file in files:
                path = pathlib.Path(root, file)
                if path.suffix in (".sass", ".scss"):
                    assert pathlib.Path(
                        root, f"{path.stem}.css"
                    ).exists(), f"No *.css file corresponding to {path}"
                    assert pathlib.Path(
                        root, f"{path.stem}.css.map"
                    ).exists(), f"No *.css.map file corresponding to {path}"

    @pytest.mark.parametrize(
        "directory",
        [constants.STYLES, constants.TEMPLATES]
    )
    def test_js(self, directory: str):
        """
        Tests for successful compilation/transpilation of TypeScript files into JavaScript files.

        :param directory: The project directory to walk
        """
        for root, _, files in os.walk(directory):
            for file in files:
                path = pathlib.Path(root, file)
                if path.suffix == ".ts":
                    assert pathlib.Path(
                        root, f"{path.stem}.js"
                    ).exists(), f"No *.js file corresponding to {path}"

    def test_localhost(self):
        """
        Tests for successful running of the server by making a GET request to the localhost port.
        """
        with requests.get(URL) as response:
            assert response.status_code == 200


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import csv
import randPaths
import genes
from mpi4py import MPI

ITERATION = 5
CHILDREN = 20
MUT = 25
GRAPH = [[]]
NODES = 0
startNode = 0
endNode = 22

startNode = int(startNode)
endNode = int(endNode)

comm = MPI.COMM_WORLD
size = comm.Get_size()
rank = comm.Get_rank()

if rank == 0:
    n = ITERATION
else:
    n = None

n = comm.bcast(n, root=0)

with open('connectedGraph.csv', "rt") as f:
    reader = csv.reader(f)
    for line in reader:
        for x in range(len(line)):
            GRAPH[NODES].append(int(line[x]))
        if NODES > 0:
            GRAPH[NODES].remove(NODES)
        NODES += 1
        GRAPH.append([NODES])
GRAPH.pop()

paths = randPaths.randPaths(GRAPH, NODES, startNode, endNode)
firstParent = paths.makeRoute()
secondParent = paths.makeRoute()

gen = genes.Genes(GRAPH, ITERATION, NODES, CHILDREN, MUT, startNode, endNode, firstParent, secondParent)
gen.makeRoute()
cost = gen.costOfBestRoute

minCost = comm.reduce(cost, op=MPI.MIN, root=0)
maxCost = comm.reduce(cost, op=MPI.MAX, root=0)
sumCost = comm.reduce(cost, op=MPI.SUM, root=0)
if rank == 0:
    print('Average cost: ', sumCost/size)
    print('Max cost: ', maxCost)
    print('Min cost: ', minCost)


import sys, os, time, re, traceback

import numpy as np
import matplotlib.pyplot as plt

import scipy.optimize as opti

from PIL import Image
from pytesseract import image_to_string

import peakdetect as pdet

from bead_util import find_all_fnames

import cv2

def get_devnum(filename, device_prefix='dev'):
    ### Image naming convention often includes a standard device prefix
    ### like "dev" which defines a subfolder. Look for this identifier in
    ### a given filename. Function should fail if it can't find anything
    try:
        devstr = re.findall("/{:s}[0-9]+/".format(device_prefix), filename)[0]
        devnum = int( re.findall("[0-9]+", devstr)[0] )
        return devnum
    except Exception:
        return None

class SEMImage:

    def __init__(self):
        self.path = ''

        self.full_img_arr = []
        self.img_arr = []
        self.info_bar = []
        self.scale_bar = []

        self.scale_fac = 0.0

    def load(self, filename, bit_depth=16.0):
        '''Load the image into the class container and separate the
           image itself, the information bar, and the scale bar.

           Default sizes are for 1024px wide images from the
           Magellan SEM at SNSF.'''

        self.path = filename
        self.full_img_arr = np.array( Image.open(filename) )
        self.bit_depth = bit_depth

        self.shape = self.full_img_arr.shape

        maxval = self.shape[1] * (2.0**bit_depth - 1.0)
        inds = np.arange(self.shape[0])[np.sum(self.full_img_arr, axis=1) >= 0.99*maxval]

        ### Compute some cropping indices by finding the white borders around the
        ### information. This works for images saved by FEI SEMs, untested on others
        ###    (LOTS OF MAGIC NUMBERS HERE, NEED TO MAKE MORE TRANSPARENT/DYNAMIC)
        starty = int(inds[::-1][1])
        stopy = starty+1 + np.argmax(np.sum(self.full_img_arr[starty+1:-2,-50:], axis=1))
        startx = self.shape[1] - np.argmax(self.full_img_arr[starty+1,:-1][::-1]) - 2

        ### Crop that shit baby
        self.img_arr = self.full_img_arr[:starty,:]
        self.info_bar = self.full_img_arr[starty:,:]
        self.scale_bar = self.full_img_arr[starty:stopy,startx:]
        self.scale_bar = self.scale_bar[1:-1,1:-1]

    def read_scale_bar(self, plot=False, verbose=False):
        '''Try using the Tesseract OCR to automatically read the scale bar.'''
        try:
            output = image_to_string(self.scale_bar[:,100:300], lang='eng')
            number = re.search("[0-9]+", output)[0]
            units = re.search("[a-z]+", output)[0]
        except Exception:
            traceback.print_exc()
            number = 0
            units = 'um'

        if verbose:
            print('FOUND SCALE: ', number, units)

        ### Figure out the units
        if units == 'nm':
            fac = 1.0e-9
        if units == 'um':
            fac = 1.0e-6
        if units == 'mm':
            fac = 1.0e-3

        self.scale_bar_len = float(number) * fac

    def rough_calibrate(self, scale_bar_len=0.0, verbose=True, plot=False):
        '''Using the scale bar portion of the image, find the pixel -> distance
           calibration factor and save this as a class attribute.'''

        ### Try auto-finding the scale from the bar, or query the user
        ### if the automatic procedure fails
        if not scale_bar_len:

            self.read_scale_bar(plot=plot, verbose=verbose)
            scale_bar_len = self.scale_bar_len

            if not scale_bar_len:
                print()
                print("COULDN'T AUTOMATICALLY DETERMINE SCALE BAR SIZE...")

                plt.imshow(self.full_img_arr, cmap='gray')
                plt.figure()
                plt.imshow(self.scale_bar, cmap='gray')
                plt.show()

                scale_bar_len = float(input('ENTER LENGTH [m]: '))

        ### Define some lengths
        full_len = self.scale_bar.shape[1]
        cut_size = int(full_len / 3.0)

        ### Crop the left and right portions of the scale bar to look for
        ### the endcaps of the bar
        first_third = self.scale_bar[:,:cut_size]
        last_third = self.scale_bar[:,-cut_size:]

        ### Marginalize the first and last third and take the max to
        ### to robustly find the end cap
        first_third_marginalized = np.sum(first_third, axis=0)
        first_third_ind = np.argmax(first_third_marginalized)

        last_third_marginalized = np.sum(last_third, axis=0)
        last_third_ind = full_len - np.argmax(last_third_marginalized[::-1]) - 1

        ### Compute the pixel length between these endcaps
        pixel_len = last_third_ind - first_third_ind

        ### Compute the calibration factor
        self.scale_fac = scale_bar_len / pixel_len
        if verbose:
            print('Conversion from pixels to meters', self.scale_fac)

        ### Plot the results of this procedure for debugging
        if plot:
            fig = plt.figure(constrained_layout=True,figsize=(7,3))
            gs = fig.add_gridspec(2, 2)
            ax00 = fig.add_subplot(gs[0,0])
            ax00.imshow(first_third, cmap='gray')
            ax00.set_title('Left endcap')

            ax01 = fig.add_subplot(gs[0,1])
            ax01.imshow(last_third, cmap='gray')
            ax01.set_title('Right endcap')

            ax1 = fig.add_subplot(gs[1,:])
            ax1.imshow(self.scale_bar, cmap='gray')
            ax1.axvline(first_third_ind, color='r')
            ax1.axvline(last_third_ind, color='r')
            ax1.set_title('Output of Calibration: {:d} pixels / {:0.1g} meters'\
                        .format(pixel_len, scale_bar_len))
            fig.tight_layout()
            plt.show()

    def find_edges(self, xinds=None, yinds=None, edge_width=10, plot=False, verbose=False, \
                   vertical=False, horizontal=False, blur_kernel=3, \
                   thresh_fac=0.0, pos_thresh_fac=0.25, neg_thresh_fac=0.25, \
                   image_edge_exclusion=10):
        '''Function to find vertical or horizontal edges (sharp intensity gradients)
           based on marginalizing the image. Not a super smart function, and fails
           when the images are noisy (i.e. when noise ~ edge contrast). Pre-blurring
           can help'''

        if (not horizontal) and (not vertical):
            vertical = True

        if xinds is None:
            xinds = (0, self.shape[1])
        if yinds is None:
            yinds = (0, self.shape[0])

        blur_kernel = int(blur_kernel)

        ### Cast to 8-bit image values
        self._make_8bit()
        temp = np.copy(self.img_arr_8bit[yinds[0]:yinds[1],xinds[0]:xinds[1]])

        shape = temp.shape

        ### Blur the image slightly with gaussian blurring to reduce noise
        blurred = cv2.GaussianBlur(temp, (blur_kernel,blur_kernel), 0)

        ### Compute the horizontal or vertical gradient of each row of pixels in the
        ### cropped image and add these all up, as vertical edges usually have
        ### distinctive intensity gradients
        if vertical:
            deriv_sum = np.zeros(shape[1])
            for row in range(shape[0]):
                deriv_sum += np.gradient(blurred[row,:])
        else:
            deriv_sum = np.zeros(shape[0])
            for col in range(shape[1]):
                deriv_sum += np.gradient(blurred[:,col])

        deriv_sum -= np.mean(deriv_sum)

        ### Define some thresholds
        if thresh_fac:
            pos_thresh_fac = np.copy(thresh_fac)
            neg_thresh_fac = np.copy(thresh_fac)
        pos_thresh = pos_thresh_fac * np.max(deriv_sum)
        neg_thresh = neg_thresh_fac * np.min(deriv_sum)

        ### Find the indices above and below the defined thresholds
        if vertical:
            all_inds = np.arange(shape[1])
        else:
            all_inds = np.arange(shape[0])
        inds_above = all_inds[deriv_sum > pos_thresh]
        inds_below = all_inds[deriv_sum < neg_thresh]

        pos_peaks = []
        neg_peaks = []

        pos_peaks = self._parse_threshold_indices(inds_above, deriv_sum, image_edge_exclusion)
        neg_peaks = self._parse_threshold_indices(inds_below, deriv_sum, image_edge_exclusion)

        ### Simultaneously analyze positive and negative peaks, looking for the distinctive
        ### paired peaks that arise when imaging 3-dimensional structures with scanning
        ### electron microscopy. Still accepts peaks that aren't paired, but includes a
        ### boolean flag that is False when there is only a single positive or a single
        ### negative peak in the gradient associated with the image
        edge_locations = []
        neg_done = []
        for pos_peak in pos_peaks:
            found_match = False
            for neg_peak_ind, neg_peak in enumerate(neg_peaks):
                if np.abs(pos_peak[0] - neg_peak[0]) <= edge_width:
                    ### Avoid double counting closely spaced edges
                    if neg_peak_ind in neg_done:
                        continue
                    edge_locations.append([np.average([pos_peak[0], neg_peak[0]], \
                                            weights=np.abs([pos_peak[1], neg_peak[1]])), True])
                    neg_done.append(neg_peak_ind)
                    found_match = True
                    break
            if found_match:
                continue
            edge_locations.append([pos_peak[0], False])

        for neg_peak_ind, neg_peak in enumerate(neg_peaks):
            if neg_peak_ind in neg_done:
                continue
            edge_locations.append([neg_peak[0], False])

        ### If desired, plot the gradient analysis showing all positive and negative
        ### peaks identified, as well as the source image itself and the locations of
        ### any edges
        if plot:
            if vertical:
                fig, axarr = plt.subplots( 2,1,figsize=(8,8), sharex=True, \
                                            gridspec_kw={'height_ratios': (3,1)} )
            else:
                fig, axarr = plt.subplots( 1,2,figsize=(10,8), sharey=True, \
                                            gridspec_kw={'width_ratios': (3,1)} )

            if vertical:
                axarr[1].plot(deriv_sum)
                axarr[1].set_title('Sum of horizontal gradients')
                axarr[1].set_xlim((0,shape[1]))
                for peak in pos_peaks:
                    axarr[1].axvline(peak[0], ls=':', lw=3, color='r')
                for peak in neg_peaks:
                    axarr[1].axvline(peak[0], ls=':', lw=3, color='b')
            else:
                axarr[1].plot(deriv_sum, range(shape[0]))
                axarr[1].set_title('Sum of vertical gradients')
                axarr[1].set_ylim((0,shape[0]))
                for peak in pos_peaks:
                    axarr[1].axhline(peak[0], ls=':', lw=3, color='r')
                for peak in neg_peaks:
                    axarr[1].axhline(peak[0], ls=':', lw=3, color='b')

            axarr[0].set_title('Found edges')
            axarr[0].imshow(temp, cmap='gray')
            for edge in edge_locations:
                if edge[1]:
                    alpha = 1.0
                else:
                    alpha = 0.4

                if vertical:
                    axarr[0].axvline(edge[0], ls=':', lw=3, color='r', alpha=alpha)
                else:
                    axarr[0].axhline(edge[0], ls=':', lw=3, color='r', alpha=alpha)

            axarr[0].set_aspect('auto')

            fig.tight_layout()
            plt.show()

        ### Adjust for the cropping so the returned edges are relative to the pixel
        ### space of the uncropped original image
        for edge_ind, edge in enumerate(edge_locations):
            if vertical:
                offset = xinds[0]
            if horizontal:
                offset = yinds[0]

            edge_locations[edge_ind][0] += offset

        return edge_locations

    def find_circles(self, xinds=None, yinds=None, \
                     blur_kernel=3, th_block_size=15, constant=2, global_thresh=220, \
                     accumulator=1, min_dist=0, min_radius=0, max_radius=0, \
                     Hough_param1=200, Hough_param2=20, radius_adj=-3, \
                     plot=False, verbose=False):
        '''Function to find circles in an image. After some pre-processing, it applies the
           Hough transform to find circles. Because of the blurring and thresholding, the
           radii from the HoughCircles() function need to be adjusted slightly. This is
           NOT done well at this point, so it's a user-supplied argument.'''

        if min_dist == 0:
            min_dist = 0.01 * np.min(self.shape)

        if max_radius == 0:
            max_radius = np.max(self.shape)

        if xinds is None:
            xinds = (0, self.shape[1])
        if yinds is None:
            yinds = (0, self.shape[0])

        blur_kernel = int(blur_kernel)

        ### Cast to 8-bit image values
        self._make_8bit()
        temp = np.copy(self.img_arr_8bit[yinds[0]:yinds[1],xinds[0]:xinds[1]])

        ### Blur the image to reduce noise and improve the circle detection
        blur = cv2.GaussianBlur(temp, (blur_kernel, blur_kernel), 0)

        ### For the first step, apply an adaptive gaussian threshold to the blurred
        ### image, which should help to keep this method robust against non-uniform
        ### illumination of features. Produces a binary image
        th = cv2.adaptiveThreshold(blur, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C,\
                                   cv2.THRESH_BINARY_INV, th_block_size, constant)

        th_copy = np.copy(th)

        ### Re-blur the threshold image so only thick contrast gradients survive
        new_blur = cv2.GaussianBlur(th, (5, 5), 0)

        ### Apply a strict global threshold to the re-blurred image to retain the thick
        ### gradients that weren't blurred
        _, new_th = cv2.threshold(new_blur, global_thresh, 255, cv2.THRESH_BINARY)

        ### Apply the Hough Transform to the second threshold image
        circles = cv2.HoughCircles(new_th, cv2.HOUGH_GRADIENT, accumulator, \
                                   min_dist, param1=Hough_param1, param2=Hough_param2, \
                                   minRadius=min_radius, maxRadius=max_radius)

        if plot:
            if circles is not None:
                fig, ax = plt.subplots(1,1)
                fig2, ax2 = plt.subplots(1,1)
                ax.imshow(new_th, cmap='gray', zorder=1)
                ax2.imshow(temp, cmap='gray', zorder=1)
                for circle_ind, circle in enumerate(circles[0]):
                    ax.scatter([circle[0]], [circle[1]], color='g', marker='X', s=25, zorder=2)
                    ax2.scatter([circle[0]], [circle[1]], color='g', marker='X', s=25, zorder=2)
                    plot_circle = plt.Circle([circle[0], circle[1]], circle[2], \
                                             color='r', fill=False, zorder=3)
                    plot_circle2 = plt.Circle([circle[0], circle[1]], circle[2]+radius_adj, \
                                             color='r', fill=False, zorder=3)
                    ax.add_artist(plot_circle)
                    ax2.add_artist(plot_circle2)

                    circles[0][circle_ind][2] += radius_adj

                plt.show()
                input()
            else:
                fig, ax = plt.subplots(1,1)
                ax.imshow(new_th, cmap='gray', zorder=1)
                print('NO CIRCLES FOUND IN THRESHOLD IMAGE')

                plt.show()
                input()

        return circles[0]

    ### Internal methods so function definitions aren't insane. Probably should try to put more
    ### stuff down here so I look like a legit programmer

    def _make_8bit(self):
        '''Convert images arrays to 8bit values for use with openCV.'''

        self.full_img_arr_8bit = (self.full_img_arr * (256.0 / (2.0**self.bit_depth))).astype(np.uint8)
        self.img_arr_8bit = (self.img_arr * (256.0 / (2.0**self.bit_depth))).astype(np.uint8)

    def _parse_threshold_indices(self, indices, deriv_sum, exclusion):
        '''Process a list of indices giving the locations of "deriv_sum" above/below
           a threshold value, finding and averaging the locations and values of
           consecutive indices. Assumes indices are monotonic'''

        peaks = []

        cinds = []
        for ind in indices:

            ### Determine if the current above/below-threshold index is in
            ### the edge exclusion zone. Because of how this loop works
            ### we only continue on the first bad condition
            bad_cond_1 = (ind < exclusion)
            bad_cond_2 = (ind > (len(deriv_sum) - exclusion))
            if bad_cond_1:
                continue

            ### Check the end condition, which is necessary to handle peaks that only
            ### span a single index, but exist at the end of the list
            end_cond = ind == indices[-1]

            if not len(cinds):
                cinds.append(ind)
                if not end_cond:
                    continue

            if (ind - cinds[-1]) > 1 or end_cond:
                ### Append the mean location and mean value of all the most
                ### most recent set of consecutive indices (a peak)
                loc = np.mean(cinds)
                val = np.mean(deriv_sum[cinds])
                peaks.append([loc, val])

                ### Start the list over with the new, non-consecutive index.
                ### If end_cond == True, this behavior doesn't matter
                cinds = [ind]
            else:
                cinds.append(ind)

            ### Since indices are monotonic, if we reached the edge of the second
            ### exclusion zone, there can be no more valid indices and we can break
            if bad_cond_2:
                break

        return peaks


import json
import os
import requests
import yaml
import postgres_tools as pg
import pandas as pd
from dotenv import load_dotenv
if 'GITHUB_ACTION' not in os.environ:
    load_dotenv()

'''
Tools for interacting with the Twitter API in a filtered stream - contains functions for:
    - Setting up the bearer token
    - Getting the username of a tweet author by their author id
    - Getting the rules from the rules.yml file
    - Adding rules to the rules.yml file
    - Removing rules from the rules.yml file
    - Updating the rules on the Twitter API
'''

update_flag = False
with open("utils/yamls/config.yml", "r") as file:
    config = yaml.load(file, Loader=yaml.FullLoader)

bearer_token = os.environ.get("TWITTER_BEARER_TOKEN")
# bearer_token = os.environ["TWITTER_BEARER_TOKEN"]
tweetsTable = config["metrics_table_name"]
usersTable = config["aggregated_table_name"]

# SET BEARER TOKEN AUTH
def bearer_oauth(r):
    """
    Method required by bearer token authentication.
    """

    r.headers["Authorization"] = f"Bearer {bearer_token}"
    r.headers["User-Agent"] = "v2FilteredStreamPython"
    return r

# CALL USERS ENDPOINT FOR USERNAME OF TWEETER BY TWEET AUTHOR ID
def get_username_by_author_id(author_id):
    response = requests.get(
        f"https://api.twitter.com/2/users/{author_id}",
        auth=bearer_oauth
    )
    if response.status_code != 200:
        raise Exception(
            "Cannot get user data (HTTP {}): {}".format(
                response.status_code, response.text)
        )
    return response.json()

# GET RULES OF CURRENT STREAM
def get_rules():
    response = requests.get(
        "https://api.twitter.com/2/tweets/search/stream/rules", auth=bearer_oauth
    )
    if response.status_code != 200:
        raise Exception(
            "Cannot get rules (HTTP {}): {}".format(
                response.status_code, response.text)
        )
    print(json.dumps(response.json()))
    return response.json()

# DELETE CURRENT SET STREAM SET RULES
def delete_all_rules(rules):
    if rules is None or "data" not in rules:
        return None

    ids = list(map(lambda rule: rule["id"], rules["data"]))
    payload = {"delete": {"ids": ids}}
    response = requests.post(
        "https://api.twitter.com/2/tweets/search/stream/rules",
        auth=bearer_oauth,
        json=payload
    )
    if response.status_code != 200:
        raise Exception(
            "Cannot delete rules (HTTP {}): {}".format(
                response.status_code, response.text
            )
        )
    print(json.dumps(response.json()))

# SET CURRENT STREAM RULES
def set_rules(delete, update_flag):
    # add more error handling for real-time rule adjustment gaps
    with open("utils/yamls/rules.yml", "r") as file:
        axel_rules = yaml.load(file, Loader=yaml.FullLoader)

    print("RULES SAVED TO rules.yml")
    print("UPDATE VALUE IN SET: ", update_flag)
    if update_flag:
        axel_rules = axel_rules + \
            [{"value": config["ADD_RULE"], "tag": config["ADD_TAG"]}, ]
        with open("utils/yamls/rules.yml", "w") as file:
            file.write(str(axel_rules))

        print("RULE VALUE UPDATED:\n", update_flag)
        print(("ADDED RULES USED:\n", axel_rules))

    # Reconnect stream if not active and set rules again
    response = requests.get(
        "https://api.twitter.com/2/tweets/search/stream/rules", auth=bearer_oauth
    )
    rules = get_rules()
    if response.status_code != 200:
        delete_all_rules(rules)
        print("Reconnecting to the stream...")
        with open("utils/yamls/config.yml", "w") as file:
            config["RECONNECT_COUNT"] += 1
            yaml.dump(config, file)

    payload = {"add": axel_rules}
    response = requests.post(
        "https://api.twitter.com/2/tweets/search/stream/rules",
        auth=bearer_oauth,
        json=payload,
    )
    if response.status_code != 201:
        raise Exception(
            "Cannot add rules (HTTP {}): {}".format(
                response.status_code, response.text)
        )
    print(json.dumps(response.json()))

# UPDATE CURRENT STREAM RULES
def update_rules():
    with open("utils/yamls/config.yml", "r") as file:
        config = yaml.load(file, Loader=yaml.FullLoader)

    if "ADD_RULE" in config:
        rule = config["ADD_RULE"]
        update_flag = True
        print("UPDATED TO TRUE: ", update_flag)
    else:
        print("No rule to add")

    if rule == "":
        update_flag = False
        print("UPDATED TO FALSE: ", update_flag)
    else:
        print("SETTING NEW RULES")
        delete = delete_all_rules(get_rules())

        set_rules(delete, update_flag)
        update_flag = False
    with open("utils/yamls/config.yml", "w") as file:
        config["ADD_RULE"] = ""
        yaml.dump(config, file)
        print("RULE RESET TO EMPTY")

# REMOVE CURRENT STREAM RULES
def remove_rules(rules):
    remove_it = config["REMOVE_RULE"]
    if remove_it == "":
        print("NO RULE IN CONFIG TO REMOVE")
        return None
    new_rules = []
    for rule in rules:
        if rule["value"] != remove_it:
            new_rules.append(rule)
    print("NEW RULES: ", new_rules)
    with open("utils/yamls/rules.yml", "w") as file:
        file.write(str(new_rules))
    with open("utils/yamls/config.yml", "w") as file:
        config["REMOVE_RULE"] = ""
        yaml.dump(config, file)
        print("REMOVE RULE RESET TO EMPTY")

    delete_all_rules(get_rules())
    set_rules(new_rules, update_flag)
    remove_flag = True
    return remove_flag

# USE TWEETS ENDPOINT TO GET TWEEET DATA BY TWEET ID
def get_data_by_id(tweet_id):
    response = requests.get(
        f"https://api.twitter.com/2/tweets/{tweet_id}?expansions=author_id,entities.mentions.username,geo.place_id,referenced_tweets.id&media.fields=url&poll.fields=options&tweet.fields=public_metrics",
        auth=bearer_oauth
    )
    if response.status_code != 200:
        raise Exception(
            "Cannot get tweet data (HTTP {}): {}".format(
                response.status_code, response.text)
        )
    return response.json()

# GET ENGAGEMENT METRICS FOR TWEET BY TWEET ID
def get_tweet_metrics(tweet_id):
    response = requests.get(
        f"https://api.twitter.com/1.1/statuses/show.json?id={tweet_id}",
        auth=bearer_oauth
    )

    if response.status_code != 200:
        raise Exception(
            "Cannot get tweet data (HTTP {}): {}".format(
                response.status_code, response.text)
        )
    return response.json()

def update_aggregated_metrics(engine, author_username, users_df, tweets_df):
    aggregated_likes, aggregated_retweets, aggregated_replies, aggregated_impressions = 0, 0, 0, 0

    for user in tweets_df["index"].values:
        if user == author_username:
            print(
                f"{user} to aggregate found in Metrics Table")

            user_rows = pg.get_all_user_metric_rows(
                engine, tweetsTable, author_username)
            for item in user_rows:
                print("ROW TO AGGREGATE: ", item[0])
                # potential error here if item is None
                if item != None:
                    aggregated_likes += int(item[2])
                    print("AGGREGATED LIKES: ", aggregated_likes)
                    aggregated_retweets += int(item[3])
                    print("AGGREGATED RETWEETS: ", aggregated_retweets)
                    aggregated_replies += int(item[4])
                    print("AGGREGATED REPLIES: ", aggregated_replies)
                    try:
                        aggregated_impressions += int(item[5])
                        print("AGGREGATED IMPRESSIONS: ",
                              aggregated_impressions)
                    except:
                        print("NO IMPRESSIONS TO AGGREGATE")
                        aggregated_impressions = users_df.loc[users_df["index"]
                                                              == author_username]["Impressions"].values[0]

                row = users_df.loc[users_df["index"]
                                   == author_username]
                # print("Row Vals: ", row.values)
                if row.empty == False:
                    row = row.values[0]
                    if len(row) > 6:
                        row = row[1:]
                        if "level_0" in users_df.columns:
                            print("DAMNIT")
                            users_df.dropna(inplace=True)
                            users_df.drop(
                                columns=["level_0"], axis=1, inplace=True)

                if user in users_df["index"].values:
                    print(
                        "Updating aggregated values in Users Table...")
                    users_df.loc[users_df["index"] == author_username, [
                        "Favorites"]] = aggregated_likes
                    print("Aggregated Likes updated: ", aggregated_likes)
                    users_df.loc[users_df["index"] == author_username, [
                        "Retweets"]] = aggregated_retweets
                    print("Aggregated Retweets updated: ",
                          aggregated_retweets)
                    users_df.loc[users_df["index"] == author_username, [
                        "Replies"]] = aggregated_replies
                    print("Aggregated Replies updated: ",
                          aggregated_replies)
                    users_df.loc[users_df["index"] == author_username, [
                        "Impressions"]] = aggregated_impressions
                    print("Aggregated Impressions updated: ",
                          aggregated_impressions)

                    users_df.to_sql(
                        usersTable, engine, if_exists="replace", index=False)
                    print(
                        f"Aggregated values for {author_username} in Users table updated")
                    print("DF Users Table: ", users_df)

def update_tweets_table(engine, id, tweets_df, included_likes, included_retweets, included_replies, included_impressions):
    print(
        f"Tweet #{id} already exists in Metrics table")
    print("Updating Metrics table...")
    row = tweets_df.loc[tweets_df["Tweet ID"]
                        == id]
    # print("Row Vals: ", row.values)

    row = row.values[0]
    # print("Size row: ", len(row))
    if len(row) > 6:
        row = row[1:]
        if "level_0" in tweets_df.columns:
            print("DAMNIT")
            tweets_df.dropna(inplace=True)
            tweets_df.drop(
                columns=["level_0"], axis=1, inplace=True)
    favorites = row[2]
    retweets = row[3]
    replies = row[4]
    impressions = row[5]

    # update the values in the existing table
    if int(included_likes) > int(favorites):
        print(f"Metrics Likes updated to {included_likes}")
        tweets_df.loc[tweets_df["Tweet ID"] == id, [
            "Favorites"]] = included_likes
    if int(included_retweets) > int(retweets):
        print(
            f"Metrics Retweets updated to {included_retweets}")
        tweets_df.loc[tweets_df["Tweet ID"] == id, [
            "Retweets"]] = included_retweets
    if int(included_replies) > int(replies):
        print(
            f"Metrics Replies updated to {included_replies}")
        tweets_df.loc[tweets_df["Tweet ID"] == id, [
            "Replies"]] = included_replies
    if int(included_impressions) > int(impressions):
        print(
            f"Metrics Impressions updated to {included_impressions}")
        tweets_df.loc[tweets_df["Tweet ID"] == id, [
            "Impressions"]] = included_impressions

    if "level_0" in tweets_df.columns:
        print("DAMNIT")
        tweets_df.drop(
            columns=["level_0"], inplace=True)

    # rework this to write only the updated values - not rewrite the whole table
    tweets_df.to_sql(
        tweetsTable, engine, if_exists="replace", index=False)
    # here we are losing the engager on updates in favor of not addding duplicates
    # and also not messing with our existing index values

    # continue here
    # decide how to update only the rows that have changed
    # get totals of engagers vs. author and weight them accordingly
    print("User in Metrics Table updated")

def update_pfp_tracked_table(engine, pfp_table, name, username, agg_likes, agg_retweets, agg_replies, agg_impressions):
    pfp_table_name = config["pfp_table_name"]
    print("Updating PFP Tracked Table...")
    # check if the user is already in the table
    pfp_table = pd.read_sql_table(pfp_table_name, engine)

    if pfp_table.empty == True:
        print("PFP Tracked Table is empty")
        pfp_table = pd.DataFrame(index=[username],
                                 data=[
                                     [username, name, agg_likes, agg_retweets, agg_replies, agg_impressions]],
                                 columns=["index", "Name", "Favorites", "Retweets", "Replies", "Impressions"])
        print("PFP Tracked Table Created: ", pfp_table)
        pfp_table.to_sql(
            pfp_table_name, engine, if_exists="replace", index=False)
        print(f"User {name} added to PFP Tracked Table")

    iter = 0
    if username in pfp_table["index"].values:
        # print(f"User {name} already exists in PFP Tracked Table")
        # update the values in the existing table only if they are greater than the existing values
        if int(agg_likes) > int(pfp_table.loc[pfp_table["index"] == username, "Favorites"].values[iter]):
            print(f"PFP Tracked Likes updated to {agg_likes}")
            pfp_table.loc[pfp_table["index"] == name, [
                "Favorites"]] = agg_likes
        if int(agg_retweets) > int(pfp_table.loc[pfp_table["index"] == username, "Retweets"].values[iter]):
            print(
                f"PFP Tracked Retweets updated to {agg_retweets}")
            pfp_table.loc[pfp_table["index"] == name, [
                "Retweets"]] = agg_retweets
        if int(agg_replies) > int(pfp_table.loc[pfp_table["index"] == username, "Replies"].values[iter]):
            print(
                f"PFP Tracked Replies updated to {agg_replies}")
            pfp_table.loc[pfp_table["index"] == name, [
                "Replies"]] = agg_replies
        if int(agg_impressions) > int(pfp_table.loc[pfp_table["index"] == username, "Impressions"].values[iter]):
            print(
                f"PFP Tracked Impressions updated to {agg_impressions}")
            pfp_table.loc[pfp_table["index"] == username, [
                "Impressions"]] = agg_impressions

        iter += 1

        pfp_table.to_sql(
            pfp_table_name, engine, if_exists="replace", index=False)
    # print("DF PFP Tracked Replaced Table: ", pfp_table)

    if username not in pfp_table["index"].values:
        print(f"User {username} does not exist in PFP Tracked Table")
        # add the user to the table
        pfp_table = pd.DataFrame(index=[name],
                                 data=[[username, name, agg_likes, agg_retweets, agg_replies, agg_impressions]], columns=["index", "Name", "Favorites", "Retweets", "Replies", "Impressions"])
        pfp_table.to_sql(
            pfp_table_name, engine, if_exists="append", index=False)
        print(
            f"User {username} added to PFP Tracked table")
    # print("DF PFP Tracked Appended Table: ", pfp_table)

    return pfp_table

def create_dataFrame(id, author_username, author_name, likes, retweets, replies, impressions):
    authors_index = [author_username]

    df0 = pd.DataFrame(
        index=authors_index, data=author_name, columns=["Author"])
    df1 = pd.DataFrame(
        index=authors_index, data=int(likes), columns=["Favorites"])
    df2 = pd.DataFrame(
        index=authors_index, data=int(retweets), columns=["Retweets"])
    df3 = pd.DataFrame(
        index=authors_index, data=int(replies), columns=["Replies"])
    df4 = pd.DataFrame(
        index=authors_index, data=int(impressions), columns=["Impressions"])
    df5 = pd.DataFrame(
        index=authors_index, data=id, columns=["Tweet ID"])
    df = pd.concat([df0, df1, df2, df3, df4, df5], axis=1)

    return df


"""
Este é um script Python que serve como interface para um modelo de IA conversacional construído com as bibliotecas `LangChain` e `LLMs`.
O código cria um aplicativo da Web usando Streamlit, uma biblioteca Python para criar aplicativos da Web interativos.
# Author: Avratanu Biswas
# Date: March 11, 2023

Link de estudo: https://medium.com/@avra42/how-to-build-a-chatbot-with-chatgpt-api-and-a-conversational-memory-in-python-8d856cda4542
"""

# Import necessary libraries
import streamlit as st
from langchain.chains import ConversationChain
from langchain.chains.conversation.memory import ConversationEntityMemory
from langchain.chains.conversation.prompt import ENTITY_MEMORY_CONVERSATION_TEMPLATE
from langchain.llms import OpenAI
from langchain.chat_models import ChatOpenAI

# Set Streamlit page configuration
st.set_page_config(page_title='🧠MemoryBot🤖', layout='wide')
# Inicializar estados de sessão
if "generated" not in st.session_state:
    st.session_state["generated"] = []
if "past" not in st.session_state:
    st.session_state["past"] = []
if "input" not in st.session_state:
    st.session_state["input"] = ""
if "stored_session" not in st.session_state:
    st.session_state["stored_session"] = []

# Definir função para obter a entrada do usuário
def get_text():
    """
    Obtenha o texto de entrada do Usuário.
    Returns:
        (str): O texto digitado pelo Usuário
    """
    input_text = st.text_input("Você: ", st.session_state["input"], key="input",
                            placeholder="Seu assistente de AI aqui! pergunte-me qualquer coisa ...",
                            label_visibility='hidden')
    return input_text

# Define function to start a new chat
def new_chat():
    """
    Limpa o estado da sessão e inicia um novo chat.
    """
    save = []
    for i in range(len(st.session_state['generated'])-1, -1, -1):
        save.append("User:" + st.session_state["past"][i])
        save.append("Bot:" + st.session_state["generated"][i])
    st.session_state["stored_session"].append(save)
    st.session_state["generated"] = []
    st.session_state["past"] = []
    st.session_state["input"] = ""
    st.session_state.entity_memory.store = {}
    st.session_state.entity_memory.buffer.clear()

# Configure a barra lateral com várias opções
with st.sidebar.expander("🛠 ", expanded=False):
    # Opção para visualizar o armazenamento de memória
    if st.checkbox("Visualizar armazenamento de memória"):
        with st.expander("Memory-Store", expanded=False):
            st.session_state.entity_memory.store
    # Opção para visualizar o buffer de memória
    if st.checkbox("Visualizar buffer de memória"):
        with st.expander("Bufffer-Store", expanded=False):
            st.session_state.entity_memory.buffer
    MODEL = st.selectbox(label='Modelo', options=['gpt-3.5-turbo','text-davinci-003','text-davinci-002','code-davinci-002'])
    K = st.number_input(' (#)Resumo dos prompts a serem considerados',min_value=3,max_value=1000)

# Configurar o layout do aplicativo Streamlit
st.title("🤖 Chat Bot com 🧠")
st.subheader(" Alimentado pela 🦜 LangChain + OpenAI + Streamlit")

# Ask the user to enter their OpenAI API key
API_O = st.sidebar.text_input("API-KEY", type="password")

# Session state storage would be ideal
if API_O:
    # Create an OpenAI instance
    llm = ChatOpenAI(temperature=0,
                openai_api_key=API_O,
                model_name=MODEL,
                verbose=False)

    # Crie um objeto ConversationEntityMemory se ainda não tiver sido criado
    if 'entity_memory' not in st.session_state:
            st.session_state.entity_memory = ConversationEntityMemory(llm=llm, k=K )

        # Create the ConversationChain object with the specified configuration
    Conversation = ConversationChain(
            llm=llm,
            prompt=ENTITY_MEMORY_CONVERSATION_TEMPLATE,
            memory=st.session_state.entity_memory
        )
else:
    st.sidebar.warning('Chave de API necessária para experimentar este aplicativo. A chave de API não está armazenada de nenhuma forma.')
    # st.stop()

# Add a button to start a new chat
st.sidebar.button("New Chat", on_click = new_chat, type='primary')

# Get the user input
user_input = get_text()

# Generate the output using the ConversationChain object and the user input, and add the input/output to the session
if user_input:
    output = Conversation.run(input=user_input)
    st.session_state.past.append(user_input)
    st.session_state.generated.append(output)

# Allow to download as well
download_str = []
# Display the conversation history using an expander, and allow the user to download it
with st.expander("Conversation", expanded=True):
    for i in range(len(st.session_state['generated'])-1, -1, -1):
        st.info(st.session_state["past"][i],icon="🧐")
        st.success(st.session_state["generated"][i], icon="🤖")
        download_str.append(st.session_state["past"][i])
        download_str.append(st.session_state["generated"][i])

    # Can throw error - requires fix
    download_str = '\n'.join(download_str)
    if download_str:
        st.download_button('Download',download_str)

# Display stored conversation sessions in the sidebar
for i, sublist in enumerate(st.session_state.stored_session):
        with st.sidebar.expander(label= f"Conversation-Session:{i}"):
            st.write(sublist)

# Allow the user to clear all stored conversation sessions
if st.session_state.stored_session:
    if st.sidebar.checkbox("Clear-all"):
        del st.session_state.stored_session


__all__ = ()
import inspect

import objc
from objc._objc import _nameForSignature

basic_types = {
    objc._C_VOID: "void",
    objc._C_INT: "int",
    objc._C_UINT: "unsigned int",
    objc._C_LNG: "long",
    objc._C_ULNG: "unsigned long",
    objc._C_LNG_LNG: "long long",
    objc._C_ULNG_LNG: "unsigned long long",
    objc._C_FLT: "float",
    objc._C_DBL: "double",
    objc._C_SHT: "short",
    objc._C_USHT: "unsigned short",
    objc._C_CHR: "char",
    objc._C_UCHR: "unsigned char",
    objc._C_CHAR_AS_INT: "int8_t",
    objc._C_CHAR_AS_TEXT: "char",
    objc._C_UNICHAR: "UniChar",
    objc._C_BOOL: "bool",
    objc._C_NSBOOL: "BOOL",
    objc._C_ID: "id",
    objc._C_CLASS: "Class",
    objc._C_SEL: "SEL",
    objc._C_CHARPTR: "char*",
}

prefixes = {
    objc._C_IN: "in ",
    objc._C_OUT: "out ",
    objc._C_INOUT: "inout ",
    objc._C_CONST: "const ",
    objc._C_ONEWAY: "oneway ",
}

def describe_type(typestr):
    nm = basic_types.get(typestr)
    if nm is not None:
        return nm

    if typestr == b"^?":
        return "<FUNCTION>"
    elif typestr == b"@?":
        return "<BLOCK>"

    if typestr.startswith(objc._C_PTR):
        nm = _nameForSignature(typestr)
        if nm is not None:
            return nm
        return describe_type(typestr[1:]) + "*"

    if typestr[:1] in prefixes:
        return prefixes[typestr[:1]] + describe_type(typestr[1:])

    if typestr.startswith(objc._C_STRUCT_B):
        nm = _nameForSignature(typestr)
        if nm is not None:
            return nm

        typestr = typestr[1:]
        idx = typestr.find(b"=")
        if idx == -1:
            return "struct <?>"

        else:
            nm = typestr[:idx]
            if not nm:
                return "struct <?>"
            nm = nm.decode("utf-8").lstrip("_")
            if "." in nm:
                return nm.rsplit(".", 1)[-1]
            return nm

    if typestr.startswith(objc._C_VECTOR_B):
        base_type = basic_types[typestr[-2:-1]]
        if base_type.startswith("unsigned "):
            base_type = "u" + base_type.split()[-1]

        count = typestr[1:-2]
        return f"simd_{base_type}{count.decode()}"

    if typestr.startswith(objc._C_ARY_B):
        typestr = typestr[1:]
        d = b""
        while typestr[:1].isdigit():
            d += typestr[:1]
            typestr = typestr[1:]

        return "{}[{}]".format(describe_type(typestr), d.decode("utf-8"))

    if typestr.startswith(objc._C_UNION_B):
        typestr = typestr[1:]
        idx = typestr.find(b"=")
        if idx == -1:
            return "union <?>"

        else:
            nm = typestr[:idx]
            if not nm:
                nm = b"<?>"
            return "union {}".format(nm.decode("utf-8"))

    return "<?>"

def describe_callable(callable_object):
    if not hasattr(callable_object, "__name__") or not hasattr(
        callable_object, "__metadata__"
    ):
        return None

    name = callable_object.__name__
    try:
        metadata = callable_object.__metadata__()
    except objc.internal_error:
        return None

    return describe_callable_metadata(
        name, metadata, ismethod=isinstance(callable_object, objc.selector)
    )

def describe_callable_metadata(name, metadata, offset="", ismethod=False):
    arg_info = []
    if ismethod:
        arg_offset = 2

        name_parts = name.split(":")
        hdr_name = []
        if len(metadata["arguments"]) > arg_offset:
            for idx, (nm, info) in enumerate(
                zip(name_parts, metadata["arguments"][arg_offset:])
            ):
                if hdr_name:
                    hdr_name.append(" ")
                hdr_name.append(nm)
                hdr_name.append(":(")
                hdr_name.append(describe_type(info["type"]))
                hdr_name.append(")arg%d" % (idx,))

                if info["type"][:1] in prefixes and info["type"][:1] not in (
                    objc._C_ONEWAY,
                    objc._C_CONST,
                ):
                    arg_info.append((idx, info))
                elif info.get("printf_format"):
                    arg_info.append((idx, info))
                elif info.get("callable"):
                    arg_info.append((idx, info))

        else:
            hdr_name.append(name)

        if metadata.get("variadic"):
            hdr_name.append(", ...")

        header = "{} ({}){};".format(
            "+" if metadata["classmethod"] else "-",
            describe_type(metadata["retval"]["type"]),
            "".join(hdr_name),
        )
    else:
        hdr_name = []
        arg_offset = 0
        for idx, info in enumerate(metadata["arguments"]):
            if idx != 0:
                hdr_name.append(", ")
            hdr_name.append(describe_type(info["type"]))
            hdr_name.append(" arg%d" % (idx,))
            if info["type"][:1] in prefixes and info["type"][:1] not in (
                objc._C_ONEWAY,
                objc._C_CONST,
            ):
                arg_info.append((idx, info))
            elif info.get("printf_format"):
                arg_info.append((idx, info))
            elif info.get("callable"):
                arg_info.append((idx, info))
        if metadata.get("variadic"):
            hdr_name.append(", ...")

        if not hdr_name:
            hdr_name.append("void")

        header = "{} {}({});".format(
            describe_type(metadata["retval"]["type"]),
            name,
            "".join(hdr_name),
        )

    result = [header]
    if metadata.get("suggestion"):
        result.append("")
        result.append("WARNING: {}".format(metadata["suggestion"]))

    if arg_info:
        result.append("")
        for idx, info in arg_info:
            if info.get("printf_format"):
                result.append("arg%d: %%-style format string" % (idx,))
                continue

            elif info.get("callable"):
                result.append(
                    "arg%d: %s"
                    % (
                        idx,
                        describe_callable_metadata(
                            "callback", info["callable"], offset="    " + offset
                        ),
                    )
                )
                continue

            else:
                arg = info.get("c_array_length_in_arg")
                if arg is not None:
                    if isinstance(arg, tuple):
                        result.append(
                            "arg%d: array with length on input in arg%d, "
                            "and output in arg%d"
                            % (idx, arg[0] - arg_offset, arg[1] - arg_offset)
                        )
                    else:
                        if info.get("c_array_length_in_result"):
                            result.append(
                                "arg%d: array with length on input in arg%d, "
                                "and output in return value" % (idx, arg - arg_offset)
                            )
                        else:
                            result.append(
                                "arg%d: array with length in arg%d"
                                % (idx, arg - arg_offset)
                            )
                    continue

                if info.get("c_array_length_in_result"):
                    result.append("arg%d: array with length in return value" % (idx,))
                    continue

                if info.get("c_array_of_fixed_length"):
                    result.append(
                        "arg%d: array with length %d"
                        % (idx, info.get("c_array_of_fixed_length"))
                    )
                    continue

                if info.get("c_array_of_variable_length"):
                    result.append("arg%d: array with unknown length" % (idx,))
                    continue

                if info.get("c_array_delimited_by_null"):
                    result.append(
                        "arg%d: array (will be NULL terminated in C)" % (idx,)
                    )
                    continue

                result.append(
                    "arg%d: pass-by-reference %sargument"
                    % (idx, prefixes.get(info["type"][:1]))
                )

    if len(metadata["arguments"]) > arg_offset:
        if metadata.get("variadic") and metadata.get("c_array_delimited_by_null"):
            if not arg_info:
                result.append("")

            result.append(
                "Variadic arguments form an array of C type %s"
                % (describe_type(metadata["arguments"][-1]["type"]),)
            )

    if metadata.get("__doc__", None):
        result.append("\n")
        result.append(str(metadata["__doc__"]))

    return ("\n" + offset).join(result).replace("\n" + offset + "\n", "\n\n")

objc.options._callable_doc = describe_callable

def callable_signature(callable_object):
    # Create an inspect.Signature for an PyObjC callable
    # both objc.function and objc.native_selector only support positional
    # arguments, and not keyword arguments.
    if not hasattr(callable_object, "__name__") or not hasattr(
        callable_object, "__metadata__"
    ):
        return None

    try:
        metadata = callable_object.__metadata__()
    except objc.internal_error:
        # This can happen with some private methods with undocumented
        # characters in type encodings
        return None

    ismethod = isinstance(callable_object, objc.selector)
    isimp = isinstance(callable_object, objc.IMP)

    if ismethod:
        # Skip 'self' and 'selector' implicit arguments
        args = metadata["arguments"][2:]
    elif isimp:
        # Ignore the 'selector' argument
        args = (metadata["arguments"][0],) + metadata["arguments"][2:]
    else:
        args = metadata["arguments"]

    parameters = []
    for idx, _arg in enumerate(args):
        p_name = "arg%d" % (idx,)
        parameters.append(inspect.Parameter(p_name, inspect.Parameter.POSITIONAL_ONLY))

    return inspect.Signature(parameters)

objc.options._callable_signature = callable_signature


import requests
import re
import sys
import argparse
import json

def profile_data(username):
    data = requests.get("https://instagram.com/{}".format(username))
    if data.status_code == 404:
        print("Specified username not found")
        sys.exit()
    json_data   = json.loads(re.findall(r'window._sharedData\s=\s(\{.*\});</script>', data.text)[0])
    data        = json_data['entry_data']['ProfilePage'][0]['graphql']['user']
    followers_count = data['edge_followed_by']['count']
    follows_count   = data['edge_follow']['count']
    profile_pic = data['profile_pic_url']
    insta_id    = data['id']
    bio         = data['biography']
    total_posts = data['edge_owner_to_timeline_media']['count']
    posts = []
    for i in range(min(12,total_posts)):
        posts.append(data['edge_owner_to_timeline_media']['edges'][i]['node']['display_url'])
    profile = {
        'instagram_id' : insta_id,
        'followers' : followers_count,
        'follows' : follows_count,
        'profile_pic' : profile_pic,
        'bio' : bio,
        'total_posts' : total_posts,
        'latest_posts' : posts
         }
    open("{}.json".format(username), "w").write(str(json.dumps(profile)))
    print("Data written to {}.json".format(username))

def post_data(post_id):
    data = requests.get("https://instagram.com/p/{}".format(post_id))
    if data.status_code == 404:
        print("Specified post not found")
        sys.exit()
    json_data       = json.loads(re.findall(r'window._sharedData\s=\s(\{.*\});</script>', data.text)[0])
    data            = json_data['entry_data']['PostPage'][0]['graphql']['shortcode_media']
    likes_count     = data['edge_media_preview_like']['count']
    comments_count  = data['edge_media_to_parent_comment']['count']
    caption         = data['edge_media_to_caption']['edges'][0]['node']['text']
    media_url       = data['display_resources'][2]['src']
    is_video        = data['is_video']
    if is_video:
        media_url = data['video_url']
    comments = []
    for i in range(min(10, comments_count)):
        comments.append(data['edge_media_to_parent_comment']['edges'][i]['node']['text'])
    post = {
        'likes' : likes_count,
        'comments_count' : comments_count,
        'caption' : caption,
        'media' : media_url,
        'is_video' : is_video,
        'latest_comments' : comments
        }
    open("{}.json".format(post_id), "w").write(str(json.dumps(post)))
    print("Data written to {}.json".format(post_id))

def main():
    usage = '''
    Example:

    Fetch Profile Data:
        python3 igdata.py -p google

    Fetch Post Data:
        python3 igdata.py -d BzRIu3_lsev
    '''
    parser = argparse.ArgumentParser(description="Instagram Scraper", epilog=usage, formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument("-p", help="Instagram public profile", type=str)
    parser.add_argument("-d", help="Instagram public post", type=str)
    args = parser.parse_args()

    profile = args.p
    post    = args.d

    if profile:
        profile_data(profile)
    elif post:
        post_data(post)

main()


from time import time
from unicodedata import name
from unittest import result

from monarch.monarch_objects import PhaseSpace
from monarch.visual_tools import plot_2d_phase_space, plot_3d_phase_space, plot_histogram
from monarch.uart import UART

from matplotlib import pyplot as plt
import numpy as np

# Define the Lorentz attractor.
sigma = 10
rho = 28
beta = 8/3

ode = (
	lambda x,y,z : sigma * (y - x),
	lambda x,y,z : x * (rho - z) - y,
	lambda x,y,z : x * y - beta * z
)

phase_space = PhaseSpace(
    ode_system = ode,
    dt = 0.001,
    resolution = 32,
    max_limit = 64,
    four_quadrant = True,
    compress_space = False
)

#plot_histogram(phase_space)

fpga = UART(phase_space)
output_data = fpga.primary_eval(timesteps=100)
test_data = phase_space.run_simulation([2,2,2],timesteps=100)
print(output_data)

ax = plt.axes(projection='3d')
ax.plot3D(output_data[:, 0], output_data[:, 1], output_data[:, 2], 'gray')
ax.plot3D(test_data[:, 0], test_data[:, 1], test_data[:, 2], 'blue')
plt.show()