from neutron.extensions import securitygroup as sg_ext
from neutron import quota
from neutron_lib import exceptions as n_exc
from oslo_config import cfg
from oslo_log import log as logging
from oslo_utils import uuidutils

from quark.db import api as db_api
from quark import environment as env
from quark import exceptions as q_exc
from quark.plugin_modules import jobs as job_api
from quark import plugin_views as v
from quark import protocols

from quark.worker_plugins import sg_update_worker as sg_rpc_api

CONF = cfg.CONF
LOG = logging.getLogger(__name__)
DEFAULT_SG_UUID = "00000000-0000-0000-0000-000000000000"
GROUP_NAME_MAX_LENGTH = 255
GROUP_DESCRIPTION_MAX_LENGTH = 255
RULE_CREATE = 'create'
RULE_DELETE = 'delete'

def _validate_security_group_rule(context, rule):
    # TODO(mdietz): As per RM8615, Remote groups are not currently supported
    if rule.get("remote_group_id"):
        raise n_exc.InvalidInput(
            error_message="Remote groups are not currently supported")

    direction = rule.get("direction")
    if direction == env.Capabilities.EGRESS:
        if env.Capabilities.EGRESS not in CONF.QUARK.environment_capabilities:
            raise q_exc.EgressSecurityGroupRulesNotEnabled()

    protocol = rule.pop('protocol')
    # NOTE(roaet): these are not required by spec
    port_range_min = rule.get('port_range_min')
    port_range_max = rule.get('port_range_max')
    # TODO(anyone): this will error as None, so defaulting to ipv4
    et = rule.get('ethertype', 'IPv4')
    ethertype = protocols.translate_ethertype(et)

    if protocol:
        protocol = protocols.translate_protocol(protocol, et)
        protocols.validate_protocol_with_port_ranges(ethertype,
                                                     protocol,
                                                     port_range_min,
                                                     port_range_max)
        rule['protocol'] = protocol
    else:
        if port_range_min is not None or port_range_max is not None:
            raise sg_ext.SecurityGroupProtocolRequiredWithPorts()

    rule["ethertype"] = ethertype

    protocols.validate_remote_ip_prefix(ethertype,
                                        rule.get("remote_ip_prefix"))

    return rule

def _validate_security_group(security_group):
    if "name" in security_group:
        if len(security_group["name"]) > GROUP_NAME_MAX_LENGTH:
            raise n_exc.InvalidInput(
                error_message="Group name must be 255 characters or less")

        if security_group["name"] == "default":
            raise sg_ext.SecurityGroupDefaultAlreadyExists()

    if ("description" in security_group and
            len(security_group["description"]) > GROUP_DESCRIPTION_MAX_LENGTH):
        raise n_exc.InvalidInput(
            error_message="Group description must be 255 characters or less")

def create_security_group(context, security_group):
    LOG.info("create_security_group for tenant %s" %
             (context.tenant_id))
    group = security_group["security_group"]
    _validate_security_group(group)

    group_name = group.get('name', '')
    group_id = uuidutils.generate_uuid()

    with context.session.begin():
        group["id"] = group_id
        group["name"] = group_name
        group["tenant_id"] = context.tenant_id
        dbgroup = db_api.security_group_create(context, **group)
    return v._make_security_group_dict(dbgroup)

def update_security_group(context, id, security_group):
    if id == DEFAULT_SG_UUID:
        raise sg_ext.SecurityGroupCannotUpdateDefault()
    new_group = security_group["security_group"]
    _validate_security_group(new_group)

    with context.session.begin():
        group = db_api.security_group_find(context, id=id, scope=db_api.ONE)
        db_group = db_api.security_group_update(context, group, **new_group)

    return v._make_security_group_dict(db_group)

def delete_security_group(context, id):
    LOG.info("delete_security_group %s for tenant %s" %
             (id, context.tenant_id))

    with context.session.begin():
        group = db_api.security_group_find(context, id=id, scope=db_api.ONE)

        # TODO(anyone): name and ports are lazy-loaded. Could be good op later
        if not group:
            raise sg_ext.SecurityGroupNotFound(id=id)
        if id == DEFAULT_SG_UUID or group.name == "default":
            raise sg_ext.SecurityGroupCannotRemoveDefault()
        if group.ports:
            raise sg_ext.SecurityGroupInUse(id=id)
        db_api.security_group_delete(context, group)

def get_security_group(context, id, fields=None):
    LOG.info("get_security_group %s for tenant %s" %
             (id, context.tenant_id))
    group = db_api.security_group_find(context, id=id, scope=db_api.ONE)
    if not group:
        raise sg_ext.SecurityGroupNotFound(id=id)
    return v._make_security_group_dict(group, fields)

def get_security_groups(context, filters=None, fields=None,
                        sorts=None, limit=None, marker=None,
                        page_reverse=False):
    LOG.info("get_security_groups for tenant %s" %
             (context.tenant_id))
    groups = db_api.security_group_find(context, **filters)
    return [v._make_security_group_dict(group) for group in groups]

@env.has_capability(env.Capabilities.SG_UPDATE_ASYNC)
def _perform_async_update_rule(context, id, db_sg_group, rule_id, action):
    """Updates a SG rule async and return the job information.

    Only happens if the security group has associated ports. If the async
    connection fails the update continues (legacy mode).
    """
    rpc_reply = None
    sg_rpc = sg_rpc_api.QuarkSGAsyncProcessClient()
    ports = db_api.sg_gather_associated_ports(context, db_sg_group)
    if len(ports) > 0:
        rpc_reply = sg_rpc.start_update(context, id, rule_id, action)
        if rpc_reply:
            job_id = rpc_reply['job_id']
            job_api.add_job_to_context(context, job_id)
        else:
            LOG.error("Async update failed. Is the worker running?")

def create_security_group_rule(context, security_group_rule):
    """Creates a rule and updates the ports (async) if enabled."""
    LOG.info("create_security_group for tenant %s" %
             (context.tenant_id))
    with context.session.begin():
        rule = _validate_security_group_rule(
            context, security_group_rule["security_group_rule"])
        rule["id"] = uuidutils.generate_uuid()

        group_id = rule["security_group_id"]
        group = db_api.security_group_find(context, id=group_id,
                                           scope=db_api.ONE)
        if not group:
            raise sg_ext.SecurityGroupNotFound(id=group_id)

        quota.QUOTAS.limit_check(
            context, context.tenant_id,
            security_rules_per_group=len(group.get("rules", [])) + 1)

        new_rule = db_api.security_group_rule_create(context, **rule)
    if group:
        _perform_async_update_rule(context, group_id, group, new_rule.id,
                                   RULE_CREATE)
    return v._make_security_group_rule_dict(new_rule)

def get_security_group_rule(context, id, fields=None):
    LOG.info("get_security_group_rule %s for tenant %s" %
             (id, context.tenant_id))
    rule = db_api.security_group_rule_find(context, id=id,
                                           scope=db_api.ONE)
    if not rule:
        raise sg_ext.SecurityGroupRuleNotFound(id=id)
    return v._make_security_group_rule_dict(rule, fields)

def delete_security_group_rule(context, id):
    """Deletes a rule and updates the ports (async) if enabled."""
    LOG.info("delete_security_group %s for tenant %s" %
             (id, context.tenant_id))
    with context.session.begin():
        rule = db_api.security_group_rule_find(context, id=id,
                                               scope=db_api.ONE)
        if not rule:
            raise sg_ext.SecurityGroupRuleNotFound(id=id)

        group = db_api.security_group_find(context, id=rule["group_id"],
                                           scope=db_api.ONE)
        if not group:
            raise sg_ext.SecurityGroupNotFound(id=id)

        rule["id"] = id
        db_api.security_group_rule_delete(context, rule)
    if group:
        _perform_async_update_rule(context, group.id, group, id, RULE_DELETE)

def get_security_group_rules(context, filters=None, fields=None,
                             sorts=None, limit=None, marker=None,
                             page_reverse=False):
    LOG.info("get_security_group_rules for tenant %s" %
             (context.tenant_id))
    rules = db_api.security_group_rule_find(context, **filters)
    return [v._make_security_group_rule_dict(rule) for rule in rules]


import logging

from WDL import values_to_json
from WDL.runtime.task import run_local_task, _eval_task_inputs, InputStdLib
from WDL.runtime import task_container

from utils import unmarshall_task, unmarshall_input, load_config

from cluster import app

def _eval_command(logger, cfg, task, inputs, run_id = None, run_dir = None, logger_prefix = None,
    _run_id_stack = None, _cache = None, _plugins = None,):
    container =  task_container.new(cfg, logger, run_id, run_dir)
    container_env=_eval_task_inputs(logger, task, inputs, container)
    stdlib = InputStdLib(task.effective_wdl_version, logger, container)
    cmd = task.command.eval(container_env, stdlib).value

@app.task
def run_task(
    task_str,
    task_digest,
    task_inputs,
    run_id = None,
    run_dir = None,
    logger_prefix = None,
    _run_id_stack = None,
    _cache = None,
    _plugins = None,
):

    cfg = load_config()

    task = unmarshall_task(task_str, task_digest)
    input_env = unmarshall_input(task, task_inputs)

    sub_args = (cfg, task, input_env)
    sub_kwargs = {
        "run_id": run_id,
        "run_dir": run_dir,
        "logger_prefix": logger_prefix,
        "_run_id_stack": _run_id_stack,
        "_cache": _cache,
    }
    #_eval_command(logger, *sub_args, **sub_kwargs)

    output_dir, output_env = run_local_task(*sub_args, **sub_kwargs)
    output_json = values_to_json(output_env, namespace=task.name)
    output = (output_dir, output_json)

    return output


import json
import os
from pathlib import Path

"""Utility script to remove a certain language from the translations mapping json"""
PATH_TO_TRANSLATIONS_MAPPING_JSON = '/Users/sunyitao/Documents/Projects/GitHub/PsychicLearners/data_utils/translations_mapping.json'
LANGUAGE_TO_REMOVE = ''

#Below can be unchanged
PATH_TO_WORD_2_LANG_JSON = Path.cwd() / 'word_to_lang.json'
OUTPUT_JSON = Path(PATH_TO_TRANSLATIONS_MAPPING_JSON).parent / 'translations_mapping_output.json'

with open(PATH_TO_TRANSLATIONS_MAPPING_JSON, 'r') as f:
    translations_mapping = json.load(f)
with open(PATH_TO_WORD_2_LANG_JSON, 'r') as f:
    word2lang = json.load(f)

new_mapping = {}
for word, translated_word in translations_mapping.items():
    if word in word2lang.keys():
        language = word2lang[word]
        if language == LANGUAGE_TO_REMOVE:
            print(f'Word:{word}, Translation: {translated_word} will be removed')
            continue
        else:
            new_mapping[word] = translated_word
    else:
        raise LookupError(f'Unrecognised word {word}, shouldnt be happening')

with open(OUTPUT_JSON, 'w') as f:
    f.write(json.dumps(new_mapping))


import json
import time
import threading
from websocket import create_connection

ws = create_connection("ws://192.168.1.101:8080",timeout=40)

login_request = '''{
	"i": "",
	"t": "login",
	"c": {
		"l": "user",
		"p": "user",
		"t": ""
	},
	"r": "websocket"
}'''
ws.send(login_request)
login_message = ws.recv()
session_id = json.loads(login_message)['i']
getVoltage= '''{
	        "i": "%s",
	        "t": "request",
	        "c": [
		            {
			            "c": "getItem",
			            "p": {
				            "p": {
					            "l": "0",
					            "a": "*",
					            "c": "*"
				            },
				            "i": "Status.firmwareRelease",
				            "v": "",
				            "u": ""
			            }
		            }
	        ],
	        "r": "websocket"
            }''' % session_id

getConfig = '''{
            "i":"%s",
            "t":"getConfig",
            "c":[],
            "r":"websocket"
            }''' % session_id

ws.send(getVoltage)

def get_json():
    while True:
        received = ws.recv()
        jsoned = json.loads(received)
        print(json.dumps(jsoned, indent=4, sort_keys=True))
        if 'c' in jsoned[0]:
            for item in jsoned[0]['c']:
                typ = item['d']['i']
                if typ == "System.time":
                    print("TIME:",item['d']['v'])
                if typ == "Status.temperature0":
                    print("TEMP0",item['d']['v'],item['d']['u'])
                if typ == "Status.temperature1":
                    print("TEMP1:",item['d']['v'],item['d']['u'])
                if typ == "Status.voltageMeasure":
                    print("VOLTAGE:",item['d']['v'],"CHANNEL:",item['d']['p']['c'],item['d']['u'])
                if typ == "Status.currentMeasure":
                    print("CURRENT:",item['d']['v'],"CHANNEL:",item['d']['p']['c'],item['d']['u'])
                if typ == "Status.safetyLoopClosed":
                    print("LOOP STATUS: (1=True,0=False)",item['d']['v'])
t_ls = []
for i in range(2):
    thrd = threading.Thread(target=get_json)
    t_ls.append(thrd)
for t in t_ls:
    t.start()
for t in t_ls:
    t.join()

ws.close()


from django.core.paginator import Paginator
from django.shortcuts import render, get_object_or_404
from django.shortcuts import redirect
from .models import Post, Group, User, Comment, Follow
from .forms import PostForm, CommentForm
from django.contrib.auth.decorators import login_required

def index(request):
    post_list = Post.objects.order_by("-pub_date").all()
    paginator = Paginator(post_list, 10) # показывать по 10 записей на странице.
    page_number = request.GET.get('page') # переменная в URL с номером запрошенной страницы
    page = paginator.get_page(page_number) # получить записи с нужным смещением
    return render(request, 'posts/index.html', {'page': page, 'paginator': paginator})

def group_posts(request, slug):
    group = get_object_or_404(Group, slug=slug)
    posts = Post.objects.filter(group=group).order_by("-pub_date")[:12]
    paginator = Paginator(posts, 10)
    page_number = request.GET.get('page')
    page = paginator.get_page(page_number)
    return render(request, "group.html", {"group": group, 'page': page, 'paginator': paginator})

def new_post(request):
    user = request.user
    form = PostForm(request.POST or None, files=request.FILES or None)
    if not request.user.is_authenticated:
        return redirect(index)
    if request.method == 'POST':
        if form.is_valid():
            n_post = form.save(commit=False)
            n_post.author = user
            n_post.save()
            return redirect('index')
        return render(request, 'posts/new_post.html', {'form': form})
    form = PostForm()
    return render(request, 'posts/new_post.html', {'form': form})

def profile(request, username):
    author = get_object_or_404(User, username = username)
    post_list = Post.objects.filter(author = author).order_by('-pub_date')
    paginator = Paginator(post_list, 10) # показывать по 10 записей на странице.
    page_number = request.GET.get('page') # переменная в URL с номером запрошенной страницы
    page = paginator.get_page(page_number) # получить записи с нужным смещением
    first_post = Post.objects.filter(author = author).order_by('-pub_date')[:1]
    followers = Follow.objects.filter()
    count_followers = Follow.objects.filter(author=author).count()
    count_followering = Follow.objects.filter(user=author).count()
    if request.user.is_authenticated:
        following = Follow.objects.filter(author=author, user=request.user)
    else:
        following = []
    count = post_list.count
    return render(request, "posts/profile.html", {'count':count, 'author':author, 'page': page,
                  'paginator': paginator, 'count_followers': count_followers,
                  "following": following, 'count_followering': count_followering, 'first_post': first_post})

def post_view(request, username, post_id):
    profile = get_object_or_404(User, username=username)
    post = get_object_or_404(Post, author=profile.pk, id=post_id)
    count = Post.objects.filter(author = profile).count
    form = CommentForm()
    comments = Comment.objects.filter(post=post_id)
    count_followers = Follow.objects.filter(author=profile).count()
    count_followering = Follow.objects.filter(user=profile).count()
    return render(request, "posts/post.html", {"profile":profile, 'post':post, "count":count, 'form': form, 'comments' : comments, "count_followers": count_followers, "count_followering": count_followering})

def post_edit(request, username, post_id):
    post = get_object_or_404(Post, id = post_id)
    if request.user != post.author:
        return redirect("post", username, post_id)
    form = PostForm(request.POST or None, files=request.FILES or None, instance=post)
    if form.is_valid():
        form.save()
        return redirect("post", username, post_id)
    form = PostForm()
    return render(request, "posts/post_edit.html", {"form": form, 'post':post})

@login_required
def add_comment(request, username, post_id):
    post = get_object_or_404(Post, pk=post_id)
    comments = Comment.objects.filter(post=post).all()
    if request.method == 'POST':
        form = CommentForm(request.POST)
        if form.is_valid():
            comment = form.save(commit=False)
            comment.post = post
            comment.author = request.user
            comment.save()
            return redirect('post', username=post.author.username, post_id=post_id,)
        return render(request, "posts/post.html", {'form': form, 'post':post})
    form = CommentForm()
    return render(request, "posts/comment.html", {"post": post, "form": form, 'comments': comments})

def page_not_found(request, exception):
    # Переменная exception содержит отладочную информацию,
    # выводить её в шаблон пользователской страницы 404 мы не станем
    return render(request, "misc/404.html", {"path": request.path}, status=404)

def server_error(request):
    return render(request, "misc/500.html", status=500)

@login_required
def follow_index(request):
    authors = Follow.objects.filter(user=request.user).values_list('author', flat=True)
    post_list = Post.objects.filter(author__in=authors).order_by("-pub_date").all()
    paginator = Paginator(post_list, 10) # показывать по 10 записей на странице.
    page_number = request.GET.get('page') # переменная в URL с номером запрошенной страницы
    page = paginator.get_page(page_number) # получить записи с нужным смещением
    return render(request, 'posts/follow.html', {'page': page, 'paginator': paginator})

@login_required
def profile_follow(request, username):
    author = get_object_or_404(User, username = username)
    user = request.user
    following = Follow.objects.filter(user=user).filter(author=author)
    if not following:
        if user != author:
            Follow.objects.create(user=user, author=author)
            return redirect("profile", username=username)
        else:
            return redirect("profile", username=username)
    else:
        return redirect("profile", username=username)

@login_required
def profile_unfollow(request, username):
    author = get_object_or_404(User, username = username)
    user = request.user
    Follow.objects.filter(user=user, author=author).delete()
    return redirect("profile", username=username)


from pathlib import Path
from stat import S_IMODE

import pytest

from omdlib.certs import CertificateAuthority  # pylint: disable=wrong-import-order

from cmk.utils.certs import root_cert_path, RootCA
from cmk.utils.crypto.certificate import (
    Certificate,
    CertificatePEM,
    PlaintextPrivateKeyPEM,
    RsaPrivateKey,
)

CA_NAME = "test-ca"

@pytest.fixture(name="ca")
def fixture_ca(tmp_path: Path) -> CertificateAuthority:
    ca_path = tmp_path / "ca"
    return CertificateAuthority(
        root_ca=RootCA.load_or_create(root_cert_path(ca_path), CA_NAME),
        ca_path=ca_path,
    )

def test_initialize(ca: CertificateAuthority) -> None:
    assert ca.root_ca.certificate.common_name == CA_NAME
    assert ca.root_ca.certificate.public_key == ca.root_ca.private_key.public_key

def _file_permissions_is_660(path: Path) -> bool:
    return oct(S_IMODE(path.stat().st_mode)) == "0o660"

def test_create_site_certificate(ca: CertificateAuthority) -> None:
    site_id = "xyz"
    assert not ca.site_certificate_exists(site_id)

    ca.create_site_certificate(site_id, key_size=1024)
    assert ca.site_certificate_exists(site_id)
    assert _file_permissions_is_660(ca._site_certificate_path(site_id))

    mixed_pem = ca._site_certificate_path(site_id).read_bytes()
    certificate = Certificate.load_pem(CertificatePEM(mixed_pem))
    private_key = RsaPrivateKey.load_pem(PlaintextPrivateKeyPEM(mixed_pem), None)

    assert certificate.common_name == site_id
    assert certificate.public_key == private_key.public_key
    certificate.verify_is_signed_by(ca.root_ca.certificate)


import getopt
import json
import os
import sys
import pandas as pd
from operator import itemgetter
from sv_portal_reader import SVPortalParser
from jcg_scraper import JCGScraper

def analyzer(argv):

    try:
        if not os.path.exists('jcg-data'):
            os.makedirs('jcg-data')

        format = ""
        jcg_code = ""
        opts, args = getopt.getopt(argv, 'c:f:')

        for opt, arg in opts:
            if opt == "-c":
                jcg_code = arg

            if opt == "-f":
                format = arg

        f = open(format)
        data = json.load(f)
        sv_portal = SVPortalParser(format_data=data)
        scraper = JCGScraper(jcg_code, parser=sv_portal)
        scraper.scrape_entries()
        scraper.scrape_results()

        jcg_data = scraper.entry_decks.values()
        sorted_data = sorted(jcg_data, key=itemgetter("top"), reverse=True)

        jcg_decks_data = []
        players_data = []

        for data in jcg_data:
            player = {
                "id": data["id"],
                "name": data["name"],
                "top": data["top"]
            }

            for (idx, deck) in enumerate(data["decks"]):
                result = (deck["craft"], deck["archetype"], deck["link"], data["top"], data["name"], data["id"])
                player[f'deck_{idx+1}'] = deck["archetype"]
                player[f'link_{idx+1}'] = deck["link"]
                jcg_decks_data.append(result)

            players_data.append(player)

        players_data_df = pd.DataFrame(players_data)
        decks_data_df = pd.DataFrame(jcg_decks_data,
            columns=[
                "craft",
                "archetype",
                "deck_link",
                "score",
                "player_name",
                "player_id"
            ]
        )
        outdir = './jcg-data'

        archetype_df = decks_data_df['archetype'].value_counts()
        craft_df = decks_data_df['craft'].value_counts()

        players_data_df.to_csv(os.path.join(outdir, f'{scraper.jcg_date}-lineups.csv'), index=False)
        decks_data_df.to_csv(os.path.join(outdir, f"{scraper.jcg_date}-classified-decks.csv"), index=False)
        archetype_df.to_csv(os.path.join(outdir, f"{scraper.jcg_date}-archetypes-data.csv"))
        craft_df.to_csv(os.path.join(outdir, f"{scraper.jcg_date}-crafts-data.csv"))
    except getopt.GetoptError:
        print('analyzer.py -c <jcg-code> -f <format-file>')
        sys.exit(2)

if __name__ == "__main__":
    analyzer(sys.argv[1:])


import re
from collections import Counter
from itertools import islice
from pathlib import Path
from typing import List, Dict, Iterable, Tuple, Generator, Collection

import jieba
import numpy as np
import pandas as pd
import tensorflow as tf
from numpy import array
from pandas import DataFrame
from sklearn.model_selection import train_test_split
from tensorflow import Tensor, constant
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.preprocessing.text import Tokenizer
from tqdm import tqdm

from user_profile_prediction.data.stopwords import StopwordsDataset
from user_profile_prediction.etl import BasePreprocess, EmbeddingModel

stop_words: StopwordsDataset = StopwordsDataset()

class PreprocessTrainingData(BasePreprocess):
    EMBEDDING_SIZE: int
    SENTENCE_LEN: int

    VOCABULARY_SIZE: int

    sample_num: int
    train_valid_test_weights: Collection

    age_label: List[int] = list()
    gender_label: List[int] = list()
    education_label: List[int] = list()

    age_label_weights: Dict[int, int]
    gender_label_weights: Dict[int, int]
    education_label_weights: Dict[int, int]

    tokenizer: Tokenizer

    path_obj: Path = Path(__file__).parent.parent / "data" / "train_split_words.csv"
    saved_path: str = path_obj.absolute().__str__()

    @classmethod
    def load_from_csv(cls, file_path: str) -> DataFrame:
        df: DataFrame = pd.read_csv(file_path, sep="###__###", header=None)
        df.columns = ['ID', 'Age', 'Gender', 'Education', 'Query_List']

        return df

    def __init__(
            self,
            csv_file_path: str,
            train_valid_weights: Collection = (0.9, 0.1),
            embedding_size: int = 100,
            sentence_len: int = 3,
            vocabulary_size: int = 10000
    ):
        super(PreprocessTrainingData, self).__init__(csv_file_path)
        self.preprocess_data.columns = list()
        self.train_valid_weights = train_valid_weights

        self.EMBEDDING_SIZE = embedding_size
        self.SENTENCE_LEN = sentence_len
        self.VOCABULARY_SIZE = vocabulary_size

    def split_sentence(self):
        indexes = self.data.index.to_list()
        # np.random.shuffle(indexes)

        saved_path: Path = Path(self.saved_path)
        if saved_path.exists():
            print(f"loading split_words.csv from '{self.saved_path}'")
            self.load_split_words_csv()
            print("finish loading split_words.csv")

            static: List = [len(sent) for sent in self.sentences_with_split_words]
            print(f"The average length of sentence is {sum(static) / len(static)}")
            return self.sentences_with_split_words

        print("start to split words by jieba")

        for index, query in tqdm(self.data.iloc[indexes[:]].iterrows()):
            # TODO 测试模型由于计算资源有限，只用1000个样本做测试

            if query["Age"] == 0 or query["Gender"] == 0 or query["Education"] == 0:
                continue

            words = []

            for sentence in query["Query_List"].split("\t"):
                res = re.findall("(?isu)(https?\://[a-zA-Z0-9\.\?/&\=\:]+)", sentence)
                if not res:
                    cut_words: List = jieba.lcut(sentence)
                    cut_words = self.filter_stop_words(cut_words)

                    words.extend(cut_words)

            self.sentences_with_split_words.append(words)

            self.age_label.append(query["Age"])
            self.gender_label.append(query["Gender"])
            self.education_label.append(query["Education"])

        self.sample_num = len(self.sentences_with_split_words)

        self.age_label_weights = dict(Counter(self.age_label))
        self.gender_label_weights = dict(Counter(self.gender_label))
        self.education_label_weights = dict(Counter(self.education_label))
        print("finish splitting words")

        print(f"saving split_words.csv to '{self.saved_path}'")
        self.save_split_words_csv()
        print("finish saving split_words.csv")

        static: List = [len(sent) for sent in self.sentences_with_split_words]
        print(f"The average length of sentence is {sum(static) / len(static)}")

        return self.sentences_with_split_words

    def save_split_words_csv(self) -> None:
        with open(self.saved_path, "w") as f:
            f.write("Query_List, Age, Gender, Education\n")

            for i in range(len(self.sentences_with_split_words)):
                f.write(
                    "{}, {}, {}, {}\n".format(
                        ' '.join(self.sentences_with_split_words[i]),
                        self.age_label[i],
                        self.gender_label[i],
                        self.education_label[i]
                    )
                )

    def load_split_words_csv(self) -> None:
        line: str

        with open(self.saved_path, "r") as f:
            for line in islice(f, 1, None):
                if line.endswith("\n"):
                    line = line[:-1]
                query_list, age, gender, edu = line.split(",")
                self.sentences_with_split_words.append(query_list.split(" "))
                self.age_label.append(int(age))
                self.gender_label.append(int(gender))
                self.education_label.append(int(edu))

    @property
    def train_valid_weights(self):
        return self._train_valid_weights

    @train_valid_weights.setter
    def train_valid_weights(self, weights: Collection):
        if weights.__len__() != 2:
            raise ValueError("set wrong dim weights")

        if sum(weights) != 1.:
            raise ValueError("sum of weights not equal to 1")

        self._train_valid_weights = weights

    @staticmethod
    def filter_stop_words(words: Iterable) -> List:
        return [w for w in words if w not in stop_words and w != " "]

    def tokenize(self) -> Tokenizer:
        print("Tokenizing words")
        self.tokenizer: Tokenizer = Tokenizer(
            num_words=self.VOCABULARY_SIZE,
            oov_token="NaN",
            filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n'
        )
        self.tokenizer.fit_on_texts(self.sentences_with_split_words)

        print(f"Numbers of total words is {len(self.tokenizer.word_index)}")

        return self.tokenizer

    def age_data_iter(self) -> Generator[Tuple[array, int], None, None]:
        for i, s in enumerate(self.sentences_with_split_words):
            yield [x[0] if len(x) != 0 else 0 for x in self.tokenizer.texts_to_sequences(s)], self.age_label[i]

    def gender_data_iter(self, model: EmbeddingModel) -> Generator[Tuple[array, int], None, None]:
        for i, s in enumerate(self.sentences_with_split_words):
            yield [x[0] if len(x) != 0 else 0 for x in self.tokenizer.texts_to_sequences(s)], self.gender_label[i]

    def education_data_iter(self, model: EmbeddingModel) -> Generator[Tuple[array, int], None, None]:
        for i, s in enumerate(self.sentences_with_split_words):
            yield [x[0] if len(x) != 0 else 0 for x in self.tokenizer.texts_to_sequences(s)], self.education_label[i]

    def split_data(self, data_iter: Generator, one_hot: bool = True) -> Tuple[Tensor, Tensor, Tensor, Tensor]:
        all_data: List[Tuple[array, int]]
        all_data = [(x, y) for x, y in data_iter]

        x_train_raw, y_train_raw = zip(*all_data)

        print("Padding sentences")
        x_train_raw = pad_sequences(
            x_train_raw,
            maxlen=self.SENTENCE_LEN,
            padding="post",
            truncating="post"
        )

        y_train_raw = array(y_train_raw)
        # x_train_raw, y_train_raw = \
        #     np.stack(x_train_raw, axis=0).reshape(-1, self.SENTENCE_LEN * self.EMBEDDING_SIZE), array(y_train_raw)

        x_train: array
        x_val: array
        y_train: array
        y_val: array

        x_train, x_val, y_train, y_val = train_test_split(
            x_train_raw, y_train_raw,
            test_size=self._train_valid_weights[1],
            random_state=101
        )

        # ros: RandomOverSampler = RandomOverSampler(random_state=202)
        # x_train, y_train = ros.fit_resample(x_train, y_train)

        # x_train: Tensor = constant(x_train.reshape(-1, self.SENTENCE_LEN, self.EMBEDDING_SIZE))
        # x_val: Tensor = constant(x_val.reshape(-1, self.SENTENCE_LEN, self.EMBEDDING_SIZE))
        x_train: Tensor = constant(x_train)
        x_val: Tensor = constant(x_val)

        if one_hot:
            y_train: Tensor = tf.one_hot(y_train, depth=np.unique(y_train_raw).__len__())
            y_val: Tensor = tf.one_hot(y_val, depth=np.unique(y_train_raw).__len__())

        return x_train, x_val, y_train, y_val

if __name__ == "__main__":
    p = PreprocessTrainingData("/Volumes/Samsung_T5/Files/Document/china_hadoop/GroupProject/project_data/data/train.csv")
    p.split_sentence()
    # a, b, c, d = p.split_data(p.age_data_iter())

    print("finish")


from decimal import Decimal
from django.conf import settings
from django.core.urlresolvers import reverse
from django.shortcuts import render, get_object_or_404
from paypal.standard.forms import PayPalPaymentsForm
from orders.models import Order
from django.views.decorators.csrf import csrf_exempt

@csrf_exempt
def payment_done(request):
    return render(request, 'payment/done.html')

@csrf_exempt
def payment_canceled(request):
    return render(request, 'payment/canceled.html')

def payment_process(request):
    order_id = request.session.get('order_id')
    order = get_object_or_404(Order, id=order_id)
    host = request.get_host()

    paypal_dict = {
        'business': settings.PAYPAL_RECEIVER_EMAIL,
        'amount': '%.2f' % order.get_total_cost().quantize(Decimal('.01')),
        'item_name': 'Order {}'.format(order.id),
        'invoice': str(order.id),
        'currency_code': 'USD',
        'notify_url': 'http://{}{}'.format(host, reverse('paypal-ipn')),
        'return_url': 'http://{}{}'.format(host, reverse('payment:done')),
        'cancel_return': 'http://{}{}'.format(host, reverse('payment:canceled')),
    }
    form = PayPalPaymentsForm(initial=paypal_dict)
    return render(request, 'payment/process.html', {'order': order,
                                                    'form':form})


#!/usr/bin/env python3
"""
maintains and tracks instances in a geth (go-ethereum) farm on NCS
"""

# standard library modules
import argparse
import asyncio
import collections
from concurrent import futures
import errno
import datetime
import json
import logging
import os
import re
import shutil
import signal
import subprocess
import sys
import threading
import time
import uuid

# third-party module(s)
import dateutil.parser
import pymongo

# neocortix modules
try:
    import ncs
except ImportError:
    # set system and python paths for default places, since path seems to be not set properly
    ncscliPath = os.path.expanduser('~/ncscli/ncscli')
    sys.path.append( ncscliPath )
    os.environ["PATH"] += os.pathsep + ncscliPath
    import ncs
import jsonToKnownHosts
import ncsgeth
import tellInstances

logger = logging.getLogger(__name__)

global resultsLogFile

# possible place for globals is this class's attributes
class g_:
    signaled = False

def sigtermHandler( sig, frame ):
    g_.signaled = True
    logger.warning( 'SIGTERM received; graceful exit may take a few minutes' )
    #raise SigTerm()

def sigtermSignaled():
    return g_.signaled

def sigtermNotSignaled():
    return not sigtermSignaled()

def boolArg( v ):
    '''use with ArgumentParser add_argument for (case-insensitive) boolean arg'''
    if v.lower() == 'true':
        return True
    elif v.lower() == 'false':
        return False
    else:
        raise argparse.ArgumentTypeError('Boolean value expected.')

def datetimeIsAware( dt ):
    if not dt: return None
    return (dt.tzinfo is not None) and (dt.tzinfo.utcoffset( dt ) is not None)

def universalizeDateTime( dt ):
    if not dt: return None
    if datetimeIsAware( dt ):
        return dt.astimezone(datetime.timezone.utc)
    return dt.replace( tzinfo=datetime.timezone.utc )

def interpretDateTimeField( field ):
    '''return utc datetime for a given datetime or a string parseable as a date/time'''
    if isinstance( field, datetime.datetime ):
        return universalizeDateTime( field )
    elif isinstance( field, str ):
        return universalizeDateTime( dateutil.parser.parse( field ) )
    else:
        raise TypeError( 'datetime or parseable string required' )

def demuxResults( inFilePath ):
    '''deinterleave jlog items into separate lists for each instance'''
    byInstance = {}
    badOnes = set()
    topLevelKeys = collections.Counter()
    # demux by instance
    with open( inFilePath, 'rb' ) as inFile:
        for line in inFile:
            decoded = json.loads( line )
            for key in decoded:
                topLevelKeys[ key ] += 1
            iid = decoded.get( 'instanceId', '<unknown>')
            have = byInstance.get( iid, [] )
            have.append( decoded )
            byInstance[iid] = have
            if 'returncode' in decoded:
                rc = decoded['returncode']
                if rc:
                    #logger.info( 'returncode %d for %s', rc, iid )
                    badOnes.add( iid )
            if 'exception' in decoded:
                #logger.info( 'exception %s for %s', decoded['exception'], iid )
                badOnes.add( iid )
            if 'timeout' in decoded:
                #logger.info( 'timeout %s for %s', decoded['timeout'], iid )
                badOnes.add( iid )
    return byInstance, badOnes

def getStartedInstances( db ):
    collName = 'launchedInstances'
    logger.info( 'getting instances from %s', collName )
    launchedColl = db[collName]
    inRecs = list( launchedColl.find() ) # fully iterates the cursor, getting all records
    if len(inRecs) <= 0:
        logger.warning( 'no launched instances found for %s', collName )
    for inRec in inRecs:
        if 'instanceId' not in inRec:
            logger.warning( 'no instance ID in input record')
    startedInstances = [inst for inst in inRecs if inst['state'] == 'started']
    return startedInstances

def ingestJson( srcFilePath, dbName, collectionName, append ):
    # uses some args from the global ArgumentParser
    cmd = [
        'mongoimport', '--host', args.mongoHost, '--port', str(args.mongoPort),
        '-d', dbName, '-c', collectionName,
        srcFilePath
    ]
    if srcFilePath.endswith( '.json' ):
        cmd.append( '--jsonArray' )
    if not append:
        cmd.append( '--drop' )
    #logger.info( 'cmd: %s', cmd )
    subprocess.check_call( cmd )

def importAnchorNodes( srcFilePath, db, collectionName ):
    # uses some args from the global ArgumentParser
    cmd = [
        'mongoimport', '--host', args.mongoHost, '--port', str(args.mongoPort),
        '-d', db.name, '-c', collectionName, '--jsonArray',
        '--mode=upsert', '--upsertFields=instanceId',
        srcFilePath
    ]
    #logger.info( 'cmd: %s', cmd )
    subprocess.check_call( cmd )
    db[collectionName].create_index( 'instanceId' )

def launchInstances( authToken, nInstances, sshClientKeyName, launchedJsonFilepath,
        filtersJson=None, encryptFiles=True ):
    returnCode = 13
    logger.info( 'launchedJsonFilepath %s', launchedJsonFilepath )
    try:
        with open( launchedJsonFilepath, 'w' ) as launchedJsonFile:
            returnCode = ncs.launchScInstances( authToken, encryptFiles, numReq=nInstances,
                sshClientKeyName=sshClientKeyName, jsonFilter=filtersJson,
                okToContinueFunc=sigtermNotSignaled, jsonOutFile=launchedJsonFile )
    except Exception as exc:
        logger.error( 'exception while launching instances (%s) %s', type(exc), exc, exc_info=True )
        returnCode = 99
    return returnCode

def terminateNcsScInstances( authToken, instanceIds ):
    '''try to terminate instances; return list of instances terminated (empty if none confirmed)'''
    terminationLogFilePath = os.path.join( dataDirPath, 'badTerminations.log' )  # using global dataDirPath
    dateTimeStr = datetime.datetime.now( datetime.timezone.utc ).isoformat()
    try:
        ncs.terminateInstances( authToken, instanceIds )
        logger.debug( 'terminateInstances returned' )
    except Exception as exc:
        logger.warning( 'got exception terminating %d instances (%s) %s',
            len( instanceIds ), type(exc), exc )
        try:
            with open( terminationLogFilePath, 'a' ) as logFile:
                for iid in instanceIds:
                    print( dateTimeStr, iid, sep=',', file=logFile )
        except Exception as exc:
            logger.warning( 'got exception (%s) appending to terminationLogFile %s',
                type(exc), terminationLogFilePath )
        return []  # empty list meaning none may have been terminated
    else:
        return instanceIds

def purgeHostKeys( instanceRecs ):
    '''try to purgeKnownHosts; warn if any exception'''
    logger.info( 'purgeKnownHosts for %d instances', len(instanceRecs) )
    try:
        ncs.purgeKnownHosts( instanceRecs )
    except Exception as exc:
        logger.warning( 'exception from purgeKnownHosts (%s) %s', type(exc), exc, exc_info=True )
        return 1
    else:
        return 0

def recruitInstances( nWorkersWanted, launchedJsonFilePath, launchWanted,
    resultsLogFilePath, installerFileName ):
    '''launch instances and install client on them;
        terminate those that could not install; return list of good instances'''
    logger.info( 'recruiting up to %d instances', nWorkersWanted )
    goodInstances = []
    if launchWanted:
        nAvail = ncs.getAvailableDeviceCount( args.authToken, filtersJson=args.filter )
        if nWorkersWanted > (nAvail + 0):
            logger.error( 'not enough devices available (%d requested, %d avail)', nWorkersWanted, nAvail )
            raise ValueError( 'not enough devices available')
        # upload an sshClientKey for launch (unless one was provided)
        if args.sshClientKeyName:
            sshClientKeyName = args.sshClientKeyName
        else:
            keyContents = loadSshPubKey().strip()
            randomPart = str( uuid.uuid4() )[0:13]
            #keyContents += ' #' + randomPart
            sshClientKeyName = 'geth_%s' % (randomPart)
            respCode = ncs.uploadSshClientKey( args.authToken, sshClientKeyName, keyContents )
            if respCode < 200 or respCode >= 300:
                logger.warning( 'ncs.uploadSshClientKey returned %s', respCode )
                sys.exit( 'could not upload SSH client key')
        #launch
        rc = launchInstances( args.authToken, nWorkersWanted,
            sshClientKeyName, launchedJsonFilePath, filtersJson=args.filter,
            encryptFiles = args.encryptFiles
            )
        if rc:
            logger.info( 'launchInstances returned %d', rc )
        # delete sshClientKey only if we just uploaded it
        if sshClientKeyName != args.sshClientKeyName:
            logger.info( 'deleting sshClientKey %s', sshClientKeyName)
            ncs.deleteSshClientKey( args.authToken, sshClientKeyName )
    launchedInstances = []
    # get instances from the launched json file
    with open( launchedJsonFilePath, 'r') as jsonInFile:
        try:
            launchedInstances = json.load(jsonInFile)  # an array
        except Exception as exc:
            logger.warning( 'could not load json (%s) %s', type(exc), exc )
    if len( launchedInstances ) < nWorkersWanted:
        logger.warning( 'could not launch as many instances as wanted (%d vs %d)',
            len( launchedInstances ), nWorkersWanted )
    nonstartedIids = [inst['instanceId'] for inst in launchedInstances if inst['state'] != 'started' ]
    if nonstartedIids:
        logger.warning( 'terminating non-started instances %s', nonstartedIids )
        terminateNcsScInstances( args.authToken, nonstartedIids )
        logger.info( 'done terminating non-started instances' )
    # proceed with instances that were actually started
    startedInstances = [inst for inst in launchedInstances if inst['state'] == 'started' ]
    if not startedInstances:
        return ([], [])
    # add instances to knownHosts
    with open( os.path.expanduser('~/.ssh/known_hosts'), 'a' ) as khFile:
        jsonToKnownHosts.jsonToKnownHosts( startedInstances, khFile )

    if not sigtermSignaled():
        installerCmd = '%s %s'% (installerFileName, args.configName )
        logger.info( 'calling tellInstances to install on %d instances', len(startedInstances))
        stepStatuses = tellInstances.tellInstances( startedInstances, installerCmd,
            resultsLogFilePath=resultsLogFilePath,
            download=None, downloadDestDir=None, jsonOut=None, sshAgent=args.sshAgent,
            timeLimit=args.timeLimit, upload=args.uploads, stopOnSigterm=True,
            knownHostsOnly=False
            )
        # COULD restore our handler because tellInstances may have overridden it
        #signal.signal( signal.SIGTERM, sigtermHandler )
        if not stepStatuses:
            logger.warning( 'no statuses returned from installer')
            startedIids = [inst['instanceId'] for inst in startedInstances]
            #logOperation( 'terminateBad', startedIids, '<recruitInstances>' )
            terminateNcsScInstances( args.authToken, startedIids )
            return ([], [])
        # separate good tellInstances statuses from bad ones
        goodIids = []
        badStatuses = []
        for status in stepStatuses:
            if isinstance( status['status'], int) and status['status'] == 0:
                goodIids.append( status['instanceId'])
            else:
                badStatuses.append( status )
                if isinstance( status['status'], asyncio.TimeoutError ):
                    logger.info( 'installer status asyncio.TimeoutError' )

        logger.info( '%d good installs, %d bad installs', len(goodIids), len(badStatuses) )
        #logger.info( 'stepStatuses %s', stepStatuses )
        goodInstances = [inst for inst in startedInstances if inst['instanceId'] in goodIids ]
        badIids = []
        for status in badStatuses:
            badIids.append( status['instanceId'] )
        if badIids:
            #logOperation( 'terminateBad', badIids, '<recruitInstances>' )
            terminateNcsScInstances( args.authToken, badIids )

    return goodInstances, badStatuses

def loadSshPubKey():
    '''returns the contents of current user public ssh client key'''
    pubKeyFilePath = os.path.expanduser( '~/.ssh/id_rsa.pub' )
    with open( pubKeyFilePath ) as inFile:
        contents = inFile.read()
    return contents

def lastGenDateTime( coll ):
    '''return modification dateTime of a mongo collection, as a (hopefully) tz-aware datetime'''
    found = coll.find().sort([( '$natural', -1 )]).limit(1)
    if found:
        return found[0]['_id'].generation_time
    return None

def getLiveInstances( startedIids, authToken ):
    '''query cloudserver to see which instances are still live'''
    logger.info( 'querying to see which instances are still live' )
    liveInstances = []
    #deadInstances = []
    for iid in startedIids:
        reqParams = {"show-device-info":True}
        try:
            response = ncs.queryNcsSc( 'instances/%s' % iid, authToken, reqParams=reqParams, maxRetries=1)
        except Exception as exc:
            logger.warning( 'querying instance status got exception (%s) %s',
                type(exc), exc )
        else:
            if response['statusCode'] != 200:
                logger.warning( 'cloud server returned bad status code %s', response['statusCode'] )
                continue
            inst = response['content']
            instState = inst['state']
            if instState not in ['started', 'stopped']:
                logger.info( 'state "%s" for instance %s', instState, iid )
            if 'instanceId' not in inst:
                inst['instanceId'] = iid
            if instState == 'started':
                liveInstances.append( inst )
            else:
                logger.info( 'state "%s" for instance %s', instState, iid )
            #    deadInstances.append( inst )
    return liveInstances

def launchEdgeNodes( dataDirPath, db, args ):
    '''recruits edge nodes, updates database; may sys.exit'''
    startDateTime = datetime.datetime.now( datetime.timezone.utc )
    dateTimeTagFormat = '%Y-%m-%d_%H%M%S'  # cant use iso format dates in filenames because colons
    dateTimeTag = startDateTime.strftime( dateTimeTagFormat )

    launchedJsonFilePath = os.path.join( dataDirPath, 'launched_%s.json' % dateTimeTag )
    resultsLogFilePath = os.path.join( dataDirPath, 'startGeth_%s.jlog' % dateTimeTag )

    logger.info( 'the launch filter is %s', args.filter )
    if args.count and (args.count > 0):
        nToLaunch = args.count
    elif args.target and (args.target > 0):
        logger.info( 'launcher target: %s', args.target )
        # try to reach target by counting existing working instances and launching more
        mQuery =  {'state': 'checked' }
        nExisting = db['checkedInstances'].count_documents( mQuery )
        logger.info( '%d workers checked', nExisting )
        nToLaunch = int( (args.target - nExisting) * 3 )
        nToLaunch = max( nToLaunch, 0 )

        nAvail = ncs.getAvailableDeviceCount( args.authToken, filtersJson=args.filter )
        logger.info( '%d devices available', nAvail )

        nToLaunch = min( nToLaunch, nAvail )
        nToLaunch = min( nToLaunch, 60 )
    else:
        sys.exit( 'error: no positive --count or --target supplied')
    logger.info( 'would launch %d instances', nToLaunch )
    if nToLaunch <= 0:
        logger.debug( 'NOT launching additional instances' )
        return
    logger.info( 'local files %s and %s', launchedJsonFilePath, resultsLogFilePath )

    if not args.installerFileName:
        logger.error( 'no installer script file name; use --installerFileName to supply one' )
        raise ValueError( 'no installer script file name' )
    if not os.path.isfile( args.installerFileName ):
        logger.error( 'installer script file not found (%s)', args.installerFileName )
        raise ValueError( 'installer script file not found' )
    # launch and install, passing name of installer script to upload and run
    (goodInstances, badStatuses) = recruitInstances( nToLaunch, launchedJsonFilePath, True,
        resultsLogFilePath, args.installerFileName )
    if nToLaunch:
        collName = 'launchedInstances'
        logger.info( 'ingesting into %s', collName )
        ingestJson( launchedJsonFilePath, dbName, collName, append=True )
        # remove the launchedJsonFile to avoid local accumulation of data
        os.remove( launchedJsonFilePath )
        if not os.path.isfile( resultsLogFilePath ):
            logger.warning( 'no results file %s', resultsLogFilePath )
        else:
            ingestJson( resultsLogFilePath, dbName, 'startGeth_'+dateTimeTag, append=False )
            # remove the resultsLogFile to avoid local accumulation of data
            #os.remove( resultsLogFilePath )
        for statusRec in badStatuses:
            statusRec['status'] = str(statusRec['status'])
            statusRec['dateTime'] = startDateTime.isoformat()
            statusRec['_id' ] = statusRec['instanceId']
            db['badInstalls'].insert_one( statusRec )
    else:
        logger.warning( 'no instances recruited' )
    return

def saveErrorsByIid( errorsByIid, db, operation=None ):
    checkedColl = db.checkedInstances
    errorsColl = db.instanceErrors
    checkedDateTime = datetime.datetime.now( datetime.timezone.utc )
    checkedDateTimeStr = checkedDateTime.isoformat()

    for iid, err in errorsByIid.items():
        record = { 'instanceId': iid, 'checkedDateTime': checkedDateTimeStr, 'operation': operation }
        if 'discrep' in err:
            discrep = err['discrep']
            logger.debug( 'updating clockOffset %.1f for inst %s', discrep, iid[0:8] )
            checkedColl.update_one( {'_id': iid}, { "$set": { "clockOffset": discrep } } )
            checkedColl.update_one( {'_id': iid}, { "$inc": { "nFailures": 1 } } )
            record.update( { 'status': 'badClock', 'returnCode': discrep } )
            errorsColl.insert_one( record )
        elif 'status' in err:
            status = err['status']
            if isinstance( status, Exception ):
                checkedColl.update_one( {'_id': iid}, { "$inc": { "nExceptions": 1 } } )
                excType = type(status).__name__  # the name of the particular exception class
                checkedColl.update_one( {'_id': iid}, { "$set": { "exceptionType": excType } } )
                record.update({ 'status': 'exception', 'exceptionType': excType, 'msg': str(status.args) })
                errorsColl.insert_one( record )
            else:
                checkedColl.update_one( {'_id': iid}, { "$inc": { "nFailures": 1 } } )
                record.update({ 'status': 'error', 'returnCode': status })
                errorsColl.insert_one( record )
        else:
            logger.warning( 'checkInstanceClocks instance %s gave status (%s) "%s"', iid[0:8], type(err), err )
            record.update({ 'status': 'misc', 'msg': str(err) })
            errorsColl.insert_one( record )
        # change (or omit) this if wanting to allow more than one failure before marking it failed
        checkedInst = checkedColl.find_one( {'_id': iid }, {'ssh':0} )
        if checkedInst['nExceptions'] > checkedInst['nFailures']:
            state = 'excepted'
        else:
            state = 'failed'
        checkedColl.update_one( {'_id': iid}, { "$set": { "state": state } } )

def checkGethProcesses( instances, dataDirPath ):
    logger.info( 'checking %d instance(s)', len(instances) )

    cmd = "ps -ef | grep -v grep | grep 'geth' > /dev/null"
    # check for a running geth process on each instance
    stepStatuses = tellInstances.tellInstances( instances, cmd,
        timeLimit=15*60,
        resultsLogFilePath = dataDirPath + '/checkGethProcesses.jlog',
        knownHostsOnly=True, stopOnSigterm=True
        )
    #logger.info( 'proc statuses: %s', stepStatuses )
    errorsByIid = {status['instanceId']: status for status in stepStatuses if status['status'] }
    logger.info( 'errorsByIid: %s', errorsByIid )
    return errorsByIid

def checkInstanceClocks( liveInstances, dataDirPath ):
    jlogFilePath = dataDirPath + '/checkInstanceClocks.jlog'
    allIids = [inst['instanceId'] for inst in liveInstances ]
    unfoundIids = set( allIids )
    cmd = "date --iso-8601=seconds"
    # check for a running geth process on each instance
    stepStatuses = tellInstances.tellInstances( liveInstances, cmd,
        timeLimit=2*60,
        resultsLogFilePath = jlogFilePath,
        knownHostsOnly=True, sshAgent=not True, stopOnSigterm=True
        )
    #logger.info( 'proc statuses: %s', stepStatuses )
    errorsByIid = {status['instanceId']: status for status in stepStatuses if status['status'] }
    #for iid, status in errorsByIid.items():
    #    logger.warning( 'instance %s gave error "%s"', iid, status )

    with open( jlogFilePath, 'rb' ) as inFile:
        for line in inFile:
            decoded = json.loads( line )
            iid = decoded['instanceId']
            if decoded.get( 'stdout' ):
                #logger.info( decoded )
                masterDateTime = dateutil.parser.parse( decoded['dateTime'] )
                try:
                    nodeDateTime = dateutil.parser.parse( decoded['stdout'] )
                except Exception as exc:
                    logger.warning( 'exception parsing %s', decoded['stdout'] )
                    errorsByIid[ iid ] = {'exception': exc }
                else:
                    unfoundIids.discard( iid )
                    delta = masterDateTime - nodeDateTime
                    discrep =delta.total_seconds()
                    logger.debug( 'discrep: %.1f seconds on inst %s',
                        discrep, iid )
                    if discrep > 4 or discrep < -1:
                        #logger.warning( 'bad time discrep: %.1f', discrep )
                        errorsByIid[ iid ] = {'discrep': discrep }
    if unfoundIids:
        logger.warning( 'unfoundIids: %s', unfoundIids )
        for iid in list( unfoundIids ):
            if iid not in errorsByIid:
                errorsByIid[ iid ] = {'found': False }
    logger.info( '%d errorsByIid: %s', len(errorsByIid), errorsByIid )
    return errorsByIid

def retrieveLogs( goodInstances, dataDirPath, farmDirPath ):
    '''retrieve logs from nodes, storing them in dataDirPath/nodeLogs subdirectories'''
    logger.info( 'retrieving logs for %d instance(s)', len(goodInstances) )

    nodeLogFilePath = farmDirPath + '/geth.*log'

    # download the log file from each instance
    stepStatuses = tellInstances.tellInstances( goodInstances,
        download=nodeLogFilePath, downloadDestDir=dataDirPath +'/nodeLogs',
        timeLimit=15*60,
        knownHostsOnly=True, stopOnSigterm=True
        )
    #logger.info( 'download statuses: %s', stepStatuses )
    errorsByIid = {status['instanceId']: status['status'] for status in stepStatuses if status['status'] }
    return errorsByIid

def checkLogs( liveInstances, dataDirPath ):
    '''check contents of logs from nodes, save errorSummary.csv, return errorsByIid'''
    logsDirPath = os.path.join( dataDirPath, 'nodeLogs' )
    logDirs = os.listdir( logsDirPath )
    logger.info( '%d logDirs found', len(logDirs ) )

    liveIids = [inst['instanceId'] for inst in liveInstances]

    errorsByIid = {}

    for logDir in logDirs:
        iid = logDir
        iidAbbrev = iid[0:8]
        if iid not in liveIids:
            logger.debug( 'not checking non-live instance %s', iidAbbrev )
            continue
        logFilePath = os.path.join( logsDirPath, logDir, 'geth.log' )
        if not os.path.isfile( logFilePath ):
            logger.warning( 'no log file %s', logFilePath )
        elif os.path.getsize( logFilePath ) <= 0:
            logger.warning( 'empty log file "%s"', logFilePath )
        else:
            connected = False
            sealed = False
            latestSealLine = None
            with open( logFilePath ) as logFile:
                try:
                    for line in logFile:
                        line = line.rstrip()
                        if not line:
                            continue
                        if 'ERROR' in line:
                            if 'Ethereum peer removal failed' in line:
                                logger.debug( 'ignoring "peer removal failed" for %s', iidAbbrev )
                                continue
                            #logger.info( '%s: %s', iidAbbrev, line )
                            theseErrors = errorsByIid.get( iid, [] )
                            theseErrors.append( line )
                            errorsByIid[iid] = theseErrors
                        elif 'Started P2P networking' in line:
                            connected = True
                        elif 'Successfully sealed' in line:
                            sealed = True
                            latestSealLine = line
                    if not connected:
                        logger.warning( 'instance %s did not initialize fully', iidAbbrev )
                    if sealed:
                        #logger.info( 'instance %s has successfully sealed', iidAbbrev )
                        logger.debug( '%s latestSealLine: %s', iidAbbrev, latestSealLine )
                        pat = r'\[(.*\|.*)\]'
                        dateStr = re.search( pat, latestSealLine ).group(1)
                        if dateStr:
                            dateStr = dateStr.replace( '|', ' ' )
                            latestSealDateTime = dateutil.parser.parse( dateStr )
                            logger.debug( 'latestSealDateTime: %s for %s', latestSealDateTime.isoformat(), iid[0:8] )
                except Exception as exc:
                    logger.warning( 'caught exception (%s) %s', type(exc), exc )
    logger.info( 'found errors for %d instance(s)', len(errorsByIid) )
    #print( 'errorsByIid', errorsByIid  )
    summaryCsvFilePath = os.path.join( dataDirPath, 'errorSummary.csv' )
    fileExisted = os.path.isfile( summaryCsvFilePath )
    with open( summaryCsvFilePath, 'a' ) as csvOutFile:
        if not fileExisted:
            print( 'instanceId', 'msg',
                sep=',', file=csvOutFile
                )
        for iid in errorsByIid:
            lines = errorsByIid[iid]
            logger.debug( 'saving %d errors for %s', len(lines), iid[0:8])
            for line in lines:
                print( iid, line,
                    sep=',', file=csvOutFile
                )
    return errorsByIid

def collectGethLogEntries( db, inFilePath, collName ):
    '''ingest records from a geth node log into a new or existing mongo collection'''
    collection = db[ collName ]
    nPreexisting = collection.count_documents( {} )
    logger.debug( '%d preexisting records in %s', nPreexisting, collName )

    lastExistingDateTime = None
    lastExisting = {}
    latestMsgs = []
    if nPreexisting > 0:
        # get the record with the latset dateTime
        cursor = collection.find().sort('dateTime', -1).limit(1)
        lastExisting = cursor[0]
        lastExistingDateTime = interpretDateTimeField( lastExisting['dateTime'] )
    if not lastExistingDateTime:
        lastExistingDateTime = datetime.datetime.fromtimestamp( 0, datetime.timezone.utc )
    logger.debug( 'lastExistingDateTime: %s', lastExistingDateTime )
    logger.debug( 'lastExisting: %s', lastExisting )

    if lastExisting:
        found = collection.find({ 'dateTime': lastExisting['dateTime'] })
        allLatest = list( found )
        latestMsgs = [rec['msg'] for rec in allLatest]
        if len( allLatest ) > 1:
            logger.info( '%d records match latest dateTime in %s', len(allLatest), collName )
            logger.debug( 'latestMsgs: %s', latestMsgs )
    datePat = r'\[(.*\|.*)\]'
    lineDateTime = datetime.datetime.fromtimestamp( 0, datetime.timezone.utc )
    lineLevel = None
    recs = []

    with open( inFilePath, 'r' ) as logFile:
        try:
            for line in logFile:
                line = line.rstrip()
                if not line:
                    continue
                levelPart = line.split()[0]
                if levelPart in ['INFO', 'WARN', 'ERROR']:
                    lineLevel = levelPart

                #lineDateTime = None
                dateMatch = re.search( datePat, line )
                dateStr = dateMatch.group(1) if dateMatch else None
                if dateStr:
                    dateStr = dateStr.replace( '|', ' ' )
                    lineDateTime = universalizeDateTime( dateutil.parser.parse( dateStr ) )
                if lineDateTime < lastExistingDateTime:
                    #logger.info( 'skipping old' )
                    continue
                msg = line
                if lineDateTime == lastExistingDateTime and msg in latestMsgs:
                    logger.debug( 'skipping dup' )
                    continue
                #logger.info( 'appending: %s', msg[26:56])
                recs.append({
                    'dateTime': lineDateTime.isoformat(),
                    'level': lineLevel,
                    'msg': msg,
                    })
                #logger.info( 'rec: %s', recs[-1] )
        except Exception as exc:
            logger.warning( 'exception (%s) %s', type(exc), exc )
        if len(recs):
            #startTime = time.time()
            collection.insert_many( recs, ordered=True )
            #logger.info( 'inserted %d records in %.1f seconds', len(recs), time.time()-startTime )
            if not nPreexisting:
                collection.create_index( 'dateTime' )
                collection.create_index( 'level' )

def ingestGethLog( logFile, coll ):
    '''ingest records from a geth node log into a mongo collection'''
    logger.debug( 'would ingest %s into %s', logFile.name, coll.name )
    datePat = r'\[(.*\|.*)\]'
    lineDateTime = datetime.datetime.fromtimestamp( 0, datetime.timezone.utc )
    lineLevel = None
    recs = []
    for line in logFile:
        line = line.rstrip()
        if not line:
            continue
        levelPart = line.split()[0]
        if levelPart in ['INFO', 'WARN', 'ERROR']:
            lineLevel = levelPart

        #lineDateTime = None
        dateMatch = re.search( datePat, line )
        dateStr = dateMatch.group(1) if dateMatch else None
        if dateStr:
            dateStr = dateStr.replace( '|', ' ' )
            lineDateTime = dateutil.parser.parse( dateStr )
        msg = line
        recs.append( {
            'dateTime': lineDateTime.isoformat(),
            'level': lineLevel,
            'msg': msg,
            } )
    if len(recs):
        coll.insert_many( recs, ordered=True )
        coll.create_index( 'dateTime' )
        coll.create_index( 'level' )

def collectGethJLogEntries( db, inFilePath, collName ):
    collection = db[ collName ]
    nPreexisting = collection.count_documents( {} )
    logger.info( '%d preexisting records in %s', nPreexisting, collName )

    lastExistingDateStr = None
    lastExisting = {}
    latestMsgs = []
    if nPreexisting > 0:
        # get the record with the latset dateTime
        cursor = collection.find().sort('dateTime', -1).limit(1)
        lastExisting = cursor[0]
        lastExistingDateStr = lastExisting['dateTime']
    if not lastExistingDateStr:
        lastExistingDateStr = datetime.datetime.fromtimestamp( 0, datetime.timezone.utc ).isoformat()
    logger.debug( 'lastExistingDateStr: %s', lastExistingDateStr )
    logger.debug( 'lastExisting: %s', lastExisting )

    if lastExisting:
        found = collection.find({ 'dateTime': lastExisting['dateTime'] })
        allLatest = list( found )
        latestMsgs = [rec['msg'] for rec in allLatest]
        if len( allLatest ) > 1:
            logger.info( '%d records match latest dateTime', len(allLatest) )
            logger.info( 'latestMsgs: %s', latestMsgs )
    recs = []

    with open( inFilePath, 'r' ) as logFile:
        for line in logFile:
            line = line.rstrip()
            if not line:
                continue
            try:
                rec = json.loads( line )
                rec['level'] = rec.pop( 'lvl' ).upper()
                rec['dateTime'] = rec.pop( 't' ).upper()

                lineDateStr = rec['dateTime']
                if lineDateStr < lastExistingDateStr:
                    #logger.info( 'skipping old' )
                    continue
                msg = rec['msg']
                if lineDateStr == lastExistingDateStr and msg in latestMsgs:
                    logger.info( 'skipping dup' )
                    continue

                recs.append( rec )
            except Exception as exc:
                logger.warning( 'could not parse json (%s) %s', type(exc), exc )
                continue
        if len(recs):
            logger.info( 'inserting %d records', len(recs) )
            collection.insert_many( recs, ordered=True )
            if not nPreexisting:
                collection.create_index( 'dateTime' )
                collection.create_index( 'level' )

def processNodeLogFiles( dataDirPath ):
    '''ingest all new or updated node logs; delete very old log files'''
    logsDirPath = os.path.join( dataDirPath, 'nodeLogs' )
    logDirs = os.listdir( logsDirPath )
    logger.debug( '%d logDirs found', len(logDirs ) )
    # but first, delete very old log files
    lookbackDays = 7
    thresholdDateTime = datetime.datetime.now( datetime.timezone.utc ) \
        - datetime.timedelta( days=lookbackDays )
    #lookBackHours = 24
    #newerThresholdDateTime = datetime.datetime.now( datetime.timezone.utc ) \
    #    - datetime.timedelta( hours=lookBackHours )

    for logDir in logDirs:
        inFilePath = os.path.join( logsDirPath, logDir, 'geth.jlog' )
        if not os.path.isfile( inFilePath ):
            inFilePath = os.path.join( logsDirPath, logDir, 'geth.log' )
        if os.path.isfile( inFilePath ):
            fileModDateTime = datetime.datetime.fromtimestamp( os.path.getmtime( inFilePath ) )
            fileModDateTime = universalizeDateTime( fileModDateTime )
            if fileModDateTime <= thresholdDateTime:
                os.remove( inFilePath )
                logDirPath = os.path.join( logsDirPath, logDir )
                try:
                    os.rmdir( logDirPath )
                except Exception as exc:
                    logger.warning( 'exception (%s) removing log dir %s', type(exc), logDirPath )
            #elif fileModDateTime <= newerThresholdDateTime:
            #    logger.warning( 'old fileModDateTime for %s (%s)', logDir, fileModDateTime )

    # ingest all new or updated node logs
    loggedCollNames = db.list_collection_names(
        filter={ 'name': {'$regex': r'^nodeLog_.*'} } )
    iStartTime = time.time()
    nIngested = 0
    for logDir in logDirs:
        # logDir is also the instanceId
        # read geth.jlog, if present, otherwise geth.log
        inFilePath = os.path.join( logsDirPath, logDir, 'geth.jlog' )
        if not os.path.isfile( inFilePath ):
            inFilePath = os.path.join( logsDirPath, logDir, 'geth.log' )
        if not os.path.isfile( inFilePath ):
            logger.info( 'no file "%s"', inFilePath )
            continue
        elif os.path.getsize( inFilePath ) <= 0:
            logger.warning( 'empty log file "%s"', inFilePath )
            continue
        collName = 'nodeLog_' + logDir
        if collName in loggedCollNames:
            existingGenTime = lastGenDateTime( db[collName] ) - datetime.timedelta(minutes=1)
            fileModDateTime = datetime.datetime.fromtimestamp( os.path.getmtime( inFilePath ) )
            fileModDateTime = universalizeDateTime( fileModDateTime )
            if existingGenTime >= fileModDateTime:
                #logger.info( 'already posted %s %s %s',
                #    logDir[0:8], fmtDt( existingGenTime ), fmtDt( fileModDateTime )  )
                continue
        logger.debug( 'ingesting log for %s', logDir[0:16] )
        try:
            if inFilePath.endswith( '.jlog' ):
                collectGethJLogEntries( db, inFilePath, collName )
            else:
                collectGethLogEntries( db, inFilePath, collName )
            nIngested += 1
        except Exception as exc:
            logger.warning( 'exception (%s) ingesting %s', type(exc), inFilePath, exc_info=not False )
    logger.info( 'ingesting %d logs took %.1f seconds', nIngested, time.time()-iStartTime)

def checkEdgeNodes( dataDirPath, db, args ):
    '''checks status of edge nodes, updates database'''
    if not db.list_collection_names():
        logger.warning( 'no collections found for db %s', dbName )
        return
    checkerTimeLimit = args.timeLimit
    startedInstances = getStartedInstances( db )

    instancesByIid = {inst['instanceId']: inst for inst in startedInstances }

    wereChecked = list( db['checkedInstances'].find() ) # fully iterates the cursor, getting all records
    checkedByIid = { inst['_id']: inst for inst in wereChecked }
    # after checking, each checked instance will have "state" set to "checked", "failed", "excepted", or "terminated"
    # "checkable" instances are ones that are started, not badly installed, and not badly checked
    checkables = []
    for iid, inst in instancesByIid.items():
        if iid in checkedByIid and checkedByIid[iid]['state'] != 'checked':
            pass
        elif db['badInstalls'].find_one( {'_id': iid }):
            pass
            #logger.info('skipping badinstalled instance %s', iid )
        else:
            checkables.append( inst )
    logger.info( '%d instances checkable', len(checkables) )
    checkedDateTime = datetime.datetime.now( datetime.timezone.utc )
    checkedDateTimeStr = checkedDateTime.isoformat()
    dateTimeTagFormat = '%Y-%m-%d_%H%M%S'  # cant use iso format dates in filenames because colons
    dateTimeTag = checkedDateTime.strftime( dateTimeTagFormat )

    # find out which checkable instances are live
    checkableIids = [inst['instanceId'] for inst in checkables ]
    liveInstances = getLiveInstances( checkableIids, args.authToken )
    liveIidSet = set( [inst['instanceId'] for inst in liveInstances ] )

    coll = db['checkedInstances']
    # update a checkedInstances record for each checkable instance, creating any that do not exist
    for inst in checkables:
        iid = inst['instanceId']
        abbrevIid = iid[0:16]
        launchedDateTimeStr = inst.get( 'started-at' )
        instCheck = checkedByIid.get( iid )
        if instCheck:
            nFailures = instCheck.get( 'nFailures', 0)
            reasonTerminated = instCheck.get( 'reasonTerminated', None)
        else:
            nFailures = 0
            reasonTerminated = None
            coll.insert_one( {'_id': iid, 'instanceId': iid,
                'state': 'started',
                'devId': inst.get('device-id'),
                'launchedDateTime': launchedDateTimeStr,
                'ssh': inst.get('ssh'),
                'nFailures': 0,
                'nExceptions': 0,
            } )
        if iid in liveIidSet:
            state = 'checked'
        else:
            state = 'terminated'
            nFailures += 1
            reasonTerminated = 'foundDead'
        coll.update_one( {'_id': iid},
            { "$set": { 'instanceId': iid, 'state': state, 'nFailures': nFailures,
                'reasonTerminated': reasonTerminated,
                'checkedDateTime': checkedDateTimeStr } },
            upsert=True
            )
    checkables = [inst for inst in checkables if inst['instanceId'] in liveIidSet]
    logger.info( '%d instances checkable', len(checkables) )
    errorsByIid = checkInstanceClocks( liveInstances, dataDirPath )
    saveErrorsByIid( errorsByIid, db, operation='checkInstanceClocks' )

    checkables = [inst for inst in checkables if inst['instanceId'] not in errorsByIid]
    logger.info( '%d instances checkable', len(checkables) )
    errorsByIid = checkGethProcesses( checkables, dataDirPath )
    saveErrorsByIid( errorsByIid, db, operation='checkGethProcesses' )

    checkables = [inst for inst in checkables if inst['instanceId'] not in errorsByIid]
    logger.info( '%d instances checkable', len(checkables) )

    # populate the primaryAccount field of any checked instances that lack it
    errorsByIid = {}
    for inst in checkables:
        iid = inst['instanceId']
        instCheck = checkedByIid.get( iid )
        if not instCheck or 'primaryAccount' not in instCheck:
            logger.info( 'getting primary account of inst %s', iid[0:16])
            instanceAccountPairs = ncsgeth.collectPrimaryAccounts( [inst], args.configName )
            if not instanceAccountPairs:
                # this shouldn't happen, because you should at least get a status return
                logger.error( 'no account info returned for inst %s', iid )
            elif instanceAccountPairs[0].get('status'):
                # an error (or exception) getting the account
                errorsByIid[ iid ] = { 'instanceIid': iid,
                    'status': instanceAccountPairs[0].get('status')
                    }
            else:
                accountAddr = instanceAccountPairs[0].get('accountAddr')
                if accountAddr:
                    coll.update_one( {'_id': iid}, { "$set": { "primaryAccount": accountAddr } } )
    logger.info( '(collectPrimaryAccounts) errorsByIid %s', errorsByIid )
    saveErrorsByIid( errorsByIid, db, operation='collectPrimaryAccounts' )

    nodeDataDirPath = 'ether/%s' % args.configName
    errorsByIid = retrieveLogs( checkables, dataDirPath, nodeDataDirPath )
    #checkLogs( checkables, dataDirPath )
    processNodeLogFiles( dataDirPath )
    return

def collectSigners( db, configName ):
    anchorInstances = list( db['anchorInstances'].find() ) # fully iterates the cursor, getting all records
    coll = db['checkedInstances']
    wereChecked = list( coll.find({'state': 'checked'} ) )
    for inst in wereChecked:
        inst['instanceId'] = inst['_id']
    instances = anchorInstances + wereChecked

    # get the currently authorized signers (and true proposees)
    signerInfos = ncsgeth.collectSignerInstances( instances, configName )
    logger.debug( 'signerInfos: %s', signerInfos )
    coll = db['allSigners']
    logger.info( 'saving signers')
    for signer in signerInfos:
        signerId = signer['accountAddr']
        iid = signer['instanceId']
        auth = signer['auth']
        logger.info( 'signer acct %s: iid %s, auth: %s', signerId, iid, auth )
        coll.update_one( {'_id': iid}, { "$set":
            { "instanceId": iid, "accountAddr": signerId, "auth": auth } }, upsert=True
            )

def waitForAuth( victimAccount, shouldAuth, instances, configName, timeLimit ):
    '''wait for the authorization of victimAccount to match shouldAuth; return success (true or false)'''
    curSigners = ncsgeth.collectAuthSigners( instances, configName )
    nowAuth = victimAccount in curSigners
    logger.info( 'now authorized? %s', nowAuth )
    deadline = time.time() + timeLimit
    while nowAuth != shouldAuth:
        if time.time() >= deadline:
            logger.warning( 'took too long for auth/deauth to propagate')
            break
        curSigners = ncsgeth.collectAuthSigners( instances, configName )
        nowAuth = victimAccount in curSigners
        logger.info( 'now authorized? %s', nowAuth )
        if nowAuth != shouldAuth:
            time.sleep(1)
    return nowAuth == shouldAuth

def authorizeSigner( db, configName, victimIid, shouldAuth ):
    logger.info( 'shouldAuth: %s', shouldAuth )

    allSigners = list( db['allSigners'].find() )
    logger.info( 'len(allSigners): %d', len(allSigners) )
    signersByIid = {signer['instanceId']: signer for signer in allSigners }
    logger.debug( 'signer iids: %s', signersByIid.keys() )

    wasAuth = (victimIid in signersByIid) and signersByIid[victimIid].get('auth)')
    if wasAuth == shouldAuth:
        logger.warning( 'instance %s auth status is already %s', victimIid, shouldAuth )
        #return

    wereChecked = list( db['checkedInstances'].find( {'state': 'checked'}) ) # fully iterates the cursor, getting all records
    logger.info( '%d instances were checked', len(wereChecked))
    for inst in wereChecked:
        inst['instanceId'] = inst['_id']
    anchorInstances = list( db['anchorInstances'].find() ) # fully iterates the cursor, getting all records

    instances = anchorInstances + wereChecked
    instancesByIid = {inst['instanceId']: inst for inst in instances }

    victimAccount = None
    if victimIid in signersByIid:
        # this instance is (or has been) a signer
        victimAccount = signersByIid[ victimIid ].get( 'accountAddr' )
    else:
        victimInst = instancesByIid[ victimIid ]
        if not victimInst:
            logger.warning( 'instance %s not found', victimIid )
            return
        logger.debug ('victimInst: %s', victimInst )
        instanceAccountPairs = ncsgeth.collectPrimaryAccounts( [victimInst], configName )
        if not instanceAccountPairs or not instanceAccountPairs[0].get( 'accountAddr' ):
            logger.error( 'NOT authorizing, no account found for inst %s', victimIid )
            return
        victimAccount = instanceAccountPairs[0]['accountAddr']

    authorizers = [inst for inst in instances if inst['instanceId'] in signersByIid ]
    logger.info( '%d authorizers', len(authorizers) )
    if not authorizers:
        logger.warning( 'no authorizers found')

    # starting all miners should not be necessary
    #minerResults = ncsgeth.startMiners( authorizers, configName )
    #logger.info( 'all minerResults: %s', minerResults )
    # start miner on the one to be authorized
    if victimIid and victimIid in instancesByIid:
        inst = instancesByIid[ victimIid ]
        minerResults = ncsgeth.startMiners( [inst], configName )
        logger.info( 'minerResults: %s', minerResults )

    authResults = ncsgeth.authorizeSigner( authorizers, configName, victimAccount, shouldAuth )
    logger.debug( 'authResults: %s', authResults )
    anySucceeded = False
    for result in authResults:
        if 'returnCode' in result and result['returnCode'] == 0:
            anySucceeded = True
        else:
            logger.warning( 'bad result from authorizeSigner: %s', result )
    if anySucceeded:
        itWorked = waitForAuth( victimAccount, shouldAuth, authorizers, configName, timeLimit=600 )
        if itWorked:
            db['allSigners'].update_one( {'_id': victimIid}, { "$set":
                { "instanceId": victimIid, "accountAddr": victimAccount, "auth": shouldAuth }
                    }, upsert=True
                )
    return

def authorizeGood( db, configName, maxNewAuths, trustedThresh ):
    # retrieve list of signers so we can know which are already authorized
    allSigners = list( db['allSigners'].find() )
    logger.info( 'len(allSigners): %d', len(allSigners) )
    signersByIid = {signer['instanceId']: signer for signer in allSigners }
    logger.debug( 'signer iids: %s', signersByIid.keys() )

    wereChecked = list( db['checkedInstances'].find( {'state': 'checked'}) ) # fully iterates the cursor, getting all records
    logger.info( '%d instances were checked', len(wereChecked))
    nNewAuths = 0
    for inst in wereChecked:
        if nNewAuths >= maxNewAuths:
            break
        iid = inst['_id']
        logger.debug( 'considering instance %s', iid )
        signerInfo = signersByIid.get(iid)
        if signerInfo and signerInfo.get( 'auth' ):
            logger.debug( 'already authorized, account %s', signerInfo.get('accountAddr') )
            continue
        launchedDateTime = dateutil.parser.parse( inst['launchedDateTime'] )
        checkedDateTime = dateutil.parser.parse( inst['checkedDateTime'] )
        #logger.info( 'uptime: %s', checkedDateTime-launchedDateTime )
        uptimeHrs = (checkedDateTime-launchedDateTime).total_seconds() / 3600
        logger.info( 'uptimeHrs: %.1f', uptimeHrs )
        if uptimeHrs < trustedThresh:
            continue
        logger.info( 'would authorize %s', iid )
        authorizeSigner( db, configName, iid, True )
        nNewAuths += 1
        if nNewAuths < maxNewAuths:
            time.sleep( 15 )

if __name__ == "__main__":
    startTime = time.time()
    # configure logger formatting
    logFmt = '%(asctime)s %(levelname)s %(module)s %(funcName)s %(message)s'
    logDateFmt = '%Y/%m/%d %H:%M:%S'
    formatter = logging.Formatter(fmt=logFmt, datefmt=logDateFmt )
    logging.basicConfig(format=logFmt, datefmt=logDateFmt)
    ncs.logger.setLevel(logging.INFO)
    #runDistributedBlender.logger.setLevel(logging.INFO)
    logger.setLevel(logging.INFO)
    tellInstances.logger.setLevel(logging.WARNING)
    logger.debug('the logger is configured')

    ap = argparse.ArgumentParser( description=__doc__,
        fromfile_prefix_chars='@', formatter_class=argparse.ArgumentDefaultsHelpFormatter )
    ap.add_argument( 'action', help='the action to perform',
        choices=['launch', 'authorizeGood', 'authorizeSigner', 'deauthorizeSigner',
            'check', 'collectSigners', 'importAnchorNodes', 'maintain',
            'terminateBad', 'terminateAll', 'reterminate']
        )
    ap.add_argument( '--authToken', help='the NCS authorization token to use (required for launch or terminate)' )
    ap.add_argument( '--count', type=int, help='the number of instances (for launch)' )
    ap.add_argument( '--target', type=int, help='target number of working instances (for launch)',
        default=2 )
    ap.add_argument( '--configName', help='the name of the network config', default='priv_5' )
    ap.add_argument( '--dataDir', help='data directory', default='./data/' )
    ap.add_argument( '--encryptFiles', type=boolArg, default=False, help='whether to encrypt files on launched instances' )
    ap.add_argument( '--farm', required=True, help='the name of the virtual geth farm' )
    ap.add_argument( '--filter', help='json to filter instances for launch',
        default='{"dpr": ">=51", "ram:": ">=4000000000", "storage": ">=20000000000"}' )
    ap.add_argument( '--installerFileName', help='a script to run on new instances',
        default='netconfig/installAndStartGeth.sh' )
    ap.add_argument( '--instanceId', help='id of instance (for authorize or deauthorize)' )
    ap.add_argument( '--maxNewAuths', type=int, default=0, help='maximum number of new authorized signers (for authorizeGood)' )
    ap.add_argument( '--uploads', help='glob for filenames to upload to workers', default='netconfig' )
    ap.add_argument( '--mongoHost', help='the host of mongodb server', default='localhost')
    ap.add_argument( '--mongoPort', help='the port of mongodb server', default=27017)
    ap.add_argument( '--sshAgent', type=boolArg, default=False, help='whether or not to use ssh agent' )
    ap.add_argument( '--sshClientKeyName', help='the name of the uploaded ssh client key to use (advanced)' )
    ap.add_argument( '--timeLimit', type=int, help='time limit (in seconds) for the whole job',
        default=20*60 )
    args = ap.parse_args()

    logger.info( 'performing action "%s" for farm "%s"', args.action, args.farm )

    dataDirPath = os.path.join( args.dataDir, args.farm )
    os.makedirs( dataDirPath, exist_ok=True )

    mclient = pymongo.MongoClient(args.mongoHost, args.mongoPort)
    dbName = 'geth_' + args.farm
    db = mclient[dbName]

    if args.action == 'launch':
        launchEdgeNodes( dataDirPath, db, args )
    elif args.action in ['authorizeSigner', 'deauthorizeSigner' ]:
        victimIid = args.instanceId
        configName = args.configName
        shouldAuth = args.action == 'authorizeSigner'
        authorizeSigner( db, configName, victimIid, shouldAuth )
    elif args.action == 'authorizeGood':
        # authorize good instances that have been up for long enough
        maxNewAuths = args.maxNewAuths
        if maxNewAuths <= 0:
            sys.exit()
        authorizeGood( db, args.configName, maxNewAuths, trustedThresh=18 )
    elif args.action == 'check':
        checkEdgeNodes( dataDirPath, db, args )
    elif args.action == 'collectSigners':
        collectSigners( db, args.configName )
    elif args.action == 'importAnchorNodes':
        anchorNodesFilePath = os.path.join( dataDirPath, 'anchorInstances.json' )
        importAnchorNodes( anchorNodesFilePath, db, 'anchorInstances' )
    elif args.action == 'maintain':
        myPid = os.getpid()
        logger.info( 'process id %d', myPid )
        signal.signal( signal.SIGTERM, sigtermHandler )
        actionFirst = args.action == sys.argv[1]
        if not actionFirst:
            logger.error( '"%s" was not specified as the first argument', args.action )
            sys.exit(1)
        if mclient:
            mclient.close()
        logger.info( 'argv: %s', sys.argv )
        cmd = sys.argv.copy()
        while sigtermNotSignaled():
            cmd[1] = 'check'
            logger.info( 'cmd: %s', cmd )
            subprocess.check_call( cmd )
            if sigtermNotSignaled():
                cmd[1] = 'authorizeGood'
                logger.info( 'cmd: %s', cmd )
                subprocess.check_call( cmd )
            if sigtermNotSignaled():
                cmd[1] = 'terminateBad'
                logger.info( 'cmd: %s', cmd )
                subprocess.check_call( cmd )
            if sigtermNotSignaled():
                if args.target > 0:
                    cmd[1] = 'launch'
                    logger.info( 'cmd: %s', cmd )
                    subprocess.check_call( cmd )
            if sigtermNotSignaled():
                time.sleep( 30 )

    elif args.action == 'terminateBad':
        if not args.authToken:
            sys.exit( 'error: can not terminate because no authToken was passed')
        terminatedDateTimeStr = datetime.datetime.now( datetime.timezone.utc ).isoformat()
        # retrieve list of signers so we can deauthorize each before terminating
        allSigners = list( db['allSigners'].find() )
        logger.info( 'len(allSigners): %d', len(allSigners) )
        signersByIid = {signer['instanceId']: signer for signer in allSigners }
        logger.debug( 'signer iids: %s', signersByIid.keys() )

        coll = db['checkedInstances']
        wereChecked = list( coll.find() ) # fully iterates the cursor, getting all records
        toTerminate = []  # list of iids
        toPurge = []  # list of instance-like dicts containing instanceId and ssh fields
        for checkedInst in wereChecked:
            state = checkedInst.get( 'state')
            #logger.info( 'checked state %s', state )
            if state in ['failed', 'excepted', 'stopped' ]:
                iid = checkedInst['_id']
                if iid in signersByIid:
                    logger.info( 'terminateBad deauthorizing %s', iid )
                    authorizeSigner( db, args.configName, iid, False )
                abbrevIid = iid[0:16]
                logger.warning( 'would terminate %s', abbrevIid )
                toTerminate.append( iid )
                if 'instanceId' not in checkedInst:
                    checkedInst['instanceId'] = iid
                toPurge.append( checkedInst )
                coll.update_one( {'_id': iid}, { "$set": { 'reasonTerminated': state } } )
        logger.info( 'terminating %d instances', len( toTerminate ))
        terminated=terminateNcsScInstances( args.authToken, toTerminate )
        logger.info( 'actually terminated %d instances', len( terminated ))
        purgeHostKeys( toPurge )
        # update states in checkedInstances
        for iid in terminated:
            coll.update_one( {'_id': iid},
                { "$set": { "state": "terminated",
                    'terminatedDateTime': terminatedDateTimeStr } } )
    elif args.action == 'terminateAll':
        if not args.authToken:
            sys.exit( 'error: can not terminate because no authToken was passed')
        logger.info( 'checking for instances to terminate')
        # will terminate all instances and update checkedInstances accordingly
        startedInstances = getStartedInstances( db )  # expensive, could just query for iids
        startedIids = [inst['instanceId'] for inst in startedInstances]

        logger.info( 'checking for instances to deauthorize')
        # retrieve list of signers so we can deauthorize each before terminating
        allSigners = list( db['allSigners'].find() )
        logger.info( 'len(allSigners): %d', len(allSigners) )
        signersByIid = {signer['instanceId']: signer for signer in allSigners }
        logger.debug( 'signer iids: %s', signersByIid.keys() )

        nToDeauth = 0
        for iid in startedIids:
            if iid in signersByIid and signersByIid[iid].get('auth'):
                nToDeauth += 1
                logger.info( 'terminateAll deauthorizing %s', iid )
                authorizeSigner( db, args.configName, iid, False )
                time.sleep( 30 )
        logger.info( 'deauthorized %d nodes', nToDeauth )

        terminatedDateTimeStr = datetime.datetime.now( datetime.timezone.utc ).isoformat()
        toTerminate = startedIids
        logger.info( 'terminating %d instances', len( toTerminate ))
        terminated=terminateNcsScInstances( args.authToken, toTerminate )
        purgeHostKeys( startedInstances )

        # update states in checkedInstances
        coll = db['checkedInstances']
        wereChecked = coll.find()  # could use a projection to make it more efficient
        checkedByIid = { inst['_id']: inst for inst in wereChecked }
        for iid in terminated:
            if iid in checkedByIid:
                checkedInst = checkedByIid[iid]
                if checkedInst['state'] != 'terminated':
                    coll.update_one( {'_id': iid},
                        { "$set": { "state": "terminated",
                            'reasonTerminated': 'manual',
                            'terminatedDateTime': terminatedDateTimeStr }
                            } )
    elif args.action == 'reterminate':
        '''redundantly terminate instances that are listed as already terminated'''
        if not args.authToken:
            sys.exit( 'error: can not terminate because no authToken was passed')
        # get list of terminated instance IDs
        coll = db['checkedInstances']
        wereChecked = list( coll.find({'state': 'terminated'},
            {'_id': 1, 'terminatedDateTime': 1 }) )
        toTerminate = []
        for checkedInst in wereChecked:
            iid = checkedInst['_id']
            abbrevIid = iid[0:16]
            if checkedInst['terminatedDateTime'] >= '2020-07-24':
                logger.info( 'would reterminate %s from %s', abbrevIid, checkedInst['terminatedDateTime'] )
                toTerminate.append( iid )
        logger.info( 'reterminating %d instances', len( toTerminate ))
        terminated = terminateNcsScInstances( args.authToken, toTerminate )
        logger.info( 'reterminated %d instances', len( terminated ) )
        #TODO purgeHostKeys
    else:
        logger.warning( 'action "%s" unimplemented', args.action )
    elapsed = time.time() - startTime
    logger.info( 'finished action "%s"; elapsed time %.1f minutes',
        args.action, elapsed/60 )


from setuptools import setup
import os

# stole this from ImSim:
def all_files_from(dir, ext=''):
    """Quick function to get all files from directory and all subdirectories
    """
    files = []
    for root, dirnames, filenames in os.walk(dir):
        for filename in filenames:
            if filename.endswith(ext) and not filename.startswith('.'):
                files.append(os.path.join(root, filename))
    return files

data_files = all_files_from('data')

setup(
    name='psfws',
    version='0.1',
    description='generate realistic wind and turbulence parameters for atmospheric point-spread function simulations',
    url='https://github.com/LSSTDESC/psf-weather-station',
    author='Claire-Alice Hebert',
    author_email='chebert@stanford.edu',
    license='MIT',
    packages=['psfws'],
    package_data={'psfws': data_files},
    python_requires='>=3.5'
)


from googlesearch import search
import pyautogui
from time import sleep
#from random import choice, shuffle
import sys
#import os
import webbrowser

linkOne = input("Primeiro link\n >  ")
#linkTwp = input("Segundo link\n >  ")
nviews = int(input("Quantas views deseja?\n >  "))
tamanho_do_video = int(input("Tamanho do vídeo\n >  "))
#tamanho_do_video2 = int(input("Tamanho do vídeo\n >  "))
# tempo de espera personalizado
tempodeespera = int(input("quantos segundos para inicializar\n >  "))
#vou testar se funciona
sleep(tempodeespera)

def bot(link):
    verificador = 'https://'
    #views = 0
    if verificador not in link:
        print("Isto não é um link =(")
        print('Porém será pesquisado =)')
        #nr = int(input("Quantos resultados deseja?\n >  "))
        #webbrowser.open_new_tab(link)
        for resultado in search('"{link}" google', stop=10):
            print(f'Resultado: {resultado}')
        sleep(5)
        dsd = input("Deseja sair? (y/n)\n >  ")
        if dsd == 'y':
            sys.exit()
        elif dsd == 'n':
            print('Você será desligado em 10000000 segundos')
            sleep(10000000)
            sys.exit()
        else:
            print('Comando inválido =(')
            sleep(5)
            sys.exit()
    else:
        if nviews > 1:
            for i in range(nviews):
                pyautogui.sleep(1)
                webbrowser.open_new_tab(linkOne)
                pyautogui.sleep(1)
                pyautogui.press('space')
                pyautogui.sleep(tamanho_do_video)
                pyautogui.sleep(1)
                pyautogui.hotkey("ctrl", "w")
                pyautogui.sleep(10)
                print(f'Número de views atual: {i}')
            print("M>1")
        else:
            print("O número de views é menor ou igual a 1.")
            sleep(5)
            sys.exit()
            #webbrowser.open_new_tab(linkTwp)
            #pyautogui.sleep(1)
            #pyautogui.press("space")
            #pyautogui.sleep(tamanho_do_video2)
            #pyautogui.sleep(1)
            #pyautogui.hotkeys("ctrl", "w")
#fim da função bot
bot(linkOne)
sleep(1)
print("Fim do script!")
sleep(5)
sys.exit()


import sys
from matplotlib import pyplot as plt

from src.support.roadnet import RoadNetwork
from src.support.simsio import Snapshot
from src.support import linkvolio

def main():
    if len(sys.argv) < 3:
        sys.stderr.write(
            "USAGE: "
            + sys.argv[0]
            + " [path to road network GeoJSON file] [path to snapshot data] [path to link volume data]\n"
        )
        sys.exit(1)

    # Set up figure and axes:
    fig = plt.figure()
    ax = fig.add_subplot(111)

    with open(sys.argv[1], "r", encoding="utf-8") as f:
        network = RoadNetwork(f)

    # Load traces and count # of frames per link:
    with open(sys.argv[2], "r", encoding="utf-8") as f:
        snapshot = Snapshot.load(f, ordered=False)
    link_counts = {}

    for road in network.links:
        link_counts[road.id] = set()

    for frame in snapshot.iter_time():
        link_counts[frame.link].add(frame.vid)

    for road in network.links:
        link_counts[road.id] = len(link_counts[road.id])

    # Load link volume snapshot data:
    with open(sys.argv[3], "r", encoding="utf-8") as f:
        vols = linkvolio.link_volumes(f)

    xs = []
    ys = []

    for road in network.links:
        if road.id in link_counts and road.id in vols:
            ys.append(link_counts[road.id])
            xs.append(vols[road.id].link_vol)

    ax.plot(xs, ys, '.')
    ax.set_ylabel("Vehicles on Link (Snapshot Data)")
    ax.set_xlabel("Vehicles on Link (Link Volume Data)")
    ax.set_title("Link Volume Correlation")

    plt.show()

if __name__ == "__main__":
    main()