from django.shortcuts import redirect
from django.views.generic import CreateView

from surveys.models import SurveyAnswer
from surveys.models.annotation import Annotation, Classification, Word
from surveys.models.survey import Survey, Question, Choice, PersonalInformation
from surveys.models.analysis import AnalysisSingle
from surveys.forms.analysis import AnalysisCreator

from django.core import serializers
from ast import literal_eval
from urllib import parse

from ...views.error import permission_user_logged_in, permission_user_owns_survey, permission_user_owns_analysis, \
                            permission_user_owns_annotation
from ...views.helper import get_age_ranges

class AnalysisSingleTerm(CreateView):
    template_name = 'analysis/single.html'
    model = Annotation
    form_class = AnalysisCreator

    def get(self, request, *args, **kwargs):
        permission_user_logged_in(request)

        self.object = None
        form_class = self.get_form_class()
        form = self.get_form(form_class)

        carry_over_terms = []
        carry_over_constraints = {}

        get_variables = request.GET

        if 'analysis' in get_variables:
            analysis = AnalysisSingle.objects.get(pk=get_variables['analysis'])

            permission_user_owns_analysis(request, analysis)

            analysis_name = analysis.name
            analysis_pk = analysis.pk
            survey = Survey.objects.get(pk=analysis.survey.pk)
            annotation = Annotation.objects.get(pk=analysis.annotation.pk)
            operation = "overwrite"

            # this is a little spaghetti, be careful if you're making changes to it
            # if we have no saved terms, just put carry over as an empty list
            if analysis.terms == "":
                carry_over_terms = []
            else:
                # if we have only one saved carry over term
                if isinstance(literal_eval(analysis.terms), int):
                    carry_over_terms = [analysis.terms]
                # if we have many saved carry over terms
                else:
                    carry_over_terms = list(literal_eval(analysis.terms))

            # since carried over constraints are saved in a more verbose way, they experience no issue
            carry_over_constraints = literal_eval(analysis.constraints)

        else:
            analysis_name = get_variables['name']
            analysis_pk = 0
            survey = Survey.objects.get(pk=get_variables['survey'])
            annotation = Annotation.objects.get(pk=get_variables['annotation'])

            permission_user_owns_survey(request, survey)
            permission_user_owns_annotation(request, annotation)

            operation = "save"

        classifications = Classification.objects.filter(annotation=annotation)
        questions = Question.objects.filter(survey=survey)
        choices = Choice.objects.filter(question__in=questions)
        words = Word.objects.filter(classification__in=classifications, choice__in=choices)

        class_ids_used = []
        for w in words.values_list('classification'):
            class_ids_used.append(w[0])

        for classif in classifications:
            if classif.pk not in class_ids_used:
                classifications = classifications.exclude(pk=classif.pk)

        survey_answers = SurveyAnswer.objects.filter(survey=survey)

        pi_js_droplist = {}
        for answer in survey_answers:
            for pi in literal_eval(survey.pi_choices):
                answer_value = getattr(answer.pi_questions, pi)
                pi = pi.replace("_", " ").capitalize()
                if isinstance(answer_value, str):
                    answer_value = answer_value.capitalize()
                if pi in pi_js_droplist:
                    pi_js_droplist[pi].add(answer_value)
                else:
                    pi_js_droplist[pi] = {answer_value}

        pi_js_droplist = get_age_ranges(pi_js_droplist)

        keylist = []
        for k in pi_js_droplist.keys():
            keylist.append(k)

        # this is useful in js
        answers = literal_eval(serializers.serialize("json", survey_answers))
        for answ in answers:
            pi = PersonalInformation.objects.filter(pk=answ['fields']['pi_questions']).values()[0]
            newpi = {}
            for k in pi.keys():
                if pi[k] != '' and k != 'id':
                    nk = k.replace("_", " ").capitalize()
                    newpi[nk] = pi[k]
                    if isinstance(pi[k], str):
                        newpi[nk] = (pi[k]).capitalize()
            answ['fields']['pi_questions'] = newpi

        return self.render_to_response(
            self.get_context_data(form=form,
                                  analysis_name=analysis_name,
                                  survey=survey,
                                  classifications=classifications,
                                  classificiations_js=serializers.serialize("json", classifications),
                                  questions=serializers.serialize("json", questions),
                                  choices=serializers.serialize("json", choices),
                                  words=serializers.serialize("json", words),
                                  pi_choices=literal_eval(survey.pi_choices),
                                  answers=answers,
                                  pi_js_droplist=pi_js_droplist,
                                  constraints_keys=keylist,
                                  operation=operation,
                                  carry_over_terms=carry_over_terms,
                                  carry_over_constraints=carry_over_constraints,
                                  analysis_pk=analysis_pk,
                                  )
        )

    def post(self, request, *args, **kwargs):
        if "delete" in request.POST:
            analysis_pk = request.POST['delete']

            analysis = AnalysisSingle.objects.get(pk=analysis_pk)

            analysis.delete()
            return redirect('./')

        else:
            terms = request.POST['terms']
            constraints = {}

            # transform js serialized dict to a normal py dict
            con_post = parse.parse_qs(request.POST['constraints'])
            for key in con_post.keys():
                nk = key.replace("[]", "")
                constraints[nk] = con_post[key]

            if request.POST['operation'] == "save":

                analysis_name = request.GET['name']
                analysis_survey = Survey.objects.get(pk=request.GET['survey'])
                analysis_annotation = Annotation.objects.get(pk=request.GET['annotation'])

                new_analysis = AnalysisSingle.objects.create(creator=request.user,
                                                             name=analysis_name,
                                                             survey=analysis_survey,
                                                             annotation=analysis_annotation,
                                                             terms=terms,
                                                             constraints=constraints)
                new_analysis.save()

                redirect_pk = new_analysis.pk
                return redirect('./single?analysis=' + str(redirect_pk))

            elif request.POST['operation'] == "overwrite":
                analysis_pk = request.GET['analysis']

                analysis = AnalysisSingle.objects.get(pk=analysis_pk)
                analysis.terms = terms
                analysis.constraints = constraints
                analysis.save()

                redirect_pk = analysis.pk
                return redirect('./single?analysis=' + str(redirect_pk))


# adapted from: https://github.com/pytorch/examples/blob/master/mnist/main.py
import argparse
import torch as pt
import torch.nn as nn
import torch.nn.functional as nnf
import torch.optim as optim
from utils import get_data_loaders

class MnistCNN(nn.Module):
    def __init__(self):
        super(MnistCNN, self).__init__()
        self.conv1 = nn.Conv2d(1, 20, 5, 1)
        self.conv2 = nn.Conv2d(20, 50, 5, 1)
        self.fc1 = nn.Linear(4 * 4 * 50, 500)
        self.fc2 = nn.Linear(500, 10)

    def forward(self, x):
        x = nnf.relu(self.conv1(x))
        x = nnf.max_pool2d(x, 2, 2)
        x = nnf.relu(self.conv2(x))
        x = nnf.max_pool2d(x, 2, 2)
        x = x.view(-1, 4 * 4 * 50)
        x = nnf.relu(self.fc1(x))
        x = self.fc2(x)
        return nnf.log_softmax(x, dim=1)

def train(args, model, device, train_loader, optimizer, epoch):
    model.train()
    for batch_idx, (data, target) in enumerate(train_loader):
        data, target = data.to(device), target.to(device)
        optimizer.zero_grad()
        output = model(data)
        loss = nnf.nll_loss(output, target)
        loss.backward()
        optimizer.step()
        if batch_idx % args.log_interval == 0:
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
                epoch, batch_idx * len(data), len(train_loader.dataset),
                       100. * batch_idx / len(train_loader), loss.item()))

def test(args, model, device, test_loader):
    model.eval()
    test_loss = 0
    correct = 0
    with pt.no_grad():
        for data, target in test_loader:
            data, target = data.to(device), target.to(device)
            output = model(data)
            test_loss += nnf.nll_loss(output, target, reduction='sum').item()  # sum up batch loss
            pred = output.argmax(dim=1, keepdim=True)  # get the index of the max log-probability
            correct += pred.eq(target.view_as(pred)).sum().item()

    test_loss /= len(test_loader.dataset)

    print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
        test_loss, correct, len(test_loader.dataset),
        100. * correct / len(test_loader.dataset)))

def main():
    # Training settings
    parser = argparse.ArgumentParser(description='Pypt MNIST Example')
    parser.add_argument('--batch-size', type=int, default=64)
    parser.add_argument('--test-batch-size', type=int, default=1000)
    parser.add_argument('--epochs', type=int, default=10)
    parser.add_argument('--lr', type=float, default=0.01)
    parser.add_argument('--momentum', type=float, default=0.5)
    parser.add_argument('--no-cuda', action='store_true', default=False)
    parser.add_argument('--seed', type=int, default=1)
    parser.add_argument('--log-interval', type=int, default=60)
    parser.add_argument('--fashion', action='store_true', default=False)
    parser.add_argument('--save-dir', type=str, default=None)
    args = parser.parse_args()
    use_cuda = not args.no_cuda and pt.cuda.is_available()

    pt.manual_seed(args.seed)

    device = pt.device("cuda" if use_cuda else "cpu")

    train_loader, test_loader = get_data_loaders(use_cuda, args, fashion=args.fashion)

    model = MnistCNN().to(device)
    optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum)

    for epoch in range(1, args.epochs + 1):
        train(args, model, device, train_loader, optimizer, epoch)
        test(args, model, device, test_loader)

    if args.save_dir is not None:
        pt.save(model.state_dict(), "softmax_mnist_cnn{}.pt".format('_fashion' if args.fashion else ''))

if __name__ == '__main__':
    main()


import discord
from discord.ext import commands

import discordSuperUtils

bot = commands.Bot(command_prefix="-", intents=discord.Intents.all())
LevelingManager = discordSuperUtils.LevelingManager(bot, award_role=True)
ImageManager = (
    discordSuperUtils.ImageManager()
)  # LevelingManager uses ImageManager to create the rank command.

@bot.event
async def on_ready():
    database = discordSuperUtils.DatabaseManager.connect(...)
    await LevelingManager.connect_to_database(database, ["xp", "roles", "role_list"])

    print("Leveling manager is ready.", bot.user)

@LevelingManager.event()
async def on_level_up(message, member_data, roles):
    await message.reply(
        f"You are now level {await member_data.level()}"
        + (f", you have received the {roles[0]}" f" role." if roles else "")
    )

@bot.command()
async def rank(ctx, member: discord.Member = None):
    mem_obj = member if member else ctx.author
    member_data = await LevelingManager.get_account(mem_obj)

    if not member_data:
        await ctx.send(f"I am still creating your account! please wait a few seconds.")
        return

    guild_leaderboard = await LevelingManager.get_leaderboard(ctx.guild)
    member = [x for x in guild_leaderboard if x.member == mem_obj]
    member_rank = guild_leaderboard.index(member[0]) + 1 if member else -1

    image = await ImageManager.create_leveling_profile(
        member=mem_obj,
        member_account=member_data,
        background=discordSuperUtils.Backgrounds.GALAXY,
        # name_color=(255, 255, 255),
        # rank_color=(127, 255, 0),
        # level_color=(255, 255, 255),
        # xp_color=(255, 255, 255),
        # bar_outline_color=(255, 255, 255),
        # bar_fill_color=(127, 255, 0),
        # bar_blank_color=(72, 75, 78),
        # profile_outline_color=(100, 100, 100),
        rank=member_rank,
        font_path=None,
        outline=5,
    )
    await ctx.send(file=image)

@bot.command()
async def set_roles(ctx, interval: int, *roles: discord.Role):
    await LevelingManager.set_interval(ctx.guild, interval)
    await LevelingManager.set_roles(ctx.guild, roles)

    await ctx.send(
        f"Successfully set the interval to {interval} and role list to {', '.join(role.name for role in roles)}"
    )

@bot.command()
async def leaderboard(ctx):
    guild_leaderboard = await LevelingManager.get_leaderboard(ctx.guild)
    formatted_leaderboard = [
        f"Member: {x.member}, XP: {await x.xp()}" for x in guild_leaderboard
    ]

    await discordSuperUtils.PageManager(
        ctx,
        discordSuperUtils.generate_embeds(
            formatted_leaderboard,
            title="Leveling Leaderboard",
            fields=25,
            description=f"Leaderboard of {ctx.guild}",
        ),
    ).run()

bot.run("token")


from collections import namedtuple
import altair as alt
import pandas as pd
import streamlit as st
import numpy as np
from collections import Counter

# Set Page Config - Main title, icon and setting the default layout to be wide
st.set_page_config(
    page_title="Randomised Trialer",
    page_icon="ðŸŽ²",
    layout="wide",
    initial_sidebar_state="expanded"
)
# Initial headings
st.title("Mr Matheson's Randomised Trialer")
st.subheader("Randomly generate probability simulations", divider="rainbow")

# Sidebar with Heading, includes slider for opts_n - how many options in each trial.
with st.sidebar:
  st.header("Set up the Simulation")
  opts_n = st.slider("Number of Options", min_value=2, max_value=20, value=4)
  variable_prob = st.toggle('Use Variable Probabilities')

  opts_name = []
  opts_val = []

  total = 0

  # Loops to allow for variable number of options. Adds the different text inputs to the sidebar dynamically
  for i in range(1, opts_n + 1):
    name = st.text_input(f"Option{i} Name:", value=f"Option{i}")
    opts_name.append(name)
    if variable_prob:
      n = st.slider(f"How many for {name}?", key=f"option{i}", min_value=1, max_value=20, value=1)
      opts_val.append(n)
      total += n
    else:
      opts_val.append(1)
      total += 1

# Sets the probability depending on the overall total of items, divided by the value of each option
prob = [x / total for x in opts_val]

# Sets up the pie graph.
pie_source = pd.DataFrame({"category": opts_name, "value": prob})

# Pie graph spinner using Altair.
pie = alt.Chart(pie_source).mark_arc().encode(
    theta="value",
    color="category"
)

# 2 column layout. Pie graph in the left, Control for the number of simulations in the right
col1, col2 = st.columns(2)
with col1:
  st.subheader("Theoretical Probability")
  st.altair_chart(pie, use_container_width=True, theme=None)

with col2:
  # n = st.slider("Number of Simulations (n):", min_value=1, max_value=10000, value=50)
  n = st.number_input("Number of Simulations (1 - 10,000)", min_value=1, max_value=10000, value= 50)

# This is using numpy to run the simulations n times, using the probabilities
sampleNumbers = np.random.choice(opts_name, n, p=prob)

sl = list(sampleNumbers)
counts = []
# To keep the original order, counts are calculated one at a time
for name in opts_name:
  counts.append(sl.count(name))

sampleVals = pd.DataFrame( {
  "Selection": opts_name,
  "Frequency": counts,
  "Theoretical Probability": prob
})
# Expected value is kind of useful, so this calculates it.
sampleVals['Expected Value'] = sampleVals['Theoretical Probability'] * n

st.subheader("Simulation Results")
col3, col4 = st.columns(2)
with col3:
  # Trying out a bar chart with tick marks from altair
  bar = alt.Chart(sampleVals).mark_bar().encode(
    x='Selection',
    y='Frequency'
  )

  #Altair's bar chart with ticks is basically two charts put on the same axis
  tick = alt.Chart(sampleVals).mark_tick(
      color='red',
      thickness=2,
      size=40 * 0.9,  # controls width of tick.
  ).encode(
      x='Selection',
      y='Expected Value'
  )
  c = bar + tick
  st.altair_chart(c, use_container_width=True, theme=None)

with col4:
  # Table with all the good stuff in it. The work is already done - just make it show
  st.subheader("Results Table")
  st.dataframe(sampleVals)
  st.subheader("Full set of results (in order)")
  st.write(sampleNumbers)


import json
from collections import Counter
from typing import Dict, Iterator, List, Tuple

import itchat
from jieba.analyse import extract_tags
from pyecharts import options
from pyecharts.charts import WordCloud
from pyecharts.globals import SymbolType, ThemeType, RenderType

if __name__ == '__main__':

    # 登陆微信
    print('登陆微信 ...')
    itchat.auto_login(hotReload=True)
    itchat.dump_login_status()

    # 读取朋友列表
    print('读取朋友列表 ...')
    friends = itchat.get_friends(update=True)

    # 微信定义的朋友属性，其中很多并没有官方的解释
    attribute_list: List[str] = [
        'MemberList', 'Uin', 'UserName', 'NickName',
        'HeadImgUrl', 'ContactFlag', 'MemberCount',
        'RemarkName', 'HideInputBarFlag', 'Sex',
        'Signature', 'VerifyFlag', 'OwnerUin',
        'PYInitial', 'PYQuanPin', 'RemarkPYInitial',
        'RemarkPYQuanPin', 'StarFriend',
        'AppAccountFlag', 'Statues', 'AttrStatus',
        'Province', 'City', 'Alias', 'SnsFlag',
        'UniFriend', 'DisplayName', 'ChatRoomId',
        'KeyWord', 'EncryChatRoomId', 'IsOwner'
    ]

    # 保存用户属性用的空列表
    friend_list: List[Dict] = list()

    # 读取用户属性
    print('读取用户属性...')
    for one_friend in friends:
        one: Dict[str, str] = dict()
        for one_attr in attribute_list:
            one[one_attr] = one_friend[one_attr]
        friend_list.append(one)

    # 可选：把读到的属性存盘
    with open('friends_data.json', 'w') as file:
        json.dump(friend_list, file, ensure_ascii=False)

    # 保存关键词的空列表
    all_tag_list: List[str] = list()

    # 从用户属性列表里，读取所有的用户签名
    print('读取所有的用户签名...')
    signature_list: Iterator = (one['Signature'] for one in friend_list)
    signature_list = filter(lambda x: len(str(x).strip()) > 0,
                            signature_list)  # 过滤掉空的签名

    # 定义忽略的词
    ignore_word_list: Tuple = (
        '<', 'span', 'class', '=', '\"', '\'', 'emoji', '>', '<', '/', 'gt',
        'lt'
    )

    # 从签名当中抽取关键词
    print('从签名当中抽取关键词...')
    for sig in signature_list:
        tag_list: Iterator = extract_tags(sig)  # 抽取关键词

        # 过滤不需要的词
        tag_list = filter(lambda x: len(str(x).strip()) > 0, tag_list)  # 过滤空词
        tag_list = filter(  # 过滤汉字、英文字母、数字之外的字符
            lambda x:
            all(uchar >= '\u4E00' and uchar <= '\u9FA5' for uchar in x)
            or
            all(uchar >= '\u0030' and uchar <= '\u0039' for uchar in x)
            or
            all(
                (uchar >= '\u0041' and uchar <= '\u005A') or (
                        uchar >= '\u0061' and uchar <= '\u007A')
                for uchar in x)
            ,
            tag_list
        )
        tag_list = filter(lambda x: x not in ignore_word_list,tag_list)  # 过滤忽略的单词

        # 把当前这条签名中过滤出来的关键词，添加到关键词列表当中
        all_tag_list += tag_list

    # 对关键词计数
    print('对关键词计数...')
    tag_counter: Counter = Counter(all_tag_list)

    # 生成词云数据
    words = [(key, tag_counter[key]) for key in tag_counter]

    # 创建词云
    print('创建词云...')
    title = f'{friend_list[0]["NickName"]}的朋友们的微信签名词云'

    tag_word_cloud: WordCloud = WordCloud(
        init_opts=options.InitOpts(
            theme=ThemeType.ESSOS,
            width='800px',
            height='600px',
            page_title=title,

        ),
    )
    tag_word_cloud.set_global_opts(
        title_opts=options.TitleOpts(title=title),
        toolbox_opts=options.ToolboxOpts(
            is_show=True,
            orient='horizontal',
        ),
    )

    tag_word_cloud.add(
        '',
        words,
        word_size_range=[10, 40],
        shape=SymbolType.RECT
    )

    tag_word_cloud.render(path=f'{title}.html')

    print('done')


# -*- coding: utf-8 -*-

import gensim
import os
import collections
import smart_open
import random

# Set file names for train and test data
test_data_dir = '{}'.format(os.sep).join([gensim.__path__[0], 'test', 'test_data'])
lee_train_file ='abstracts.cor'
lee_test_file = 'test.cor'
print("corpus: " + lee_train_file)
def read_corpus(fname, tokens_only=False):
    with smart_open.smart_open(fname, encoding="iso-8859-1") as f:
        for i, line in enumerate(f):
            if tokens_only:
                yield gensim.utils.simple_preprocess(line)
            else:
                # For training data, add tags
                yield gensim.models.doc2vec.TaggedDocument(gensim.utils.simple_preprocess(line), [i])

train_corpus = list(read_corpus(lee_train_file))
test_corpus = list(read_corpus(lee_test_file, tokens_only=True))

model = gensim.models.doc2vec.Doc2Vec(vector_size=20, min_count=0, epochs=100, dbow_words=1)
model.build_vocab(train_corpus)
model.infer_vector(['only', 'you', 'can', 'prevent', 'forest', 'fires'])

ranks = []
second_ranks = []
for doc_id in range(len(train_corpus)):
    inferred_vector = model.infer_vector(train_corpus[doc_id].words)
    sims = model.docvecs.most_similar([inferred_vector], topn=len(model.docvecs))
    rank = [docid for docid, sim in sims].index(doc_id)
    ranks.append(rank)

    second_ranks.append(sims[1])
collections.Counter(ranks)  # Results vary between runs due to random seeding and very small corpus

print('Document ({}): {}\n'.format(doc_id, ' '.join(train_corpus[doc_id].words)))
print(u'SIMILAR/DISSIMILAR DOCS PER MODEL %s:\n' % model)
for label, index in [('MOST', 0), ('SECOND-MOST', 1), ('MEDIAN', len(sims)//2), ('LEAST', len(sims) - 1)]:
    print(u'%s %s: %s\n' % (label, sims[index], ' '.join(train_corpus[sims[index][0]].words)))
    # Pick a random document from the corpus and infer a vector from the model
for i in range(50):
    try:

        doc_id = random.randint(0, len(train_corpus) - 1)

        # Compare and print the second-most-similar document
        print('Train Document ({}): {}\n'.format(doc_id, ' '.join(train_corpus[doc_id].words)))
        sim_id = second_ranks[doc_id]
        print('Similar Document {}: «{}»\n'.format(sim_id, ' '.join(train_corpus[sim_id[0]].words)))
    except:
        continue

# # Pick a random document from the test corpus and infer a vector from the model
# doc_id = random.randint(0, len(test_corpus) - 1)
# inferred_vector = model.infer_vector(test_corpus[doc_id])
# sims = model.docvecs.most_similar([inferred_vector], topn=len(model.docvecs))

# # Compare and print the most/median/least similar documents from the train corpus
# print('Test Document ({}): «{}»\n'.format(doc_id, ' '.join(test_corpus[doc_id])))
# print(u'SIMILAR/DISSIMILAR DOCS PER MODEL %s:\n' % model)
# for label, index in [('MOST', 0), ('MEDIAN', len(sims)//2), ('LEAST', len(sims) - 1)]:
#     print(u'%s %s: «%s»\n' % (label, sims[index], ' '.join(train_corpus[sims[index][0]].words)))


import datetime
import os
import pathlib
import stat
import time
import typing as tp

from pyvcs.index import GitIndexEntry, read_index  # type: ignore
from pyvcs.objects import hash_object  # type: ignore
from pyvcs.refs import get_ref, is_detached, resolve_head, update_ref  # type: ignore

def write_tree(gitdir: pathlib.Path, index: tp.List[GitIndexEntry], dirname: str = "") -> str:
    tree_inputs = []
    for entry in index:
        _, title = os.path.split(entry.name)
        if dirname:
            titles = dirname.split("/")
        else:
            titles = entry.name.split("/")
        if len(titles) != 1:
            prefix = titles[0]
            title = f"/".join(titles[1:])
            mode = "40000"
            tree_input = f"{mode} {prefix}\0".encode()
            tree_input += bytes.fromhex(write_tree(gitdir, index, title))
            tree_inputs.append(tree_input)
        else:
            if dirname and entry.name.find(dirname) == -1:
                continue
            with open(entry.name, "rb") as file:
                info = file.read()
            mode = str(oct(entry.mode))[2:]
            tree_input = f"{mode} {title}\0".encode()
            tree_input += bytes.fromhex(hash_object(info, "blob", write=True))
            tree_inputs.append(tree_input)

    tree_binary = b"".join(tree_inputs)
    return hash_object(tree_binary, "tree", write=True)

def commit_tree(
    gitdir: pathlib.Path,
    tree: str,
    message: str,
    parent: tp.Optional[str] = None,
    author: tp.Optional[str] = None,
) -> str:
    if not author:
        author = (
            os.getenv("GIT_AUTHOR_NAME") + " " + f"<{os.getenv('GIT_AUTHOR_EMAIL')}>"  # type: ignore
        )
    if time.timezone > 0:
        timezone = "-"
    else:
        timezone = "+"
    timezone += f"{abs(time.timezone) // 60 // 60:02}{abs(time.timezone) // 60 % 60:02}"
    info = [f"tree {tree}"]
    if parent is not None:
        info.append(f"parent{parent}")
    info.append(f"author {author} {int(time.mktime(time.localtime()))} {timezone}")
    info.append(f"committer {author} {int(time.mktime(time.localtime()))} {timezone}")
    info.append(f"\n{message}\n")
    return hash_object("\n".join(info).encode(), "commit", write=True)


import unittest
from unittest import TestCase
from xml.etree import ElementTree as ET

from osgeo import gdal

from aws.osml.gdal import NITFDESAccessor

class TestNITFDESAccessor(TestCase):
    def test_get_segments_by_name(self):
        ds = gdal.Open("./test/data/sicd_example_1_PFA_RE32F_IM32F_HH-0-0.NITF")
        des_accessor = NITFDESAccessor(ds.GetMetadata("xml:DES"))
        xml_segments = des_accessor.get_segments_by_name("XML_DATA_CONTENT")
        assert len(xml_segments) == 1
        assert isinstance(xml_segments[0], ET.Element)

    def test_extract_des_header(self):
        ds = gdal.Open("./test/data/sicd_example_1_PFA_RE32F_IM32F_HH-0-0.NITF")
        des_accessor = NITFDESAccessor(ds.GetMetadata("xml:DES"))
        xml_segments = des_accessor.get_segments_by_name("XML_DATA_CONTENT")

        des_header = NITFDESAccessor.extract_des_header(xml_segments[0])
        assert des_header is not None
        assert des_header.startswith("01UUS")
        assert len(des_header) == 946

if __name__ == "__main__":
    unittest.main()


import os
from ESP.settings import TOPDIR,CODEDIR
import datetime,logging
from ESP.esp.models import config
locals = config.objects.all()
LOCALSITE='HVMA' # hack!
for l in locals:
    if l.institution_name in os.listdir(TOPDIR):
        LOCALSITE=l.institution_name

###################################

###################################
def getJavaInfo():
    javadir="/usr/java/jdk1.5.0_09/bin/"
#    javadir="/usr/bin/"
    javaclass=" /home/ESP/ESP/sendMsgs:/home/ESP/axis-1_4/activation.jar:/usr/local/axis-1_4/lib/axis.jar:/usr/local/axis-1_4/lib/commons-logging-1.0.4.jar:/usr/local/axis-1_4/lib/commons-discovery-0.2.jar:/usr/local/axis-1_4/lib/jaxrpc.jar:/usr/local/axis-1_4/lib/wsdl4j-1.5.1.jar:/usr/local/axis-1_4/lib/saaj.jar:/usr/local/axis-1_4/lib/axis-ant.jar:/usr/local/axis-1_4/lib/log4j-1.2.8.jar:/home/ESP/axis-1_4/mail.jar:/home/ESP/ESP/sendMsgs/bcdc.jar "
    sendMsgcls = 'sendMsg'
    return (javadir, javaclass,sendMsgcls)

###############################
def getLogging(appname,debug=0):
    logdir = os.path.join(TOPDIR,LOCALSITE, 'logs/')
    if not os.path.isdir(logdir):
        os.mkdir(logdir)

    appname = logdir+appname
    debug=debug
    today=datetime.datetime.now().strftime('%Y%m%d')
    if debug:
        logging.basicConfig(level=logging.DEBUG,
                     format='%(asctime)s %(levelname)s %(message)s',
                     datefmt='%a, %d %b %Y %H:%M:%S',
                     filename='%s.%s.log' % (appname,today),
                     filemode='a')

    else:
        logging.basicConfig(level=logging.INFO,
                     format='%(asctime)s %(levelname)s %(message)s',
                     datefmt='%a, %d %b %Y %H:%M:%S',
                     filename='%s.%s.log' % (appname,today),
                     filemode='a')

    console = logging.StreamHandler()
    console.setLevel(logging.INFO)
    formatter = logging.Formatter('%(levelname)-4s %(message)s')
    console.setFormatter(formatter)
    logging.getLogger('').addHandler(console)

    return logging


import os
import csv
import sys
import glob
import win32com.client
from docx import Document
import tqdm

def doc_to_docx(doc_file, docx_file):
    # Initialize Word.Application
    word = win32com.client.Dispatch("Word.Application")
    word.Visible = False

    # Open the .doc file and save it as .docx
    doc = word.Documents.Open(doc_file)
    doc.SaveAs(docx_file, FileFormat=16)  # 16 = wdFormatDocumentDefault
    doc.Close()

    # Close Word
    word.Quit()

def docx_table_to_csv(docx_file, output_folder):
    # Load the document
    doc = Document(docx_file)

    # Iterate through tables in the document
    for i, table in enumerate(doc.tables):
        # Prepare the output CSV file
        output_file = os.path.join(output_folder, f"{os.path.splitext(os.path.basename(docx_file))[0]}_table_{i}.csv")

        # Write table data to the CSV file
        with open(output_file, 'w', newline='', encoding='utf-8') as csvfile:
            writer = csv.writer(csvfile)

            for row in table.rows:
                writer.writerow([cell.text for cell in row.cells])

# Set input folder and output folder
input_folder = "doc_files"
output_folder = "csv_files"

# Check if running on Windows
if sys.platform != "win32":
    print("This script requires the win32com library, which is only available on Windows.")
    sys.exit(1)

# Process each Word file in the input folder
for filename in os.listdir(input_folder):
    if filename.endswith(".doc"):
        current_dir = os.getcwd()

        doc_file = os.path.join(current_dir, input_folder, filename)
        docx_file = os.path.join(current_dir, input_folder, f"{os.path.splitext(filename)[0]}.docx")

        # Convert .doc to .docx
        print(doc_file)
        doc_to_docx(doc_file, docx_file)

        # Extract tables to CSV files
        docx_table_to_csv(docx_file, output_folder)

        # Optionally, remove the .docx file after processing
    #    os.remove(docx_file)


#! /usr/bin/env python
# -*- coding:utf-8 -*-

################################################################
#
# Class 07: 1-D Euler system of equations
# Solve equation using the Roe scheme
# Author: P. S. Volpiani
# Date: 01/08/2020
#
################################################################

#import os, sys
import numpy as np
from numpy import *
import matplotlib.pyplot as plt

from matplotlib import rc
rc('font', family='serif')
rc('lines', linewidth=1.5)
rc('font', size=14)
#plt.rc('legend',**{'fontsize':11})

######################################################################
#             Solving 1-D Euler system of equations
#                     using the Roe scheme
#
#             dq_i/dt + df_i/dx = 0, for x \in [a,b]
#
# This code solves the Sod's shock tube problem (IC=1)
#
# t=0                                 t=tEnd
# Density                             Density
#   ****************|                 *********\
#                   |                           \
#                   |                            \
#                   |                             ****|
#                   |                                 |
#                   |                                 ****|
#                   ***************                       ***********
#
# Domain cells (I{i}) reference:
#
#                |           |   u(i)    |           |
#                |  u(i-1)   |___________|           |
#                |___________|           |   u(i+1)  |
#                |           |           |___________|
#             ...|-----0-----|-----0-----|-----0-----|...
#                |    i-1    |     i     |    i+1    |
#                |-         +|-         +|-         +|
#              i-3/2       i-1/2       i+1/2       i+3/2
#
######################################################################

#def func_cons2prim(q,gamma):
#    # Primitive variables
#    r=q[0];
#    u=q[1]/r;
#    E=q[2]/r;
#    p=(gamma-1.)*r*(E-0.5*u**2);
#
#    return (r,u,p)

#def func_prim2cons(r,u,p,gamma):
#    # Conservative variables
#    q0=r;
#    q1=r*u;
#    q2=p/(gamma-1.)+0.5*r*u**2;
#    q =np.array([ q0, q1, q2 ]);
#
#    return (q)

def func_flux(q,gamma):
    # Primitive variables
    r=q[0];
    u=q[1]/r;
    E=q[2]/r;
    p=(gamma-1.)*r*(E-0.5*u**2);

    # Flux vector
    F0 = np.array(r*u)
    F1 = np.array(r*u**2+p)
    F2 = np.array(u*(r*E+p))
    flux=np.array([ F0, F1, F2 ])

    return (flux)

def flux_roe(q,dx,gamma,a,nx):

    # Compute primitive variables and enthalpy
    r=q[0];
    u=q[1]/r;
    E=q[2]/r;
    p=(gamma-1.)*r*(E-0.5*u**2);
    htot = gamma/(gamma-1)*p/r+0.5*u**2

    # Initialize Roe flux
    Phi=np.zeros((3,nx-1))

    for j in range (0,nx-1):

        # Compute Roe averages
        R=sqrt(r[j+1]/r[j]);                          # R_{j+1/2}
        rmoy=R*r[j];                                  # {hat rho}_{j+1/2}
        umoy=(R*u[j+1]+u[j])/(R+1);                   # {hat U}_{j+1/2}
        hmoy=(R*htot[j+1]+htot[j])/(R+1);             # {hat H}_{j+1/2}
        amoy=sqrt((gamma-1.0)*(hmoy-0.5*umoy*umoy));  # {hat a}_{j+1/2}

        # Auxiliary variables used to compute P_{j+1/2}^{-1}
        alph1=(gamma-1)*umoy*umoy/(2*amoy*amoy);
        alph2=(gamma-1)/(amoy*amoy);

        # Compute vector (W_{j+1}-W_j)
        wdif = q[:,j+1]-q[:,j];

        # Compute matrix P^{-1}_{j+1/2}
        Pinv = np.array([[0.5*(alph1+umoy/amoy), -0.5*(alph2*umoy+1/amoy),  alph2/2],
                        [1-alph1,                alph2*umoy,                -alph2 ],
                        [0.5*(alph1-umoy/amoy),  -0.5*(alph2*umoy-1/amoy),  alph2/2]]);

        # Compute matrix P_{j+1/2}
        P    = np.array([[ 1,              1,              1              ],
                        [umoy-amoy,        umoy,           umoy+amoy      ],
                        [hmoy-amoy*umoy,   0.5*umoy*umoy,  hmoy+amoy*umoy ]]);

        # Compute matrix Lambda_{j+1/2}
        lamb = np.array([[ abs(umoy-amoy),  0,              0                 ],
                        [0,                 abs(umoy),      0                 ],
                        [0,                 0,              abs(umoy+amoy)    ]]);

        # Compute Roe matrix |A_{j+1/2}|
        A=np.dot(P,lamb)
        A=np.dot(A,Pinv)

        # Compute |A_{j+1/2}| (W_{j+1}-W_j)
        Phi[:,j]=np.dot(A,wdif)

    #==============================================================
    # Compute Phi=(F(W_{j+1}+F(W_j))/2-|A_{j+1/2}| (W_{j+1}-W_j)/2
    #==============================================================
    F = func_flux(q,gamma);
    Phi=0.5*(F[:,0:nx-1]+F[:,1:nx])-0.5*Phi

    dF = (Phi[:,1:-1]-Phi[:,0:-2])

    return (dF)

# Parameters
CFL    = 0.50               # Courant Number
gamma  = 1.4                # Ratio of specific heats
ncells = 400                # Number of cells
x_ini =0.; x_fin = 1.       # Limits of computational domain
dx = (x_fin-x_ini)/ncells   # Step size
nx = ncells+1               # Number of points
x = np.linspace(x_ini+dx/2.,x_fin,nx) # Mesh

# Build IC
r0 = zeros(nx)
u0 = zeros(nx)
p0 = zeros(nx)
halfcells = int(ncells/2)

IC = 1 # 6 IC cases are available
if IC == 1:
    print ("Configuration 1, Sod's Problem")
    p0[:halfcells] = 1.0  ; p0[halfcells:] = 0.1;
    u0[:halfcells] = 0.0  ; u0[halfcells:] = 0.0;
    r0[:halfcells] = 1.0  ; r0[halfcells:] = 0.125;
    tEnd = 0.20;
elif IC== 2:
    print ("Configuration 2, Left Expansion and right strong shock")
    p0[:halfcells] = 1000.; p0[halfcells:] = 0.1;
    u0[:halfcells] = 0.0  ; u0[halfcells:] = 0.0;
    r0[:halfcells] = 3.0  ; r0[halfcells:] = 0.2;
    tEnd = 0.01;
elif IC == 3:
    print ("Configuration 3, Right Expansion and left strong shock")
    p0[:halfcells] = 7.   ; p0[halfcells:] = 10.;
    u0[:halfcells] = 0.0  ; u0[halfcells:] = 0.0;
    r0[:halfcells] = 1.0  ; r0[halfcells:] = 1.0;
    tEnd = 0.10;
elif IC == 4:
    print ("Configuration 4, Shocktube problem of G.A. Sod, JCP 27:1, 1978")
    p0[:halfcells] = 1.0  ; p0[halfcells:] = 0.1;
    u0[:halfcells] = 0.75 ; u0[halfcells:] = 0.0;
    r0[:halfcells] = 1.0  ; r0[halfcells:] = 0.125;
    tEnd = 0.17;
elif IC == 5:
    print ("Configuration 5, Lax test case: M. Arora and P.L. Roe: JCP 132:3-11, 1997")
    p0[:halfcells] = 3.528; p0[halfcells:] = 0.571;
    u0[:halfcells] = 0.698; u0[halfcells:] = 0.0;
    r0[:halfcells] = 0.445; r0[halfcells:] = 0.5;
    tEnd = 0.15;
elif IC == 6:
    print ("Configuration 6, Mach = 3 test case: M. Arora and P.L. Roe: JCP 132:3-11, 1997")
    p0[:halfcells] = 10.33; p0[halfcells:] = 1.0;
    u0[:halfcells] = 0.92 ; u0[halfcells:] = 3.55;
    r0[:halfcells] = 3.857; r0[halfcells:] = 1.0;
    tEnd = 0.09;

E0 = p0/((gamma-1.)*r0)+0.5*u0**2 # Total Energy density
a0 = sqrt(gamma*p0/r0)            # Speed of sound
q  = np.array([r0,r0*u0,r0*E0])   # Vector of conserved variables

if (False):
    fig = plt.subplots()
    ax1 = plt.subplot(4, 1, 1)
    #plt.title('Lax-Wendroff scheme')
    plt.plot(x, r0, 'k-')
    plt.ylabel('$rho$',fontsize=18)
    plt.tick_params(axis='x',bottom=False,labelbottom=False)
    plt.grid(True)

    ax2 = plt.subplot(4, 1, 2)
    plt.plot(x, u0, 'r-')
    plt.ylabel('$U$',fontsize=18)
    plt.tick_params(axis='x',bottom=False,labelbottom=False)
    plt.grid(True)

    ax3 = plt.subplot(4, 1, 3)
    plt.plot(x, p0, 'b-')
    plt.ylabel('$P$',fontsize=18)
    plt.tick_params(axis='x',bottom=False,labelbottom=False)
    plt.grid(True)

    ax4 = plt.subplot(4, 1, 4)
    plt.plot(x, E0, 'g-')
    plt.ylabel('$E$',fontsize=18)
    plt.grid(True)
    plt.xlim(x_ini,x_fin)
    plt.xlabel('x',fontsize=18)
    plt.subplots_adjust(left=0.2)
    plt.subplots_adjust(bottom=0.15)
    plt.subplots_adjust(top=0.95)

    plt.show()

# Solver loop
t  = 0
it = 0
a  = a0
dt=CFL*dx/max(abs(u0)+a0)         # Using the system's largest eigenvalue

while t < tEnd:

    q0 = q.copy();
    dF = flux_roe(q0,dx,gamma,a,nx);

    q[:,1:-2] = q0[:,1:-2]-dt/dx*dF;
    q[:,0]=q0[:,0]; q[:,-1]=q0[:,-1]; # Dirichlet BCs

    # Compute primary variables
    rho=q[0];
    u=q[1]/rho;
    E=q[2]/rho;
    p=(gamma-1.)*rho*(E-0.5*u**2);
    a=sqrt(gamma*p/rho);
    if min(p)<0: print ('negative pressure found!')

    # Update/correct time step
    dt=CFL*dx/max(abs(u)+a);

    # Update time and iteration counter
    t=t+dt; it=it+1;

    # Plot solution
    if it%40 == 0:
        print (it)
        fig,axes = plt.subplots(nrows=4, ncols=1)
        plt.subplot(4, 1, 1)
        #plt.title('Roe scheme')
        plt.plot(x, rho, 'k-')
        plt.ylabel('$rho$',fontsize=16)
        plt.tick_params(axis='x',bottom=False,labelbottom=False)
        plt.grid(True)

        plt.subplot(4, 1, 2)
        plt.plot(x, u, 'r-')
        plt.ylabel('$U$',fontsize=16)
        plt.tick_params(axis='x',bottom=False,labelbottom=False)
        plt.grid(True)

        plt.subplot(4, 1, 3)
        plt.plot(x, p, 'b-')
        plt.ylabel('$p$',fontsize=16)
        plt.tick_params(axis='x',bottom=False,labelbottom=False)
        plt.grid(True)

        plt.subplot(4, 1, 4)
        plt.plot(x, E, 'g-')
        plt.ylabel('$E$',fontsize=16)
        plt.grid(True)
        plt.xlim(x_ini,x_fin)
        plt.xlabel('x',fontsize=16)
        plt.subplots_adjust(left=0.2)
        plt.subplots_adjust(bottom=0.15)
        plt.subplots_adjust(top=0.95)
        plt.show()
        fig.savefig("fig_Sod_Roe_it"+str(it)+".pdf", dpi=300)


from rich import print as rprint
from rich.table import Table

def fillMatrix(matrix):
    cloneMatrix = matrix

    rprint('\n[yellow bold]Advertencia: \n- Inserte los números que desee separados por coma. \n- En caso de escribir menos números que el tamaño de la fila el resto serán rellenados con 0\n')

    for fila in range(len(matrix)):
        numbers = input(f'>>> Inserte los números de la fila {fila}: ')

        # separar los números por comas
        # utilizar 'strip()' para eliminar los espacios en blanco alrededor del número
        separatedNumbers = [num.strip() for num in numbers.split(',') if num.strip() != '']

        if len(separatedNumbers) > 0:
            # recorrer cada uno de los números insertados
            for numIndex in range(len(separatedNumbers)):
                # verificar que es un número
                if not separatedNumbers[numIndex].isnumeric():
                    rprint(f'[bold red] {separatedNumbers[numIndex]} no es un número')
                # verificar que el número se puede insertar mientras que no supere el tamaño de la fila
                elif numIndex < len(cloneMatrix[fila]):
                    cloneMatrix[fila][numIndex] = float(separatedNumbers[numIndex])

    # imprimir matriz en consola
    table = Table()

    # añadir columnas
    table.add_column('')
    for column in range(len(cloneMatrix[0])):
        table.add_column(f'Columna {column + 1}')

    for rowIndex in range(len(cloneMatrix)):
        # añadir filas
        stringNums = [str(num) for num in cloneMatrix[rowIndex]]
        table.add_row(f'Fila {rowIndex + 1}', *stringNums)

    rprint('\n', table)


# -*- coding: utf-8 -*-
"""
Created on Mon Oct 28 16:12:23 2019
通过learning curve判断迭代效果，通过validation curve寻找最优lambda
@author: 34276
"""
import ex5
import numpy as np
import scipy.io as sio
import matplotlib.pyplot as plt

data = sio.loadmat('ex5data1.mat')
X, y, Xtest, ytest, Xval, yval = data['X'], data['y'], data['Xtest'], data['ytest'], data['Xval'], data['yval']
X, Xval, Xtest = np.insert(X, 0, 1, axis=1), np.insert(Xval, 0, 1, axis=1), np.insert(Xtest, 0, 1, axis=1)

#使用训练集和验证集直接汇总learning curve，lamb为0
J_train, J_val = ex5.learningCurve(X, y, Xval, yval, 0)
ex5.plotLC(J_train, J_val)#J_train较大，欠拟合

#对训练集和验证集都进行多项式化之后绘制learning curve，lamb为0
X_poly = ex5.polyFeatures(X, 8)
Xval_poly = ex5.polyFeatures(Xval, 8)
Xtest_poly = ex5.polyFeatures(Xtest, 8)
J_train, J_val = ex5.learningCurve(X_poly, y, Xval_poly, yval, 0)
ex5.plotLC(J_train, J_val)#J_train几乎为0，过拟合

#通过绘制validation curve得到最优lamb
lambda_vec, J_train, J_val = ex5.validationCurve(X_poly, y, Xval_poly, yval)
fig, ax = plt.subplots(figsize=(12,8))
ax.plot(lambda_vec, J_train, label='Traning Data')
ax.plot(lambda_vec, J_val, label='Validation Data')
ax.set_xlabel('lamda')
ax.set_ylabel('error')
ax.set_title('Validation Curves')
plt.legend(loc='upper right')

#使用最优lamb绘制learning curve
best_lamb = lambda_vec[np.argmin(J_val)]
J_train, J_val = ex5.learningCurve(X_poly, y, Xval_poly, yval, best_lamb)
ex5.plotLC(J_train, J_val)#J_train较小，拟合效果较好


# -*- coding: utf-8 -*-

import hashlib
from datetime import datetime

from scrapy.exceptions import DropItem
from scrapy.pipelines.images import ImagesPipeline
from scrapy.utils.python import to_bytes
from scrapy.http import Request
from LINEBlogScraper.items import LineblogscraperItem

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html

class LineblogscraperPipeline(object):

    def process_item(self, item, spider):
        return item

class ValidationPipeline(object):

    ITEM = LineblogscraperItem()

    def process_item(self, item, spider):

        for key in self.ITEM.fields.keys():

            if key in ['article_tag', 'image_urls', 'images']:
                continue

            if not item[key]:
                raise DropItem('Missing {}'.format(key.split(' ')))

        return item

class FormatArticleBody(object):

    def process_item(self, item, spider):
        item['article_body'] = self.remove_non_break_space(item['article_body'])
        item['article_body'] = self.remove_tab(item['article_body'])
        item['article_body'] = self.remove_space(item['article_body'])
        item['article_body'] = self.remove_newline_code(item['article_body'])

        return item

    def remove_non_break_space(self, sentence_list):
        return [s.replace('\xa0', '') for s in sentence_list]

    def remove_space(self, sentence_list):
        return [s.replace(' ', '') for s in sentence_list]

    def remove_tab(self, sentence_list):
        return [s.replace('\t', '') for s in sentence_list]

    def remove_newline_code(self, sentence_list):
        sentence_list = [s.replace('\n', '').replace('\r', '') for s in sentence_list]
        while sentence_list.count('') > 0:
            sentence_list.remove('')

        return sentence_list

class FormatArticleTitle(object):

    def process_item(self, item, spider):

        item['article_title'] = self.remove_newline_code(item['article_title'])
        item['article_title'] = self.remove_space(item['article_title'])

        return item

    def remove_newline_code(self, s):
        return s.replace('\n', '').replace('\r', '')

    def remove_space(self, s):
        return s.replace(' ', '')

class LineBlogImagesPipeline(ImagesPipeline):

    def get_media_requests(self, item, info):

        tdatetime = datetime.strptime(item['article_datetime'],
                                      '%Y-%m-%dT%H:%M:%S%z')
        date = tdatetime.strftime('%Y-%m-%d')

        return [Request(x, meta={'author': item['author'],
                                 'date': date})
                for x in item.get(self.images_urls_field, [])]

    def file_path(self, request, response=None, info=None):
        # start of deprecation warning block (can be removed in the future)
        def _warn():
            from scrapy.exceptions import ScrapyDeprecationWarning
            import warnings
            warnings.warn('ImagesPipeline.image_key(url) and file_key(url) methods are deprecated, '
                          'please use file_path(request, response=None, info=None) instead',
                          category=ScrapyDeprecationWarning, stacklevel=1)

        # check if called from image_key or file_key with url as first argument
        if not isinstance(request, Request):
            _warn()
            url = request
        else:
            url = request.url

        # detect if file_key() or image_key() methods have been overridden
        if not hasattr(self.file_key, '_base'):
            _warn()
            return self.file_key(url)
        elif not hasattr(self.image_key, '_base'):
            _warn()
            return self.image_key(url)
        # end of deprecation warning block

        author = request.meta['author']
        date = request.meta['date']

        image_guid = hashlib.sha1(to_bytes(url)).hexdigest()  # change to request.url after deprecation
        return 'full/%s/%s/%s.jpg' % (author, date, image_guid)


#!/usr/bin/python3
#-----------------------------------------------------------
#  Extracts metadata from HST Photo Database
#  Wed 27 Nov 2019 03:58:49 PM CST
#-----------------------------------------------------------

import requests
from bs4 import BeautifulSoup
import sys
import dateparser
import datetime
import re

#accno = "72-3113"   # - entry with three element date
#accno = "95-22-87"  # - entry with two element date
#accno = "2009-1504" # - entry with single element date
#accno = "2006-487"  # - entry with no date
#accno = "80-16"     # - entry with no date or rights statement

accno = sys.argv[1]

url = 'https://www.trumanlibrary.gov/photograph-records/' + accno

response = requests.get(url, timeout=5)
html = response.content

soup = BeautifulSoup(html,'lxml')

#### Extract Title #####

print("Title: " + str(soup.h1.text).strip())

alltext = soup.get_text()

#### Extract Description #####

left = (alltext.find("Description"))
right = (alltext.find("Date(s)"))

Description = (alltext[left+11:right].strip())
print(" ")
print("Description: " + Description)

#### Extracts Date #####

x = (alltext.find("Date(s)"))     # extracts text only

D1=(alltext[x+12:x+30].rstrip())  # extracts date string as D1

D2 = dateparser.parse(D1)   # parses D1 date string into date variable D2

try:
    D2 = datetime.datetime(D2.year, D2.month, D2.day)

    D2 = datetime.datetime(D2.year, D2.month, D2.day)
    #, settings={'STRICT_PARSING': True}

    print(" ")
    w = (len(re.findall(r'\w+', D1)))
    D2 = (D2.year,D2.month,D2.day)
    print("Date: " + D1)

    if w == 1:
        print(str(D2[0]) + ":01:01")
    elif w == 2:
        print(str(D2[0]) + ":" + str('{:02d}'.format(D2[1]) + ":01"))
    else:
        print(str(D2[0]) + ":" + str('{:02d}'.format(D2[1]))+ ":" + str('{:02d}'.format(D2[2])))

except AttributeError:
    print(" ")
    print("Date: No Date string for this item in HST PDB")

#### Extract Rights Statement #####

Rights = soup.find('div', {'class': "field field--name-field-rights-undetermined-text field--type-markup field--label-hidden field__item"})

print(" ")

try:
    print("Rights: " + Rights.text)
except AttributeError:
    print("Rights: No Rights information included for this item in HST PDB")


import numpy as np
import matplotlib.pyplot as plt
import pdb
# 解决中文显示问题
plt.rcParams['font.sans-serif'] = ['KaiTi'] # 指定默认字体
plt.rcParams['axes.unicode_minus'] = False # 解决保存图像是负号'-'显示为方块的问题

"旅行商问题 ( TSP , Traveling Salesman Problem )"
coordinates = np.array([[565.0, 575.0], [25.0, 185.0], [345.0, 750.0], [945.0, 685.0], [845.0, 655.0],
                   [880.0, 660.0], [25.0, 230.0], [525.0, 1000.0], [580.0, 1175.0], [650.0, 1130.0],
                   [1605.0, 620.0], [1220.0, 580.0], [1465.0, 200.0], [1530.0, 5.0], [845.0, 680.0],
                   [725.0, 370.0], [145.0, 665.0], [415.0, 635.0], [510.0, 875.0], [560.0, 365.0],
                   [300.0, 465.0], [520.0, 585.0], [480.0, 415.0], [835.0, 625.0], [975.0, 580.0],
                   [1215.0, 245.0], [1320.0, 315.0], [1250.0, 400.0], [660.0, 180.0], [410.0, 250.0],
                   [420.0, 555.0], [575.0, 665.0], [1150.0, 1160.0], [700.0, 580.0], [685.0, 595.0],
                   [685.0, 610.0], [770.0, 610.0], [795.0, 645.0], [720.0, 635.0], [760.0, 650.0],
                   [475.0, 960.0], [95.0, 260.0], [875.0, 920.0], [700.0, 500.0], [555.0, 815.0],
                   [830.0, 485.0], [1170.0, 65.0], [830.0, 610.0], [605.0, 625.0], [595.0, 360.0],
                   [1340.0, 725.0], [1740.0, 245.0]])

# 得到距离矩阵的函数
def getdistmat(coordinates):
    num = coordinates.shape[0]  # 52个坐标点
    distmat = np.zeros((52, 52))  # 52X52距离矩阵
    for i in range(num):
        for j in range(i, num):
            distmat[i][j] = distmat[j][i] = np.linalg.norm(coordinates[i] - coordinates[j])
    return distmat

def initpara():
    alpha = 0.99
    t = (1, 100)
    markovlen = 1000
    return alpha, t, markovlen

num = coordinates.shape[0]
distmat = getdistmat(coordinates)  # 得到距离矩阵

solutionnew = np.arange(num)
# valuenew = np.max(num)

solutioncurrent = solutionnew.copy()
valuecurrent = 99000  # np.max这样的源代码可能同样是因为版本问题被当做函数不能正确使用，应取一个较大值作为初始值
# print(valuecurrent)

solutionbest = solutionnew.copy()
valuebest = 99000  # np.max
alpha, t2, markovlen = initpara()
t = t2[1]

result = []  # 记录迭代过程中的最优解
while t > t2[0]:
    for i in np.arange(markovlen):

        # 下面的两交换和三角换是两种扰动方式，用于产生新解
        if np.random.rand() > 0.5:  # 交换路径中的这2个节点的顺序
            # np.random.rand()产生[0, 1)区间的均匀随机数
            while True:  # 产生两个不同的随机数
                loc1 = np.int(np.ceil(np.random.rand() * (num - 1)))
                loc2 = np.int(np.ceil(np.random.rand() * (num - 1)))
                ## print(loc1,loc2)
                if loc1 != loc2:
                    break
            solutionnew[loc1], solutionnew[loc2] = solutionnew[loc2], solutionnew[loc1]
        else:  # 三交换
            while True:
                loc1 = np.int(np.ceil(np.random.rand() * (num - 1)))
                loc2 = np.int(np.ceil(np.random.rand() * (num - 1)))
                loc3 = np.int(np.ceil(np.random.rand() * (num - 1)))

                if ((loc1 != loc2) & (loc2 != loc3) & (loc1 != loc3)):
                    break

            # 下面的三个判断语句使得loc1<loc2<loc3
            if loc1 > loc2:
                loc1, loc2 = loc2, loc1
            if loc2 > loc3:
                loc2, loc3 = loc3, loc2
            if loc1 > loc2:
                loc1, loc2 = loc2, loc1

            # 下面的三行代码将[loc1,loc2)区间的数据插入到loc3之后
            tmplist = solutionnew[loc1:loc2].copy()
            solutionnew[loc1:loc3 - loc2 + 1 + loc1] = solutionnew[loc2:loc3 + 1].copy()
            solutionnew[loc3 - loc2 + 1 + loc1:loc3 + 1] = tmplist.copy()

        valuenew = 0
        for i in range(num - 1):
            valuenew += distmat[solutionnew[i]][solutionnew[i + 1]]
        valuenew += distmat[solutionnew[0]][solutionnew[51]]
        # print (valuenew)
        if valuenew < valuecurrent:  # 接受该解

            # 更新solutioncurrent 和solutionbest
            valuecurrent = valuenew
            solutioncurrent = solutionnew.copy()

            if valuenew < valuebest:
                valuebest = valuenew
                solutionbest = solutionnew.copy()
        else:  # 按一定的概率接受该解
            if np.random.rand() < np.exp(-(valuenew - valuecurrent) / t):
                valuecurrent = valuenew
                solutioncurrent = solutionnew.copy()
            else:
                solutionnew = solutioncurrent.copy()
    t = alpha * t
    result.append(valuebest)
    print(t)  # 程序运行时间较长，打印t来监视程序进展速度

# 用来显示结果
plt.plot(np.array(result))
plt.ylabel("最佳值")
plt.xlabel("时间")
plt.show()


"""
cell_builder.py
Neuron Cell Builder
"""
import logging
from neuron_ui import neuron_utils
from neuron_ui import neuron_geometries_utils
from neuron_ui.singleton import Singleton
from jupyter_geppetto.geppetto_comm import GeppettoCoreAPI as G
from jupyter_geppetto.geppetto_comm import GeppettoJupyterModelSync

@Singleton
class CellBuilder:

    def __init__(self):
        logging.debug('Initializing Cell Builder')

        self.geometry = None
        self.segment = None
        section_diam_panel = neuron_utils.add_text_field_with_label('Section Diam', None)
        section_length_panel = neuron_utils.add_text_field_with_label('Section Length', None)

        segment_diam_panel = neuron_utils.add_text_field_with_label(
            'Segment Diam', None)

        # Get textfields from panels
        self.section_diam = section_diam_panel.items[1]
        self.section_length = section_length_panel.items[1]
        self.segment_diam = segment_diam_panel.items[1]
        self.save_button = neuron_utils.add_button('Save', self.modify_segment)

        self.cellBuilderPanel = neuron_utils.add_panel('Cell Builder', items=[
            segment_diam_panel, section_diam_panel, section_length_panel, self.save_button], widget_id='cellBuilderPanel', position_x=955, position_y=374, width = 340, height = 190)
        self.cellBuilderPanel.register_to_event(
            [GeppettoJupyterModelSync.events_controller._events['Select']], self.updateValues)
        self.cellBuilderPanel.on_close(self.close)
        self.cellBuilderPanel.display()

    def close(self, component, args):
        self.cellBuilderPanel.unregister_to_event(
            [GeppettoJupyterModelSync.events_controller._events['Select']], self.updateValues)

        # Close Jupyter object
        self.cellBuilderPanel.close()
        del self.cellBuilderPanel

        # Destroy this class
        CellBuilder.delete()
        # del RunControl._instance

    def shake_panel(self):
        self.cellBuilderPanel.shake()

    def updateValues(self, dataId, geometry_identifier, point):
        try:
            logging.debug('Updating values for Cell Builder')

            for geometry in GeppettoJupyterModelSync.current_model.geometries_raw:
                if geometry.id == geometry_identifier:
                    logging.debug('Loading values for geometry ' +
                                  str(geometry_identifier))
                    GeppettoJupyterModelSync.current_model.highlight_visual_group_element(geometry.python_variable["section"].name())

                    # Update segment and geometry
                    distance_to_selection, section_length = neuron_geometries_utils.calculate_distance_to_selection(
                        geometry, point)
                    distance_to_selection_normalised = distance_to_selection/section_length
                    self.segment = geometry.python_variable[
                        "section"](distance_to_selection_normalised)
                    self.geometry = geometry

                    # Update values
                    self.segment_diam.sync_value =  str(
                        self.segment.diam)
                    self.section_diam.sync_value = str(
                        geometry.python_variable["section"].diam)
                    self.section_length.sync_value = str(
                        geometry.python_variable["section"].L)

                    return True
            return False
        except Exception as exception:
            logging.exception(
                "Unexpected error executing callback for component:")
            raise

    def modify_segment(self, triggeredComponent, args):
        logging.debug('Modifying segment')
        self.geometry.python_variable[
            "section"].diam = float(self.section_diam.sync_value)
        self.geometry.python_variable[
            "section"].L = float(self.section_length.sync_value)
        self.segment = float(self.segment_diam.sync_value)

        # Recreating Scene
        neuron_geometries_utils.extractGeometries(reload=True)


# import sqlite3
# conn = sqlite3.connect("somedatabase")
# curs = conn.cursor()
# curs.execute()
# conn.commit()
# conn.close()

import sqlite3

def convert(value):
    if value.startswith('~'):
        return value.strip("~")
    if not value:
        value = "0"
    return float(value)

s = "~07276~^~HORMEL SPAM ... PORK W/ HAM MINCED CND~^^~1 serving~^^~~^0^^^"
conn = sqlite3.connect("food.db")
curs = conn.cursor()

curs.execute("""
CREATE TABLE IF NOT EXISTS food (
id        TEXT PRIMARY KEY,
desc      TEXT,
water     FLOAT,
kcal      FLOAT,
protein   FLOAT,
fat       FLOAT,
ash       FLOAT,
carbs     FLOAT,
fiber     FLOAT,
sugar     FLOAT
)
""")

query = 'INSERT INTO food VALUES(?,?,?,?,?,?,?,?,?,?)'

fields = s.split('^')
vals = [convert(f) for f in fields[:len(fields)]]
curs.execute(query,vals)

conn.commit()
conn.close()

conn = sqlite3.connect("food.db")
curs = conn.cursor()

query = "SELECT * FROM food"
curs.execute(query)
conn.commit()
conn.close()


import os
import json
import torchaudio
from tqdm import tqdm
from glob import glob

from utils.util import has_existed

def main(output_path, dataset_path):
    print("-" * 10)
    print("Dataset splits for ljspeech...\n")

    save_dir = os.path.join(output_path, "ljspeech")
    ljspeech_path = dataset_path

    wave_files = glob(ljspeech_path + "/wavs/*.wav")

    train_output_file = os.path.join(save_dir, "train.json")
    test_output_file = os.path.join(save_dir, "test.json")

    if has_existed(train_output_file):
        return

    utts = []

    for wave_file in tqdm(wave_files):
        res = {
            "Dataset": "ljspeech",
            "Singer": "female1",
            "Uid": "{}".format(wave_file.split("/")[-1].split(".")[0]),
        }
        res["Path"] = wave_file
        assert os.path.exists(res["Path"])

        waveform, sample_rate = torchaudio.load(res["Path"])
        duration = waveform.size(-1) / sample_rate
        res["Duration"] = duration

        if duration <= 1e-8:
            continue

        utts.append(res)

    test_length = len(utts) // 20

    train_utts = []
    train_index_count = 0
    train_total_duration = 0

    for i in tqdm(range(len(utts) - test_length)):
        tmp = utts[i]
        tmp["index"] = train_index_count
        train_index_count += 1
        train_total_duration += tmp["Duration"]
        train_utts.append(tmp)

    test_utts = []
    test_index_count = 0
    test_total_duration = 0

    for i in tqdm(range(len(utts) - test_length, len(utts))):
        tmp = utts[i]
        tmp["index"] = test_index_count
        test_index_count += 1
        test_total_duration += tmp["Duration"]
        test_utts.append(tmp)

    print("#Train = {}, #Test = {}".format(len(train_utts), len(test_utts)))
    print(
        "#Train hours= {}, #Test hours= {}".format(
            train_total_duration / 3600, test_total_duration / 3600
        )
    )

    # Save
    os.makedirs(save_dir, exist_ok=True)
    with open(train_output_file, "w") as f:
        json.dump(train_utts, f, indent=4, ensure_ascii=False)
    with open(test_output_file, "w") as f:
        json.dump(test_utts, f, indent=4, ensure_ascii=False)


from typing import Optional

import torch
import torch.nn as nn
import torchaudio

class MelSpec(nn.Module):
    """log-Mel scale spectrogram.
    """
    def __init__(self,
                 strides: int,
                 windows: int,
                 mel: int,
                 fmin: int = 0,
                 fmax: Optional[int] = 8000,
                 sr: int = 16000):
        """Initializer.
        Args:
            strides: hop length, the number of the frames between adjacent windows.
            windows: length of the windows.
            mel: size of the mel filterbanks.
            fmin, fmax: minimum, maximum frequency,
                if fmax is None, use half of the sample rate as default.
            sr: sample rate.
        """
        super().__init__()
        self.strides, self.windows = strides, windows
        # [mel, windows // 2 + 1]
        # use slaney-scale mel filterbank for `librosa.filters.mel` compatibility.
        self.register_buffer(
            'melfilter',
            torchaudio.functional.melscale_fbanks(
                windows // 2 + 1,
                fmin, fmax, mel, sr,
                norm='slaney', mel_scale='slaney').T,
            persistent=False)
        # [windows], use hann window
        self.register_buffer(
            'hann', torch.hann_window(windows), persistent=False)

    def forward(self, audio: torch.Tensor) -> torch.Tensor:
        """Generate the log-mel scale spectrogram.
        Args:
            audio: [torch.float32; [B, T]], audio signal, [-1, 1]-ranged.
        Returns:
            [torch.float32; [B, mel, T / strides]], log-mel spectrogram
        """
        # [B, windows // 2 + 1, T / strides, 2]
        fft = torch.stft(
            audio, self.windows, self.strides,
            window=self.hann,
            center=True, pad_mode='reflect', return_complex=False)
        # [B, windows // 2 + 1, T / strides]
        mag = torch.sqrt(fft.square().sum(dim=-1) + 1e-7)
        # [B, mel, T / strides]
        return torch.log(torch.matmul(self.melfilter, mag) + 1e-7)


from collections import deque

def neighbours(N, x, y):
    # returns a list of valid positions a knight can jump to from an initial position (x,y)
    # valid position means not outside the N x N chessboard
    # (x,y) is provided in 0-based index
    dx = [ 2, 2,-2,-2, 1, 1,-1,-1]
    dy = [ 1,-1, 1,-1, 2,-2, 2,-2]
    output = []
    for j in range(len(dx)):
        new_x = x + dx[j]
        new_y = y + dy[j]
        if 0 <= new_x < N and 0 <= new_y < N:
            output.append((new_x, new_y))
    return output

N = int(input()) # chessboard length
chessboard = [] # initialize grid

# Get game grid from user
for i in range(N):
    row = []
    row.extend(input())
    chessboard.append(row)
    if 'K' in row:
        source = (i, row.index('K'))

chessboard[source[0]][source[1]] = 0 # set knight's initial position to 0

# BFS
# BFS works as all neighbours are 1 move away (1 weight)
queue = deque()
queue.append(source) # enqueue initial knight position

while queue:
    front = queue.popleft() # dequeue front of queue
    for neighbour in neighbours(N, front[0], front[1]):
        if chessboard[neighbour[0]][neighbour[1]] == '#': continue # blocked cell
        if chessboard[neighbour[0]][neighbour[1]] == '.': # not visited yet
            queue.append(neighbour) # enqueue neighbour
            chessboard[neighbour[0]][neighbour[1]] = chessboard[front[0]][front[1]] + 1 # +1 distance (move) away
        else: # already visited, update shortest path
            if chessboard[neighbour[0]][neighbour[1]] > chessboard[front[0]][front[1]] + 1:
                chessboard[neighbour[0]][neighbour[1]] = chessboard[front[0]][front[1]] + 1

if chessboard[0][0] == '.':
    print(-1) # not visited
else:
    print(chessboard[0][0]) # 0-based indexing


import os
import platform
import random
import string

from flask import Flask, request, jsonify

import ioiprint.settings as settings
import ioiprint.modifier as modifier
from ioiprint.print import print_file, JOB_PRIORITY_HIGH
from ioiprint.contestant_data import get_contestant_data
from ioiprint.utils import create_temp_directory, generate_print_id
from ioiprint.metrics import get_metrics

app = Flask('ioiprint')

def _real_ip():
    return request.headers.get('X-Real-IP', request.remote_addr)

@app.route('/healthcheck', methods=['GET'])
def healthcheck():
    return 'OK'

@app.route('/mass', methods=['POST'])
def mass():
    temp_directory = create_temp_directory()
    original_pdf_path = os.path.join(temp_directory, 'original.pdf')
    request.files['pdf'].save(original_pdf_path)

    printer = request.form.get('printer', settings.printer_for_mass())
    count = int(request.form['count'])
    for _ in range(count):
        print_file(original_pdf_path, printer, 'mass')
    return "OK"

@app.route('/translation', methods=['POST'])
def translation():
    temp_directory = create_temp_directory()
    original_pdf_path = os.path.join(temp_directory, 'original.pdf')
    request.files['pdf'].save(original_pdf_path)

    country_code = request.form['country_code']
    country_name = request.form['country_name']
    cover_page = int(request.form['cover_page']) > 0
    count = int(request.form['count'])

    if cover_page:
        final_pdf_path = modifier.make_translation_pdf(
            original_pdf_path,
            country_code,
            country_name,
            temp_directory
        )
    else:
        final_pdf_path = original_pdf_path

    job_name = 'translation:%s'%country_code
    for _ in range(count):
        print_file(final_pdf_path, settings.printer_for_translation(), job_name)
    return "OK"

@app.route('/staff_call', methods=['POST'])
def staff_call():
    req_data = request.get_json()
    message = req_data['message']
    contestant = req_data['contestant']
    contestant_id = req_data['contestant']['id']
    desk = req_data['desk']
    desk_zone = req_data['desk'].get('zone')

    temp_directory = create_temp_directory()
    pdf_path = modifier.make_staff_call_pdf(
        message=message,
        contestant=contestant,
        desk=desk,
        temp_directory=temp_directory,
    )

    job_name = 'staff_call:%s'%contestant_id
    print_file(pdf_path, settings.printer_for_contestant(desk_zone),
               job_name, job_priority=JOB_PRIORITY_HIGH)
    return jsonify({'status': 'OK'})

@app.route('/contestant', methods=['POST'])
def contestant():
    temp_directory = create_temp_directory()
    original_pdf_path = os.path.join(temp_directory, 'original.pdf')
    request.files['pdf'].save(original_pdf_path)

    contestant_data = get_contestant_data(_real_ip())
    contestant_id = (contestant_data['contestant'] or {}).get('id', '')
    desk_zone = (contestant_data['desk'] or {}).get('zone')

    print_id = generate_print_id()
    hostname = platform.uname().node

    final_pdf_path = modifier.make_contestant_pdf(
        pdf_file_path=original_pdf_path,
        contestant=contestant_data['contestant'],
        desk=contestant_data['desk'],
        print_id=print_id,
        temp_directory=temp_directory,
    )

    job_name = 'contestant:%s:%s'%(contestant_id, print_id)
    metric_name = 'contestant:%s'%(contestant_id)
    print_file(
        file_path=final_pdf_path,
        printer=settings.printer_for_contestant(desk_zone),
        job_name=job_name,
        metric_name=metric_name
    )
    return "Queued as %s on %s"%(print_id, hostname)

@app.route('/password', methods=['POST'])
def password():
    req_data = request.get_json()
    title = req_data['title']
    users = req_data['users']

    temp_directory = create_temp_directory()
    pdf_path = modifier.make_password_pdf(
        title=title,
        users=users,
        temp_directory=temp_directory,
    )

    job_name = 'password'
    print_file(pdf_path, settings.printer_for_mass(), job_name)
    return jsonify({'status': 'OK'})

@app.route('/metrics', methods=['GET'])
def metrics():
    metrics = get_metrics()
    result = ''
    for job_type, count_by_key in metrics.items():
        prometheus_key = 'ioiprint_%s_jobs'%(job_type)
        result += '''# HELP {prometheus_key} It shows how many {job_type} jobs are called
# TYPE {prometheus_key} counter
'''.format(prometheus_key=prometheus_key, job_type=job_type)
        for job_value, count in count_by_key.items():
            prometheus_key = 'ioiprint_%s_jobs'%(job_type)
            result += '''{prometheus_key}{{{job_type}="{job_value}"}} {count}
'''.format(prometheus_key=prometheus_key, job_type=job_type, job_value=job_value, count=count)
    return result


from selenium import webdriver
import unittest
import time
import datetime

class bing_soso_po(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        cls.browser = webdriver.Chrome(executable_path="/Users/lindafang/PycharmProjects/seleniumforpython2020/driver/chromedriver78")
        cls.browser.get('https://cn.bing.com/')
        cls.browser.maximize_window()
        cls.browser.implicitly_wait(10)
        assert '微软 Bing 搜索 ' in cls.browser.title

    @classmethod
    def tearDownClass(cls):
        if(cls.browser!=None):
            print("环境销毁")
            print("运行时间总计:"+str(datetime.datetime.now()))
            cls.browser.quit()

    def test_soso(self):
        elem = self.browser.find_element_by_id('sb_form_q')
        elem.click()
        elem.clear()
        elem.send_keys('seleniumhq')
        elec = self.browser.find_element_by_id('sb_form_go')
        elec.click()
        self.browser.get_screenshot_as_file(str(datetime.datetime.now())+"_sousuo.png")
        time.sleep(3)
        assert 'selenium' in self.browser.page_source

if __name__ == '__main__':
    unittest.main()


import numpy as np
from scipy.optimize import fmin_l_bfgs_b, NonlinearConstraint, minimize, Bounds, fmin_cobyla
from sklearn import gaussian_process as gp
import warnings
from scipy.stats import norm
from matplotlib import pyplot as plt

DOMAIN = np.array([[0, 10]])  # restrict \theta in [0, 10]
SAFETY_THRESHOLD = 4  # threshold, upper bound of SA
SEED = 0

""" Solution """

warnings.filterwarnings("ignore", category=RuntimeWarning)
warnings.filterwarnings("ignore", category=UserWarning)
class BO_algo():
    def __init__(self):
        """Initializes the algorithm with a parameter configuration. """

        self.beta = 4
        self.beta_v = 4
        self.v_mean = 4
        # self.mean = 4
        # self.var = 0.0001
        self.gpf = gp.GaussianProcessRegressor(kernel = 0.5 * gp.kernels.Matern(length_scale=1, nu=2.5, length_scale_bounds="fixed"), alpha=0.15)
        self.gpv = gp.GaussianProcessRegressor(kernel = 4 + np.sqrt(2) * gp.kernels.Matern(length_scale=1,nu=2.5, length_scale_bounds="fixed"), alpha=0.001)
        self.x = []
        self.f = []
        self.v = []
        self.recs = []

    #maybe the model works best for regret around 0.03?
    def next_recommendation(self):
        """
        Recommend the next input to sample.

        Returns
        -------
        recommendation: np.ndarray
            1 x domain.shape[0] array containing the next point to evaluate
        """

        # TODO: enter your code here
        # In implementing this function, you may use optimize_acquisition_function() defined below.

        x_next = self.optimize_acquisition_function()
        # while self.gpv.predict(x_next) <= SAFETY_THRESHOLD:
        #     x_next = self.optimize_acquisition_function()

        self.recs.append(x_next[0])
        return x_next

    def optimize_acquisition_function(self):
        """
        Optimizes the acquisition function.

        Returns
        -------
        x_opt: np.ndarray
            1 x DOMAIN.shape[0] array containing the point that maximize the acquisition function.
        """

        def objective(x):
            return -self.acquisition_function(x)

        def upperb(x):
            mean_v,std_v = self.gpv.predict(np.atleast_2d(x), return_std=True)
            fc = (mean_v + self.beta_v * std_v)
            return np.mean(fc)

        def lowerb(x):
            mean_v,std_v = self.gpv.predict(np.atleast_2d(x), return_std=True)
            fc = (mean_v - self.beta_v * std_v)
            return np.mean(fc)

        f_values = []
        x_values = []

        # Restarts the optimization 20 times and pick best solution
        for _ in range(20):
            x0 = DOMAIN[:, 0] + (DOMAIN[:, 1] - DOMAIN[:, 0]) * \
                 np.random.rand(DOMAIN.shape[0])
            # bounds = NonlinearConstraint(lb=lowerb(x0), ub=upperb(x0))
            const = NonlinearConstraint(fun=upperb, lb=-np.inf, ub=SAFETY_THRESHOLD)
            const_lo = NonlinearConstraint(fun=lowerb, lb=-np.inf, ub=SAFETY_THRESHOLD)
            # result = fmin_l_bfgs_b(objective, x0=x0, bounds=DOMAIN, approx_grad=True)
            # result = fmin_cobyla(objective, x0=x0, cons=upperb)
            # x_values.append(np.clip(result[0], *DOMAIN[0]))
            # f_values.append(-result[1])
            #print(result)
            result = minimize(objective, x0, bounds=DOMAIN, constraints=const)
            x_values.append(np.clip(result["x"], *DOMAIN[0]))
            f_values.append(-result["fun"])
        ind = np.argmax(f_values)
        x_opt = x_values[ind].item()
        #print(x_values[ind])
        return np.atleast_2d(x_opt)

    def acquisition_function(self, x):
        """
        Compute the acquisition function.

        Parameters
        ----------
        x: np.ndarray
            x in DOMAIN of f

        Returns
        ------
        af_value: float
            Value of the acquisition function at x
        """
        #print(x.shape)
        mean, std = self.gpf.predict(np.atleast_2d(x), return_std=True)
        mean_v, std_v = self.gpv.predict(np.atleast_2d(x), return_std=True)
        nd = norm
        fmax = 1.0
        xi = 0

        # Z = (mean - fmax - xi) / std
        # af_value = (mean - fmax - xi) * nd.cdf(Z) + std * norm.pdf(Z)
        # print(af_value.shape)
        # print((mean_v + std_v)[0])
        af_value = mean +self.beta * std
        if mean_v + std_v * self.beta_v >= SAFETY_THRESHOLD:
            af_value -= self.beta * np.max([mean_v + std_v, 4])

        return np.mean(af_value)

    def add_data_point(self, x, f, v):
        """
        Add data points to the model.

        Parameters
        ----------
        x: np.ndarray
            Hyperparameters
        f: np.ndarray
            Model accuracy
        v: np.ndarray
            Model training speed
        """

        self.x.append(x)
        self.f.append(f)
        self.v.append(v)

        x_n = np.reshape(self.x, [-1,1])
        f_n = np.reshape(self.f, [-1,1])
        v_n = np.reshape(self.v, [-1,1])

        #print(x_n, f_n, v_n)

        self.gpf.fit(x_n, f_n)
        self.gpv.fit(x_n, v_n)

    def get_solution(self):
        """
        Return x_opt that is believed to be the maximizer of f.

        Returns
        -------
        solution: np.ndarray
            1 x DOMAIN.shape[0] array containing the optimal solution of the problem
        """
        mask = np.array(self.v) < SAFETY_THRESHOLD
        f_msk = np.array(self.f)[mask]
        x_msk = np.array(self.x)[mask]
        ind = np.argmax(f_msk)
        return np.atleast_2d(x_msk[ind])

    def plot(self, plot_recommendation: bool = True):
        """Plot objective and constraint posterior for debugging (OPTIONAL).

        Parameters
        ----------
        plot_recommendation: bool
            Plots the recommended point if True.
        """
        plt.ylabel("values for x")
        plt.plot(self.x)
        plt.plot(self.f)
        plt.show()

""" Toy problem to check code works as expected """

def check_in_domain(x: float):
    """Validate input"""
    x = np.atleast_2d(x)
    return np.all(x >= DOMAIN[None, :, 0]) and np.all(x <= DOMAIN[None, :, 1])

def f(x: float):
    """Dummy logP objective"""
    mid_point = DOMAIN[:, 0] + 0.5 * (DOMAIN[:, 1] - DOMAIN[:, 0])
    return - np.linalg.norm(x - mid_point, 2)

def v(x: float):
    """Dummy SA"""
    return 2.0

def get_initial_safe_point():
    """Return initial safe point"""
    x_domain = np.linspace(*DOMAIN[0], 4000)[:, None]
    c_val = np.vectorize(v)(x_domain)
    x_valid = x_domain[c_val < SAFETY_THRESHOLD]
    np.random.seed(0)
    np.random.shuffle(x_valid)
    x_init = x_valid[0]

    return x_init

def main():
    """FOR ILLUSTRATION / TESTING ONLY (NOT CALLED BY CHECKER)."""
    # Init problem
    agent = BO_algo()

    # Add initial safe point
    x_init = get_initial_safe_point()
    obj_val = f(x_init)
    cost_val = v(x_init)
    agent.add_data_point(x_init, obj_val, cost_val)

    # Loop until budget is exhausted
    for j in range(20):
        # Get next recommendation
        x = agent.next_recommendation()

        # Check for valid shape
        assert x.shape == (1, DOMAIN.shape[0]), \
            f"The function next recommendation must return a numpy array of " \
            f"shape (1, {DOMAIN.shape[0]})"

        # Obtain objective and constraint observation
        obj_val = f(x) + np.random.randn()
        cost_val = v(x) + np.random.randn()
        agent.add_data_point(x, obj_val, cost_val)
        agent.plot()

    # Validate solution
    solution = agent.get_solution()
    assert check_in_domain(solution), \
        f'The function get solution must return a point within the' \
        f'DOMAIN, {solution} returned instead'

    # Compute regret
    regret = (0 - f(solution))

    print(f'Optimal value: 0\nProposed solution {solution}\nSolution value '
          f'{f(solution)}\nRegret {regret}\nUnsafe-evals TODO\n')

if __name__ == "__main__":
    main()


import numpy as np
import matplotlib.pyplot as plt
# plt.switch_backend('agg')  # 服务器上跑
import os
import sys
from ready_for_train import Vector_Reader as v_reader
head_path = os.path.dirname(
    os.path.dirname(os.path.abspath(__file__)))
# print(head_path)
sys.path.append(head_path)
import config
from utils import *
from collections import defaultdict
import logging

def _gen_defaultdict():
    # return copy.copy(defaultdict(int))
    return defaultdict(int)

class Coin_Reader(v_reader):
    def __init__(self, input_dir='null', out_put_dir='null'):
        v_reader.__init__(self, input_dir, out_put_dir)

    def _do_read_user_data(self, file_machine, file_bonus, file_payin, file_odds, file_line, seq_len, max_len, mid_line_odds_count):
        """
        max_len : 原始文件的最大序列长度
        """
        # mid_lv_count = [defaultdict(_gen_defaultdict),
        #                 defaultdict(_gen_defaultdict)]
        if (not os.path.exists(file_machine)):
            return False
        f_machine = open(file_machine, 'r')
        f_bonus = open(file_bonus, 'r')
        f_payin = open(file_payin, 'r')
        f_odds = open(file_odds, 'r')
        f_line = open(file_line, 'r')
#            has_zero = False
        while True:
            machine_line = f_machine.readline().strip()
            if not machine_line:
                break
            bonus_line = f_bonus.readline().strip()
            payin_line = f_payin.readline().strip()
            odds_line = f_odds.readline().strip()
            line_line = f_line.readline().strip()

            machine_line = machine_line.split(" ")
            bonus_line = bonus_line.split(" ")
            payin_line = payin_line.split(" ")
            odds_line = odds_line.split(" ")
            line_line = line_line.split(" ")
            # if len(line) < seq_len or line[-1] == "-1":  # 被抛掉的数据
            if machine_line[-1] == '-1':
                continue
            cr_len = len(machine_line)
            # for in line[::-1]
            for i in range(seq_len):
                # cr_index = max(0, cr_len - seq_len + i)
                cr_index = cr_len - seq_len + i
                if cr_index >= cr_len or cr_index < 0:
                    continue
                # print(line[cr_index])
                try:
                    cr_mid = int(float(machine_line[cr_index]))
                    cr_bonus = int(float(bonus_line[cr_index]))
                    cr_payin = int(float(payin_line[cr_index]))
                    cr_odds = float(odds_line[cr_index])
                    cr_line = int(float(line_line[cr_index]))
                    if cr_bonus != 0:
                        cr_odds = (cr_odds * cr_payin + cr_bonus) / cr_payin if cr_payin != 0 else 0
                    cr_odds = round(cr_odds, 1)
                    mid_line_odds_count[cr_mid][cr_line][cr_odds] += 1

                except Exception as ex:
                    logging.exception(ex)
                    print(ex)
                    break
            #         # [cr_mid] += 1
            # f_machine.close()
            # f_bonus.close()
            # f_coin.close()
            # f_time.close()
        # return [data, lable]
        f_machine.close()
        f_bonus.close()
        f_line.close()
        f_payin.close()
        f_odds.close()
        return True

    """
    读玩家数据，继承类可以重写这个函数
    """

    def _read_user_data(self, file_dir, seq_len, max_len):
        """
        max_len : 原始文件的最大序列长度
        """

        arr_dir = file_dir.split(os.path.sep)
        uid = arr_dir[-1]
        # print(uid)
        # exit()

        mid_line_odds_count = defaultdict(lambda: defaultdict(lambda: defaultdict(int)))

        for payed in range(2)[::-1]:
            pre_fix = ""
            if payed == 1:
                pre_fix = 'pay_'
            file_machine = os.path.join(file_dir, pre_fix + "machine_id.txt")
            if (not os.path.exists(file_machine)):
                continue
            file_bonus = os.path.join(file_dir, pre_fix + "win_bonus.txt")
            file_payin = os.path.join(file_dir, pre_fix + "pay_in.txt")
            file_odds = os.path.join(file_dir, pre_fix + "odds.txt")
            file_line = os.path.join(file_dir, pre_fix + "line.txt")
            self._do_read_user_data(
                file_machine, file_bonus, file_payin, file_odds, file_line, seq_len, max_len, mid_line_odds_count)
        # 读drop out的pay记录
        drop_dir = file_dir + os.path.sep + '..' + os.path.sep + \
            '__waring__' + os.path.sep + 'drop_pay' + os.path.sep + uid
        # print(drop_dir)
        file_machine = os.path.join(drop_dir, "machine_id.txt")
        if (os.path.exists(file_machine)):
            file_bonus = os.path.join(drop_dir, "win_bonus.txt")
            file_payin = os.path.join(drop_dir, "pay_in.txt")
            file_odds = os.path.join(drop_dir, "odds.txt")
            file_line = os.path.join(drop_dir, "line.txt")
            self._do_read_user_data(
                file_machine, file_bonus, file_payin, file_odds, file_line, seq_len, max_len, mid_line_odds_count)

        return mid_line_odds_count