"""Drop options table

Revision ID: 2eb9c6cfe0b4
Revises: b4a0653caa88
Create Date: 2020-05-08 16:59:22.535043

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '2eb9c6cfe0b4'
down_revision = 'b4a0653caa88'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('options')
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('options',
    sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False),
    sa.Column('label', sa.VARCHAR(), autoincrement=False, nullable=True),
    sa.Column('question_id', sa.INTEGER(), autoincrement=False, nullable=False),
    sa.Column('next_diagnose_id', sa.INTEGER(), autoincrement=False, nullable=True),
    sa.Column('next_question_id', sa.INTEGER(), autoincrement=False, nullable=True),
    sa.CheckConstraint('((next_question_id IS NULL) AND (next_diagnose_id IS NOT NULL)) OR ((next_question_id IS NOT NULL) AND (next_diagnose_id IS NULL))', name='one-of-two'),
    sa.ForeignKeyConstraint(['next_diagnose_id'], ['diagnoses.id'], name='options_next_diagnose_id_fkey', onupdate='CASCADE', ondelete='CASCADE'),
    sa.ForeignKeyConstraint(['next_question_id'], ['questions.id'], name='options_next_question_id_fkey', onupdate='CASCADE', ondelete='CASCADE'),
    sa.ForeignKeyConstraint(['question_id'], ['questions.id'], name='options_question_id_fkey', onupdate='CASCADE', ondelete='CASCADE'),
    sa.PrimaryKeyConstraint('id', name='options_pkey')
    )
    # ### end Alembic commands ###


from __future__ import annotations
from dataclasses import dataclass, field
from travelport.models.base_rsp_2 import BaseRsp2
from travelport.models.field_1 import Field1
from travelport.models.field_group_1 import FieldGroup1

__NAMESPACE__ = "http://www.travelport.com/schema/sharedUprofile_v20_0"

@dataclass
class ProfileCreateFieldRsp1(BaseRsp2):
    """
    Response containing details of new fields and/or groups created.

    Parameters
    ----------
    field_value
        Defines a field.
    field_group
        Defines a field group.
    """
    class Meta:
        name = "ProfileCreateFieldRsp"
        namespace = "http://www.travelport.com/schema/sharedUprofile_v20_0"

    field_value: list[Field1] = field(
        default_factory=list,
        metadata={
            "name": "Field",
            "type": "Element",
        }
    )
    field_group: list[FieldGroup1] = field(
        default_factory=list,
        metadata={
            "name": "FieldGroup",
            "type": "Element",
        }
    )


#!/usr/bin/env python3

import glob
from gensim.models import KeyedVectors
from scipy.sparse import coo_matrix, hstack
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
import numpy as np
import pickle
from sklearn.utils import resample
from random import shuffle
from sklearn.metrics import accuracy_score, classification_report, confusion_matrix

def create_pairs_all(files):
    #creates pairs from the given files, including all non-causal pairs possible
    list_of_files = files
    pairs = []
    for file_name in list_of_files:
        f = open(file_name, 'r', encoding='utf8')
        lines = f.readlines()
        events = []
        for line in lines:
            splitted  = line.split()
            if len(splitted) > 0:
                if splitted[4] != 'O':
                    events.append([splitted[2], splitted[0], splitted[4],splitted[5] ,splitted[6]]) #getting needed features from file
        for event in events:
            sentence = event[0]
            location = events.index(event)
            for e in events[location+1:]:
                if e[0] == sentence: #checking if event pairs are in same sentence
                    if [file_name[6:], event, e] not in pairs:
                        pairs.append([file_name[6:], event, e])
    return pairs

def create_pairs_select(files):
    #creates pairs from the given files, adds non-causal pairs when sentence contains causal verb or signal (in pickle)
    signal_sents = pickle.load( open( "signal_sents.p", "rb" ) )
    list_of_files = files
    pairs = []
    for file_name in list_of_files:
        f = open(file_name, 'r', encoding='utf8')
        lines = f.readlines()
        events = []
        for line in lines:
            if file_name[6:] in signal_sents.keys(): #check for inclusion in dict of allowed sentences
                splitted  = line.split()
                if len(splitted) > 0:
                    if splitted[2] in signal_sents[file_name[6:]]:
                        if splitted[4] != 'O':
                            events.append([splitted[2], splitted[0], splitted[4],splitted[5] ,splitted[6]]) #getting needed features from file
        for event in events:
            sentence = event[0]
            location = events.index(event)
            for e in events[location+1:]:
                if e[0] == sentence: #checking if event pairs are in same sentence
                    if [file_name[6:], event, e] not in pairs:
                        pairs.append([file_name[6:], event, e])
    return pairs

def find_training_files():
    #returns all filenames that will be used for training
    return glob.glob('train/*.col')

def find_testing_files():
    #returns all filenames that will be used for training
    f = open('test.txt', 'r')
    lines = f.readlines()
    files = []
    for line in lines:
        items = line.split('\t')
        filename = items[0]
        files.append('tests/'+filename)
    return files

def get_causal_from_file(filename):
    #making a dictionairy of causal pairs to stimulate fast look-up, key = filename, values = causal event=pairs
    d = {}
    f = open(filename, 'r')
    lines = f.readlines()
    for line in lines:
        items = line.split('\t')
        if items[0] in d.keys():
            d[items[0]].append((items[1], items[2]))
        else:
            d.update({items[0]: [(items[1], items[2])]})
    return d

def get_science_causal():
    #making a dictionairy of causal pairs to stimulate fast look-up, key = sentence nr., values = causal event=pairs
    d = {}
    f = open('tests/science.clink', 'r')
    lines = f.readlines()
    for line in lines:
        items = line.split('\t')
        pairs = items[1:]
        for pair in pairs:
            parts = pair.split()
            if items[0] in d.keys():
                d[items[0]].append((parts[0], parts[2]))
            else:
                d.update({items[0]: [(parts[0], parts[2])]})
    return d

def science_edit(pairs):
    #making scienceset applicable to functions made for the other sets
    for pair in pairs:
        pair[0] = pair[1][0]
    return pairs

def science_edit_reverse(pairs):
    #making scienceset applicable to functions made for the other sets < undoing this
    for pair in pairs:
        pair[0] = 'science.col'
    return pairs

def feature_sets(train, test):
    #getting all possible values for both features and making sure there are no doubles
    feature2 = []
    feature3 = []

    for x in train:
        feature2.append(x[1][3])
        feature2.append(x[2][3])
        feature3.append(x[1][4])
        feature3.append(x[2][4])
    for y in test:
        feature2.append(y[1][3])
        feature2.append(y[2][3])
        feature3.append(y[1][4])
        feature3.append(y[2][4])

    return list(set(feature2)), list(set(feature3))

def causal_pairs(pairs, causal):
    #checking causality for every pair
    pairs_causality = []
    for pair in pairs:
        pairx = pair
        if pair[0] in causal.keys():
            if (pair[1][2], pair[2][2]) in causal[pair[0]]:
                pairx.append(1)
            else:
                pairx.append(0)
        else:
            pairx.append(0)
        pairs_causality.append(pairx)
    return pairs_causality

def balancing_over(pairs_causality):
    #add random instances of minority class until both classes have equal size
    causal = []
    non_causal = []
    for pair in pairs_causality:
        if pair[3] == 1:
            causal.append(pair)
        else:
            non_causal.append(pair)
    diff = len(pairs_causality) - len(causal)
    causal = resample(causal, n_samples=diff, random_state=0)
    causal += non_causal
    shuffle(causal)
    return causal

def balancing_under(pairs_causality):
    #remove random instances of majority class until both classes have equal size
    signal_sents = pickle.load( open( "signal_sents.p", "rb" ) )
    causal = []
    non_causal = []
    for pair in pairs_causality:
        if pair[3] == 1:
            causal.append(pair)
        else:
            if pair[0] in signal_sents.keys(): #remove selected sentences that do not contain causal signal or verb.
                if pair[1][0] in signal_sents[pair[0]]:
                    non_causal.append(pair)
    num_pairs = len(causal) + 1
    shuffle(non_causal)
    causal += non_causal[0:num_pairs]
    return causal

def test_select_non_causal(pairs_causality):
    #only contain selected sentences in data to lower number of non-causal.
    signal_sents = pickle.load( open( "signal_sents.p", "rb" ) )
    causal = []
    non_causal = []
    for pair in pairs_causality:
        if pair[3] == 1:
            causal.append(pair)
        else:
            if pair[0] in signal_sents.keys():
                if pair[1][0] in signal_sents[pair[0]]:
                    non_causal.append(pair)

    causal += non_causal
    return causal

def convert_to_features(pairs,model, featset2, featset3):
    #make the features machine readable
    feats = []
    y = []
    vectoruniversal = model['the']
    for pair in pairs:
        try:
            vector1 = model[pair[1][1]]

        except:
            vector1 = vectoruniversal
        try:
            vector2 = model[pair[2][1]]
        except:
            vector2 = vectoruniversal
        feats.append(vector1.tolist() + [featset2.index(pair[1][3]), featset3.index(pair[1][4])] + vector2.tolist() + [featset2.index(pair[2][3]), featset3.index(pair[2][4])])
        y.append(pair[3])

    return feats, y

def main():

	#finding all files that will be used for training and testing:
    training_files = find_training_files()
    testing_files = find_testing_files()
    print('...Progress Update: done finding files')

	#Opening files and creating event pairs:
    pairs_train = create_pairs_all(training_files) # change '_all' to '_select' for other approach
    pairs_test = create_pairs_all(testing_files)
    science_train = pickle.load( open( "science_train_all.p", "rb" ) ) #using pickles for science data because of very long loading times
    science_test = pickle.load( open( "science_test.p", "rb" ) )
    science_train = science_edit(science_train)
    science_test = science_edit(science_test)
    print('...Progress Update: done creating pairs')

        #creating sets of features, .index() of features will later be used to numerize the features
    feature2set, feature3set = feature_sets(pairs_train, pairs_test)

	#creating dictionaries to easily look up causal pairs
    training_causal = get_causal_from_file('train.txt')
    testing_causal = get_causal_from_file('test.txt')
    science_causal = get_science_causal()
    print('...Progress Update: done making dictionairies')

	#adding the feature causal (yes or no) to the pairs of events
    causal_pairs_train = causal_pairs(pairs_train, training_causal)
    causal_pairs_test = causal_pairs(pairs_test, testing_causal)
    causal_pairs_science_train = causal_pairs(science_train, science_causal)
    causal_pairs_science_test = causal_pairs(science_test, science_causal)
    causal_pairs_science_train = science_edit_reverse(causal_pairs_science_train)
    causal_pairs_science_test = science_edit_reverse(causal_pairs_science_test)
    print('...Progress Update: done adding causality')

	#undersampling the minority class to avoid imbalanced data:
    causal_pairs_train = balancing_under(causal_pairs_train)
    causal_pairs_test = test_select_non_causal(causal_pairs_test)
    causal_pairs_science_train = balancing_under(causal_pairs_science_train)
    causal_pairs_science_test = test_select_non_causal(causal_pairs_science_test)
    print('...Progress Update: done balancing classes')
        #preparing the features for training:
    model = KeyedVectors.load_word2vec_format('GoogleNews-vectors-negative300.bin', binary=True, limit=1500000)
    train_x, train_y =  convert_to_features(causal_pairs_train, model, feature2set, feature3set)
    test_x, test_y = convert_to_features(causal_pairs_test, model,feature2set, feature3set)
    sciencetrain_x, sciencetrain_y = convert_to_features(causal_pairs_science_train, model, feature2set, feature3set)
    sciencetest_x, sciencetest_y = convert_to_features(causal_pairs_science_test, model, feature2set, feature3set)

    print('...Progress Update: done converting to features')
    print('...Starting the model:' )
    rf = LogisticRegression()
    #uncomment these to test on whole set
    #train_x += sciencetrain_x
    #train_y += sciencetrain_y
    #test_x += sciencetest_x
    #test_y += sciencetest_y

    rf.fit(train_x,train_y)
    predictions = rf.predict(test_x)
    accuracy = accuracy_score(test_y, predictions)

        # printing scores

    print('Accuracy: ', accuracy)
    cm = confusion_matrix(test_y, predictions)
    print('Confusion Matrix: \n')
    print(cm)
    target_names = ['class 0', 'class 1']
    print(classification_report(test_y, predictions, target_names=target_names))
    sent_dict_test = pickle.load( open( "sent_dict_test.p", "rb" ) ) #loading pickle containing every sentence of the test files, in a dictionairy
    i = 0
    f = open("name.out", "w") #choose filename to write to
        #writing all sentences with classified pairs to file
    for pred in predictions:
           filename = causal_pairs_test[i][0] #change to 'science.col' for science set
           sent = causal_pairs_test[i][1][0]#use causal_pairs_science_test on these for the science set
           e1 = causal_pairs_test[i][1][2]
           e2 = causal_pairs_test[i][2][2]
           string = sent_dict_test[filename][sent] + ' ' +  e1 + ' ' + e2 + ' '+ str(pred)+'\n'
           f.write(string)
           i += 1
    f.close()

if __name__ == '__main__':
	main()


# -*- coding: utf-8 -*-

from __future__ import unicode_literals

import logging

from flask import render_template, flash, redirect, url_for
from flask_login import login_required, current_user
from sqlalchemy.exc import IntegrityError

import pybel
from .constants import integrity_message
from .extension import get_manager
from .forms import UploadForm
from .models import add_network_reporting

log = logging.getLogger(__name__)

def build_pickle_uploader_service(app):
    """Adds the endpoints for uploading pickle files

    :param flask.Flask app: A Flask application
    """
    manager = get_manager(app)

    @app.route('/upload', methods=['GET', 'POST'])
    @login_required
    def view_upload():
        """An upload form for a BEL script"""
        form = UploadForm()

        if not form.validate_on_submit():
            return render_template('upload.html', form=form, current_user=current_user)

        log.info('uploading %s', form.file.data.filename)

        try:
            graph_bytes = form.file.data.read()
            graph = pybel.from_bytes(graph_bytes)
        except Exception as e:
            log.exception('gpickle error')
            flash('The given file was not able to be unpickled [{}]'.format(str(e)), category='error')
            return redirect(url_for('view_upload'))

        try:
            network = manager.insert_graph(graph)
        except IntegrityError:
            log.exception('integrity error')
            flash(integrity_message.format(graph.name, graph.version), category='error')
            manager.rollback()
            return redirect(url_for('view_upload'))
        except Exception as e:
            log.exception('upload error')
            flash("Error storing in database [{}]".format(e), category='error')
            return redirect(url_for('view_upload'))

        log.info('done uploading %s [%d]', form.file.data.filename, network.id)

        try:
            add_network_reporting(manager, network, current_user, graph.number_of_nodes(),
                                  graph.number_of_edges(), len(graph.warnings), precompiled=True,
                                  public=form.public.data)
        except IntegrityError:
            log.exception('integrity error')
            flash('problem with reporting service', category='warning')
            manager.rollback()

        return redirect(url_for('view_summary', graph_id=network.id))

    log.info('Added pickle uploader endpoint to %s', app)


from django.shortcuts import render
from .forms import UserLoginForm
from django.shortcuts import redirect
from django.contrib import auth

# Create your views here.
def login(request):
    if request.method == "POST":
        form = UserLoginForm(request.POST)
        if form.is_valid():
            user = form.login(request)
            if user:
                return redirect('main')
            else:
                statement = form.user_statement()
                return render(request, 'login.html', {'form': form, 'login_is_failed': statement['incorrect'], 'need_authentication': statement['uncertified_email']})
        else:
            return redirect('login')
    else:
        if request.user.is_authenticated:
            return redirect('main')
        form = UserLoginForm()
        return render(request, 'login.html', {'form': form})

def logout(request):
    auth.logout(request)
    return redirect('login')


import time
from octoprint.printer.standard import Printer, StateMonitor
from octoprint.events import eventManager, Events
from octoprint_mrbeam.mrbeam_events import MrBeamEvents
from octoprint_mrbeam.printing import comm_acc2 as comm
from octoprint_mrbeam.mrb_logger import mrb_logger
from octoprint_mrbeam.filemanager.analysis import beam_analysis_queue_factory
from octoprint_mrbeam.util import dict_merge

class Laser(Printer):
    HOMING_POSITION = [-1.0, -1.0, 0]

    def __init__(self, fileManager, analysisQueue, printerProfileManager):
        # MR_BEAM_OCTOPRINT_PRIVATE_API_ACCESS -- start --
        # TODO OP v1.4 : Remove the followingline - see octoprint_mrbeam.__plugin_load__
        # The necessary lines for this to keep working are already written as a hook.
        analysisQueue._queues.update(
            beam_analysis_queue_factory(callback=analysisQueue._analysis_finished)
        )
        # MR_BEAM_OCTOPRINT_PRIVATE_API_ACCESS -- end --
        Printer.__init__(self, fileManager, analysisQueue, printerProfileManager)
        self._logger = mrb_logger("octoprint.plugins.mrbeam.printing.printer")
        self._stateMonitor = LaserStateMonitor(
            interval=0.5,
            on_update=self._sendCurrentDataCallbacks,
            on_add_temperature=self._sendAddTemperatureCallbacks,
            on_add_log=self._sendAddLogCallbacks,
            on_add_message=self._sendAddMessageCallbacks,
            on_get_progress=self._updateProgressDataCallback,
        )
        self._stateMonitor.reset(
            state={"text": self.get_state_string(), "flags": self._getStateFlags()},
            job_data={
                "file": {"name": None, "size": None, "origin": None, "date": None},
                "estimatedPrintTime": None,
                "lastPrintTime": None,
                "filament": {"length": None, "volume": None},
            },
            progress={
                "completion": None,
                "filepos": None,
                "printTime": None,
                "printTimeLeft": None,
            },
            current_z=None,
        )
        self._event_bus = eventManager()
        self._event_bus.subscribe(
            MrBeamEvents.LASER_JOB_ABORT, self._on_laser_job_abort
        )
        self._event_bus.subscribe(
            MrBeamEvents.LASER_COOLING_RE_TRIGGER_FAN, self._on_re_trigger_fan
        )

    # overwrite connect to use comm_acc2
    def connect(self, port=None, baudrate=None, profile=None):
        """Connects to the printer.

        If port and/or baudrate is provided, uses these settings,
        otherwise autodetection will be attempted.
        """
        self._init_terminal()

        if self._comm is not None:
            self._comm.close()

        eventManager().fire(Events.CONNECTING, payload=dict(profile=profile))
        self._printerProfileManager.select(profile)
        self._comm = comm.MachineCom(
            port,
            baudrate,
            callbackObject=self,
            printerProfileManager=self._printerProfileManager,
        )

    # overwrite operational state to accept commands in locked state
    def is_operational(self):
        return Printer.is_operational(self) or self.is_locked()

    # send color settings to commAcc to inject settings into Gcode
    def set_colors(self, currentFileName, value):
        if self._comm is None:
            return
        self._comm.setColors(currentFileName, value)

    # extend commands: home, position, increase_passes, decrease_passes
    def home(self, axes):
        printer_profile = self._printerProfileManager.get_current_or_default()
        params = dict(
            x=printer_profile["volume"]["width"]
            + printer_profile["volume"]["working_area_shift_x"],
            y=printer_profile["volume"]["depth"]
            + printer_profile["volume"]["working_area_shift_y"],
            z=0,
        )
        self._comm.rescue_from_home_pos()
        command = "G92X{x}Y{y}Z{z}".format(**params)
        self.commands(["$H", command, "G90", "G21"])

    def is_homed(self):
        return self._stateMonitor._machinePosition == self.HOMING_POSITION

    def cancel_print(self):
        """Cancel the current printjob and do homing."""
        super(Laser, self).cancel_print()
        time.sleep(0.5)
        self.home(axes="wtf")
        eventManager().fire(MrBeamEvents.PRINT_CANCELING_DONE)

    def fail_print(self, error_msg=None):
        """Cancel the current printjob (as it failed) and do homing."""
        if self._comm is None:
            return

        # If we want the job to show as failed instead of cancelled, we have to mimic self._printer.cancel_print()
        self._comm.cancelPrint(failed=True, error_msg=error_msg)

        time.sleep(0.5)
        self.home(axes="wtf")
        eventManager().fire(MrBeamEvents.PRINT_CANCELING_DONE)

    def abort_job(self, event):
        """Abort the current printjob and do homing. This is a hard abort, the compressor and fan should be immediately turned off.

        Args:
            event: event that triggered the abort

        Returns:
            None
        """
        self._comm.abort_lasering(event)
        time.sleep(0.5)
        self.home(axes="wtf")
        self._event_bus.fire(MrBeamEvents.LASER_JOB_ABORTED, {"trigger": event})

    def position(self, x, y):
        printer_profile = self._printerProfileManager.get_current_or_default()
        movement_speed = min(
            printer_profile["axes"]["x"]["speed"], printer_profile["axes"]["y"]["speed"]
        )
        self.commands(["G90", "G0 X%.3f Y%.3f F%d" % (x, y, movement_speed)])

    def increase_passes(self):
        """increase the number of passes by one."""
        if self._comm is None:
            return
        self._comm.increasePasses()

    def set_passes(self, value):
        if self._comm is None:
            return
        self._comm.setPasses(value)

    def decrease_passes(self):
        """decrease the number of passes by one."""
        if self._comm is None:
            return
        self._comm.decreasePasses()

    def pause_print(self, force=False, trigger=None):
        """Pause the current printjob."""
        if self._comm is None:
            return

        if not force and self._comm.isPaused():
            return

        self._comm.setPause(True, send_cmd=True, trigger=trigger)

    def resume_print(self, trigger=None):
        """Resume the current laser job."""
        if self._comm is None:
            return

        if not self._comm.isPaused():
            return

        self._comm.setPause(False, send_cmd=True, trigger=trigger)

    def cooling_start(self):
        """Pause the laser for cooling."""
        if self._comm is None:
            return

        if self._comm.isPaused():
            return

        self._comm.setPause(True, pause_for_cooling=True, trigger="Cooling")

    # extend flags
    def is_locked(self):
        return self._comm is not None and self._comm.isLocked()

    def is_flashing(self):
        return self._comm is not None and self._comm.isFlashing()

    def _getStateFlags(self):
        # Extra gymnastics in case state flags are a frozen dict
        flags = Printer._getStateFlags(self)
        _dict = flags.__class__
        flags = dict_merge(
            {
                "locked": self.is_locked(),
                "flashing": self.is_flashing(),
            },
            flags,
        )
        return _dict(flags)

    # position update callbacks
    def on_comm_pos_update(self, MPos, WPos):
        self._add_position_data(MPos, WPos)

    # progress update callbacks
    def on_comm_progress(self):
        self._updateProgressData(
            self._comm.getPrintProgress(),
            self._comm.getPrintFilepos(),
            self._comm.getPrintTime(),
            self._comm.getCleanedPrintTime(),
        )
        self._stateMonitor.trigger_progress_update()

    def _add_position_data(self, MPos, WPos):
        if MPos is not None:
            self._stateMonitor.setMachinePosition(MPos)
        if WPos is not None:
            self._stateMonitor.setWorkPosition(WPos)

    def _init_terminal(self):
        from collections import deque

        terminalMaxLines = _mrbeam_plugin_implementation._settings.get(
            ["dev", "terminalMaxLines"]
        )
        if terminalMaxLines is not None and terminalMaxLines > 0:
            self._log = deque(self._log, terminalMaxLines)

    def _on_laser_job_abort(self, event, payload):
        """Abort the job if the laser is lasering or paused.

        Args:
            event: event that triggered the action
            payload: payload of the event

        Returns:
            None
        """
        if self.is_lasering() or self.is_paused():
            self._logger.warn(
                "Firing warning, will abort print e:%s payload: %s", event, payload
            )
            self.abort_job(event)

    def is_lasering(self):
        return self.is_printing()

    def _on_re_trigger_fan(self, event, payload):
        """Re trigger the fan if the laser is printing or paused.
        This can be done by sending atomic command start and pause after each other.

        Args:
            event: event that triggered the action
            payload: payload of the event

        Returns:
            None
        """
        if self.is_printing() or self.is_paused():
            self._logger.info("Re triggering fan e:%s payload: %s", event, payload)
            self._comm.retrigger_cooling_fan()

    # maybe one day we want to introduce special MrBeam commands....
    # def commands(self, commands):
    # 	"""
    # 	Sends one or more gcode commands to the printer.
    # 	"""
    # 	if self._comm is None:
    # 		return
    #
    # 	if not isinstance(commands, (list, tuple)):
    # 		commands = [commands]
    #
    # 	for command in commands:
    # 		self._logger.debug("Laser.commands() %s", command)
    # 		sendCommandToPrinter = True
    # 		if _mrbeam_plugin_implementation is not None:
    # 			sendCommandToPrinter = _mrbeam_plugin_implementation.execute_command(command)
    # 		if sendCommandToPrinter:
    # 			self._comm.sendCommand(command)

class LaserStateMonitor(StateMonitor):
    def __init__(self, *args, **kwargs):
        StateMonitor.__init__(self, *args, **kwargs)
        self._machinePosition = None
        self._workPosition = None

    def setWorkPosition(self, workPosition):
        self._workPosition = workPosition
        self._change_event.set()

    def setMachinePosition(self, machinePosition):
        self._machinePosition = machinePosition
        self._change_event.set()

    def get_current_data(self):
        data = StateMonitor.get_current_data(self)
        data.update(
            {
                "workPosition": self._workPosition,
                "machinePosition": self._machinePosition,
            }
        )
        mrb_state = _mrbeam_plugin_implementation.get_mrb_state()
        if mrb_state:
            data["mrb_state"] = mrb_state
        return data


'''
Descripttion:
version:
Author: Shicript
Date: 2021-06-22 20:21:07
LastEditors: Shicript
LastEditTime: 2021-06-23 10:19:51
'''
#
# @lc app=leetcode.cn id=13 lang=python3
#
# [13] 罗马数字转整数
#
# @lc code=start
from functools import reduce

class Solution:
    def romanToInt(self, s: str) -> int:
        s_reversed = list(reversed(s))
        node = RomaNode(s_reversed[0])
        s_reversed = [node] + s_reversed[1:]
        node = reduce(self.build_node, s_reversed)
        return self.get_node_value(node)

    @staticmethod
    def get_node_value(node):
        left_char_dict = {"I": {"V": 4, "X": 9}, "X": {
            "L": 40, "C": 90}, "C": {"D": 400, "M": 900}}
        char_num_dict = {
            "I": 1,
            "V": 5,
            "X": 10,
            "L": 50,
            "C": 100,
            "D": 500,
            "M": 1000
        }
        res_number = 0
        end_with_left = False
        while node.next:
            value1 = node.value
            node = node.next
            value2 = node.value
            if value1 in left_char_dict.keys():
                if value2 in left_char_dict[value1].keys():
                    res_number += left_char_dict[value1][value2]
                    if node.next:
                        node = node.next
                    else:
                        end_with_left = True
                    continue
            res_number += char_num_dict[value1]
            end_with_left = False
        if not end_with_left:
            res_number += char_num_dict[node.value]
        return res_number

    @staticmethod
    def build_node(node1, char2):
        node2 = RomaNode(char2)
        node2.next = node1
        return node2

class RomaNode:
    def __init__(self, x):
        self.value = x
        self.next = None

# @lc code=end

13.罗马数字转整数.py

13.罗马数字转整数.py


from typing import Tuple

from dagster import Out, Output, op
from pandas import DataFrame
from pyspark.sql import DataFrame as SparkDF
from pyspark.sql.types import (
    ArrayType,
    BooleanType,
    DoubleType,
    LongType,
    StringType,
    StructField,
    StructType,
)

HN_ACTION_SCHEMA = StructType(
    [
        StructField("id", LongType()),
        StructField("parent", DoubleType()),
        StructField("time", LongType()),
        StructField("type", StringType()),
        StructField("by", StringType()),
        StructField("text", StringType()),
        StructField("kids", ArrayType(LongType())),
        StructField("dead", BooleanType()),
        StructField("score", DoubleType()),
        StructField("title", StringType()),
        StructField("descendants", DoubleType()),
        StructField("url", StringType()),
    ]
)

ACTION_FIELD_NAMES = [field.name for field in HN_ACTION_SCHEMA.fields]

@op(
    out={"items": Out(io_manager_key="parquet_io_manager", dagster_type=DataFrame)},
    required_resource_keys={"hn_client"},
    description="Downloads all of the items for the id range passed in as input and creates a DataFrame with all the entries.",
)
def download_items(context, id_range: Tuple[int, int]) -> Output:
    """
    Downloads all of the items for the id range passed in as input and creates a DataFrame with
    all the entries.
    """
    start_id, end_id = id_range

    context.log.info(f"Downloading range {start_id} up to {end_id}: {end_id - start_id} items.")

    rows = []
    for item_id in range(start_id, end_id):
        rows.append(context.resources.hn_client.fetch_item_by_id(item_id))
        if len(rows) % 100 == 0:
            context.log.info(f"Downloaded {len(rows)} items!")

    non_none_rows = [row for row in rows if row is not None]

    return Output(
        DataFrame(non_none_rows).drop_duplicates(subset=["id"]),
        "items",
        metadata={"Non-empty items": len(non_none_rows), "Empty items": rows.count(None)},
    )

@op(
    out=Out(
        io_manager_key="warehouse_io_manager",
        metadata={"table": "hackernews.comments"},
    ),
    description="Creates a dataset of all items that are comments",
)
def build_comments(context, items: SparkDF) -> SparkDF:
    context.log.info(str(items.schema))
    return items.where(items["type"] == "comment").select(ACTION_FIELD_NAMES)

@op(
    out=Out(
        io_manager_key="warehouse_io_manager",
        metadata={"table": "hackernews.stories"},
    ),
    description="Creates a dataset of all items that are stories",
)
def build_stories(context, items: SparkDF) -> SparkDF:
    context.log.info(str(items.schema))
    return items.where(items["type"] == "story").select(ACTION_FIELD_NAMES)


from flask import Flask, render_template, redirect, request

app = Flask(__name__)

distance = 0.1

@app.route("/")
def home():
    global distance
    return render_template('test8/page.html', distance=distance)

@app.route("/setdistance", methods=["POST"])
def setdistance():
    global distance
    distance = float(request.form["distance"])
    print("set distance to", distance)
    return redirect(request.referrer)

app.run(debug = True)

Python/Get started with flask/8 request form.py


import matplotlib.pyplot as plt
import pylab as p
import numpy as np

t1 = p.arange(0, 5, 0.1)
t2 = p.arange(0, 2, 0.05)
x1 = np.cos(2*np.pi*t1)*np.exp(-t1)
x2 = np.cos(2*np.pi*t2)
plt.subplot(2, 1, 1)
plt.plot(t1, x1, marker='.')
plt.ylabel("Oscylacja tłumiona")
plt.subplot(2, 1, 2)
plt.plot(t2, x2, marker='.')
plt.ylabel("Oscylacja harmoniczna")
plt.xlabel("Czas (s)")
plt.show()


"""
把繁体的ipynb、md和py文档转化为简体。

opencc -c tw2s.json -i "01 Jupyter 玩乐程式.ipynb"  -o "01 Jupyter玩乐程式.ipynb"
ffmpeg -i "Python pandas—Deleting—Wine.mp4" -vf "subtitles='Python pandas—Deleting—Wine.srt':force_style='Alignment=6'"  test1.mp4
"""
#遍历所有文件
import os
import opencc

cc = opencc.OpenCC('tw2s.json')

g = os.walk("/Users/rayhe/Desktop/Book/Python-Finance-Data-Science/misc")
for path, dir_list, file_list in g:
    for file_name in file_list:
        if os.path.splitext(file_name)[1] in [".md", ".py", ".ipynb"]:
            f_n = os.path.join(path, file_name)
            if '00' in file_name:
                os.system('rm -f "'+f_n+'"')
                print('成功删除：' + file_name)
            else:
                file_name_sc = cc.convert(file_name)
                f_n_sc = os.path.join(path, file_name_sc)
                os.system('opencc -c tw2s.json -i "'+f_n+'" -o "'+f_n_sc+'"')
                print('成功转换：'+file_name)
                if f_n_sc != f_n:
                    os.system('rm -f "'+f_n+'"')


from django.core.management.base import BaseCommand, CommandError
from optparse import make_option
from hlidskjalf.importers.movies import MoviesImporter

class Command(BaseCommand):

    option_list = BaseCommand.option_list + (
        make_option('-i', '--input', action='store', dest='file', default=False, help='Input file'),
        make_option('-n', '--name', action='store', dest='name', default=False, help='Set name'),
        make_option('-l', '--lang', action='store', dest='lang', default="en", help='Set language'),
        make_option('-m', '--more', action='store_true', dest='verbose', default=False, help='Verbose output'),
    )

    def handle(self, *args, **options):
        if not options['file'] or not options['name']:
            raise CommandError('File and set name are necessary')

        result = MoviesImporter(options['name'], options['file'], options['lang']).run()
        if options['verbose']:
            self.stdout.write("Added: %s" % result)


# DVS image reconstruction demo
#
# Given DVS events and camera poses (rotational motion), reconstruct the
# gradient map that caused the events.
#
# Celine Nauer, Institute of Neuroinformatics, University of Zurich
# Joel Bachmann, ETH
# Nik Dennler, Institute of Neuroinformatics, University of Zurich
#
# Original MATLAB code by
#  Guillermo Gallego
#  Robotics and Perception Group
#  University of Zurich

import os
import time
import math

from scipy import io
import numpy as np
import pandas as pd
import matplotlib

matplotlib.use("TkAgg")
import matplotlib.pyplot as plt
import pyquaternion
import pylab

# Provide function handles to convert from rotation matrix to axis-angle and vice-versa
import sample.helpers.integration_methods as integration_methods
import sample.helpers.coordinate_transforms as coordinate_transforms
import sample.helpers.helpers as helpers

## Run settings:
num_events_batch = 3000
num_events_display = 600000
scale_res = 1  # Use Zweierpotenz
plot_events_animation = False  # True
plot_events_pm_animation = False  # True

# Methods used:
# 1) Select measurement function used for brightness gradient estimation
#    via Extended Kalman Filter (EKF). Options are:
#    'contrast'    : Constrast criterion (Gallego et al. arXiv 2015)
#    'event_rate'  : Event rate criterion (Kim et al. BMVC 2014)
measurement_criterion = "contrast"

## ___Dataset___

time_0 = time.time()
# Data directories
data_dir = "../data/Datasets/FPU/Regal_z"
data_dir = "../data/Datasets/RedRoom/second"
# data_dir = '../data/Datasets/BigRoom/2019-04-29-17-20-59'
calibration_dir = "../data/calibration"
output_dir = "../output/ourdataset"
images_dir = os.path.join(
    output_dir, "{0}pbatch_{1}".format(num_events_batch, measurement_criterion)
)
print(images_dir)
if not os.path.exists(images_dir):
    os.makedirs(images_dir)

# Calibration data
dvs_calibration = io.loadmat(
    os.path.join(calibration_dir, "DVS_synth_undistorted_pixels_rroom.mat")
)
# Dictionary with entries ['__header__', '__version__', '__globals__',
#                          'Kint', 'dist_coeffs', 'image_height', 'image_width',
#                          'pixels_grid', 'undist_pix', 'undist_pix_calibrated',
#                          'undist_pix_mat_x', 'undist_pix_mat_y'])

dvs_parameters = {
    "sensor_height": 260,
    "sensor_width": 346,
    "contrast_threshold": 0.202,
}

## ___Main algorithmic parameters___

# Size of the output reconstructed image mosaic (panorama)
output_height = int(1024 / scale_res)
output_width = 2 * output_height

# Set the expected noise level (variance of the measurements).
if measurement_criterion == "contrast":
    var_R = 0.035 ** 2  # units[C_th] ^ 2
if measurement_criterion == "event_rate":
    var_R = 1e2 ** 2  # units[1 / second] ^ 2

# 2) Select the gradient integration method. Options are:
#    'poisson_dirichlet'   : Requires the MATLAB PDE toolbox
#    'poisson_neumann'     : Requires the MATLAB Image Processing toolbox
#    'frankotchellappa'    : Does not require a MATLAB toolbox
integration_method = "frankotchellappa"

## Loading Events
print("Loading Events")
filename_events = os.path.join(data_dir, "events.txt")
filename_events = os.path.join(data_dir, "events_cropped.txt")

events = pd.read_csv(
    filename_events, delimiter=" ", header=None, names=["time", "x", "y", "pol"]
)
print(events.head())

# print("Head: \n", events.head(10))
num_events = events.size
print("Number of events in file: ", num_events)
first_event = events.loc[0, "time"]
print(first_event)
events["t"] = events["time"] - first_event
events = events[["t", "x", "y", "pol"]]
print(events.head())

# print("Head: \n", events.head(10))
# print("Tail: \n", events.tail(10))

##Loading Camera poses
# print("Loading Camera Orientations")
filename_poses = os.path.join(data_dir, "imu.txt")
# filename_poses = os.path.join(data_dir, 'poses_orb_euler.txt')

poses = helpers.load_poses_angvel(
    filename_poses=filename_poses, includes_translations=True, t_first_event=first_event
)
# print(poses.head())
# exit()

# poses = pd.read_csv(filename_events, delimiter=' ', header=None, names=['time', 'x', 'y', 'qx', 'qy', 'qz', 'qw'])
num_poses = poses.size
print("Number of poses in file: ", num_poses)

# print(first_event)
# poses['t'] = poses['time'] - poses['time'].loc[0]
# poses = poses[['t', 'qw', 'qx', 'qy', 'qz']] # Quaternions
print("Head: \n", poses.head(10))
print("Tail: \n", poses.tail(10))

# exit()
# Convert quaternions to rotation matrices and save in a dictionary TODO: UGLY AS HELL!!
rotmats_dict = coordinate_transforms.angvel2R_dict(
    poses
)  # TODO: Not tested, but seems reasonable
# print(rotmats_dict)
# exit()

## Image reconstruction using pixel-wise EKF
# Input: events, contrast threshold and camera orientation (discrete poses + interpolation)
# Output: reconstructed intensity image
# profile('on','-detail','builtin','-timer','performance')
starttime = time.time()

# num_events_batch = 300
# num_events_display = 100000
num_batches_display = math.floor(num_events_display / num_events_batch)

# Variables related to the reconstructed image mosaic. EKF initialization
# Gradient map
grad_map = {}
grad_map["x"] = np.zeros((output_height, output_width))
grad_map["y"] = np.zeros((output_height, output_width))
# % Covariance matrix of each gradient pixel
grad_initial_variance = 50
grad_map_covar = {}
grad_map_covar["xx"] = np.ones((output_height, output_width)) * grad_initial_variance
grad_map_covar["xy"] = np.zeros((output_height, output_width))
grad_map_covar["yx"] = np.zeros((output_height, output_width))
grad_map_covar["yy"] = np.ones((output_height, output_width)) * grad_initial_variance

# For efficiency, a structure, called event map, contains for every pixel
# the time of the last event and its rotation at that time.
s = {"sae": -1e-6, "rotation": np.zeros((3, 3))}
s["rotation"].fill(np.NaN)
event_map = np.zeros((dvs_parameters["sensor_height"], dvs_parameters["sensor_width"]))
event_map = event_map.tolist()
for i in range(dvs_parameters["sensor_height"]):
    for j in range(dvs_parameters["sensor_width"]):
        event_map[i][j] = s.copy()
event_map = np.array(event_map)
print(event_map.shape)

rotmats_1stkey = list(rotmats_dict.keys())[0]
rot0 = rotmats_dict[rotmats_1stkey]  # to later center the map around the first pose
one_vec = np.ones((num_events_batch, 1))  # ??
Id = np.eye(2)  # ??

## Processing events
print("Processing events")
# plt.figure()
# fig_show_evol = plt.figure(facecolor='w')
# units = normalized should be true
first_plot = True  # for efficient plotting

iEv = 0  # event counter
iBatch = 1  # packet-of-events counter

i = 0
# counter = -1
while True:
    # counter += 1
    # print("Here")
    # print(iEv)
    # print(num_events_batch)
    # print(num_events.loc['time'])
    # exit()
    if iEv + num_events_batch > num_events:
        print("No more events")
        break

    #% Get batch of events
    events_batch = events[iEv : iEv + num_events_batch]
    iEv = iEv + num_events_batch

    events_batch_pos = events_batch[events_batch["pol"] == 1]
    events_batch_neg = events_batch[events_batch["pol"] == 0]

    t_events_batch = events_batch["t"]
    x_events_batch = events_batch["x"].astype(int)
    y_events_batch = events_batch["y"].astype(int)
    pol_events_batch = 2 * (events_batch["pol"] - 0.5)

    ## Get the two map points correspondig to each event and update the event map (time and rotation of last event)

    # Get time of previous event at same DVS pixel
    idx_to_mat = x_events_batch * dvs_parameters["sensor_height"] + y_events_batch
    # print(idx_to_mat)

    t_prev_batch = np.array(
        [
            event_map[int(y), int(x)]["sae"]
            for x, y in zip(x_events_batch, y_events_batch)
        ]
    ).T

    # Get (interpolated) rotation of current event
    first_idx = t_events_batch.index[0]
    last_idx = t_events_batch.index[-1]
    t_ev_mean = (t_events_batch.loc[first_idx] + t_events_batch.loc[last_idx]) * 0.5
    if t_ev_mean > poses["t"].iloc[-1]:
        print("Event later than last known pose")
        break  # event later than last known pose

    Rot = coordinate_transforms.rotation_interpolation(
        poses["t"], rotmats_dict, t_ev_mean
    )

    try:
        bearing_vec = np.vstack(
            (
                dvs_calibration["undist_pix_calibrated"][idx_to_mat, :].T[0],
                dvs_calibration["undist_pix_calibrated"][idx_to_mat, :].T[1],
                one_vec[:, 0],
            )
        )  # 3xN
    except ValueError:
        print(dvs_calibration["undist_pix_calibrated"][idx_to_mat, :].T[0].shape)
        print(dvs_calibration["undist_pix_calibrated"][idx_to_mat, :].T[1].shape)
        print(one_vec[:, 0].shape)
        print("Event  # {}".format(iEv))
        break

    # Get map point corresponding to current event
    # print(Rot)
    # print(rot0)
    # print(bearing_vec)
    # exit()
    rotated_vec = rot0.T.dot(Rot).dot(bearing_vec)
    pm = coordinate_transforms.project_equirectangular_projection(
        rotated_vec, output_width, output_height
    )

    #  Get map point corresponding to previous event at same pixel
    rotated_vec_prev = np.zeros(rotated_vec.shape)
    for ii in range(num_events_batch):
        Rot_prev = event_map[y_events_batch.iloc[ii]][x_events_batch.iloc[ii]][
            "rotation"
        ].copy()
        rotated_vec_prev[:, ii] = rot0.T.dot(Rot_prev).dot(bearing_vec[:, ii])
        # Update last rotation and time of event(SAE)
        event_map[y_events_batch.iloc[ii]][x_events_batch.iloc[ii]][
            "sae"
        ] = t_events_batch.iloc[ii]
        event_map[y_events_batch.iloc[ii]][x_events_batch.iloc[ii]]["rotation"] = Rot

    pm_prev = coordinate_transforms.project_equirectangular_projection(
        rotated_vec_prev, output_width, output_height
    )

    if (t_prev_batch[-1] < 0) or (t_prev_batch[-1] < poses["t"][0]):
        continue  # initialization phase. Fill in event_map

    #  Discard nan values
    mask_uninitialized = np.isnan(pm_prev[0, :]) | np.isnan(pm_prev[1, :])
    num_uninitialized = sum(mask_uninitialized)

    if True:  # (num_uninitialized > 0):
        # % Delete uninitialized events
        # print('Deleting {} points'.format(str(num_uninitialized)))
        t_events_batch = np.array(t_events_batch.iloc[~mask_uninitialized].tolist())
        t_prev_batch = np.array(t_prev_batch[~mask_uninitialized].tolist())

        pol_events_batch = np.array(pol_events_batch[~mask_uninitialized].tolist())
        pm_ = pm.copy()
        pm_prev_ = pm_prev.copy()
        pm = []
        pm_prev = []
        for row in pm_:
            pm.append(row[~mask_uninitialized].tolist())
        for row in pm_prev_:
            pm_prev.append(row[~mask_uninitialized].tolist())
        pm = np.array(pm)
        pm_prev = np.array(pm_prev)

    # Get time since previous event at same pixel
    tc = t_events_batch - t_prev_batch
    event_rate = 1.0 / (tc + 1e-12)  #% measurement or observation(z)

    # Get velocity vector
    vel = (
        pm - pm_prev
    ) * event_rate  # TODO: Check for a point after the first to evaluate

    ## Extended Kalman Filter (EKF) for the intensity gradient map.
    # Get gradient and covariance at current map points pm
    ir = np.floor(pm[1, :]).astype(int)  # row is y coordinate
    ic = np.floor(pm[0, :]).astype(int)  # col is x coordinate

    gm = np.array([[grad_map["x"][i][j], grad_map["y"][i][j]] for i, j in zip(ir, ic)])
    Pg = np.array(
        [
            [
                grad_map_covar["xx"][i][j],
                grad_map_covar["xy"][i][j],
                grad_map_covar["yx"][i][j],
                grad_map_covar["yy"][i][j],
            ]
            for i, j in zip(ir, ic)
        ]
    )

    # EKF update
    if measurement_criterion == "contrast":
        # Use contrast as measurement function
        dhdg = (
            vel.T * np.array([tc * pol_events_batch, tc * pol_events_batch]).T
        )  # derivative of measurement function

        nu_innovation = dvs_parameters["contrast_threshold"] - np.sum(dhdg * gm, axis=1)
    else:
        # Use the event rate as measurement function
        dhdg = (
            vel.T
            / np.array(
                [
                    dvs_parameters["contrast_threshold"] * pol_events_batch,
                    dvs_parameters["contrast_threshold"] * pol_events_batch,
                ]
            ).T
        )  # deriv. of measurement function
        nu_innovation = event_rate - np.sum(dhdg * gm, axis=1)

    Pg_dhdg = np.array(
        [
            Pg[:, 0] * dhdg[:, 0] + Pg[:, 1] * dhdg[:, 1],
            Pg[:, 2] * dhdg[:, 0] + Pg[:, 3] * dhdg[:, 1],
        ]
    ).T

    S_covar_innovation = dhdg[:, 0] * Pg_dhdg[:, 0] + dhdg[:, 1] * Pg_dhdg[:, 1] + var_R
    Kalman_gain = Pg_dhdg / np.array([S_covar_innovation, S_covar_innovation]).T
    # Update gradient and covariance
    gm = gm + Kalman_gain * np.array([nu_innovation, nu_innovation]).T
    Pg = (
        Pg
        - np.array(
            [
                Pg_dhdg[:, 0] * Kalman_gain[:, 0],
                Pg_dhdg[:, 0] * Kalman_gain[:, 1],
                Pg_dhdg[:, 1] * Kalman_gain[:, 0],
                Pg_dhdg[:, 1] * Kalman_gain[:, 1],
            ]
        ).T
    )

    k = 0
    for i, j in zip(ir, ic):
        grad_map["x"][i][j] = gm[k, 0]
        grad_map["y"][i][j] = gm[k, 1]
        grad_map_covar["xx"][i][j] = Pg[k, 0]
        grad_map_covar["xy"][i][j] = Pg[k, 1]
        grad_map_covar["yx"][i][j] = Pg[k, 2]
        grad_map_covar["yy"][i][j] = Pg[k, 3]
        k += 1

    iBatch = iBatch + 1

    if plot_events_animation or plot_events_pm_animation:
        print("Update display: Event # {}".format(iEv))
        idx_pos = pol_events_batch > 0
        idx_neg = pol_events_batch < 0

        trace_map = grad_map_covar["xx"] + grad_map_covar["yy"]
        grad_map_clip = {}
        grad_map_clip["x"] = grad_map["x"]
        grad_map_clip["y"] = grad_map["y"]
        mask = trace_map > 0.05  # % reconstruct only gradients with small covariance
        grad_map_clip["x"][mask] = 0
        grad_map_clip["x"][mask] = 0

        rec_image = integration_methods.frankotchellappa(
            grad_map_clip["x"], grad_map_clip["y"]
        )
        rec_image = rec_image - np.mean(rec_image)

        if first_plot:
            first_plot = False

            if plot_events_animation:
                # Plot events on sensor image
                fig_events_raw = plt.figure(0)
                ax_events_raw = fig_events_raw.add_subplot(111)
                events_p, = ax_events_raw.plot(
                    events_batch_pos["x"], events_batch_pos["y"], ",b"
                )
                events_n, = ax_events_raw.plot(
                    events_batch_neg["x"], events_batch_neg["y"], ",r"
                )
                plt.xlim([0, dvs_parameters["sensor_width"]])
                plt.ylim([0, dvs_parameters["sensor_height"]])
                plt.title("Events on Sensor")

                plt.ion()
                plt.show()

            if plot_events_pm_animation:
                # Plot points on panoramic image, colored according to polarity
                fig_events = plt.figure(1)
                ax_events = fig_events.add_subplot(111)
                h_map_pts_p, = ax_events.plot(pm[0, idx_pos], pm[1, idx_pos], ",b")
                h_map_pts_n, = ax_events.plot(pm[0, idx_neg], pm[1, idx_neg], ",r")
                plt.xlim([0, output_width])
                plt.ylim([0, output_height])
                plt.title("Map points from events")

                plt.ion()
                plt.show()

            # Display reconstructed image
            # fig_reconstructed = plt.figure(2)
            # ax_reconstructed = fig_reconstructed.add_subplot(111)
            # maximum = np.max(np.abs(rec_image))
            # minimum = np.min(np.abs(rec_image))
            # h_img = plt.imshow(rec_image / maximum, cmap=plt.cm.binary, vmin=-1, vmax=1)

            # Display one of the gradient images
            # fig_gradient = plt.figure(3)
            # ax_gradient = fig_gradient.add_subplot(111)
            # h_gx = plt.imshow(grad_map['x'] / np.std(grad_map['x']), cmap=plt.cm.binary, vmin=-5, vmax=5)
            # # plt.xlim([0, output_width])
            # # plt.ylim([0, output_height])
            # plt.xlim([0, output_width])
            # plt.ylim([0, output_height])
            # plt.ion()
            # plt.show()

            # print("MAX: ", np.max([np.abs(np.max(rec_image[0])), np.abs(np.min(rec_image[0]))]))
            # print(np.max(np.abs(rec_image[0])))

        else:
            if plot_events_animation:
                events_p.set_data(events_batch_pos["x"], events_batch_pos["y"])
                events_n.set_data(events_batch_neg["x"], events_batch_neg["y"])
                plt.savefig(
                    output_dir
                    + "/animation/events_raw/"
                    + "fig_events_{}.png".format(str(iEv).zfill(10))
                )

            if plot_events_pm_animation:
                h_map_pts_p.set_data(pm[0, idx_pos], pm[1, idx_pos])
                h_map_pts_n.set_data(pm[0, idx_neg], pm[1, idx_neg])
                plt.savefig(
                    output_dir
                    + "/animation/events_pm/"
                    + "fig_events_{}.png".format(str(iEv).zfill(10))
                )

            # maximum = np.max(np.abs(rec_image))
            # minimum = np.min(np.abs(rec_image))
            # h_img.set_data(rec_image / maximum)

            # h_gx.set_data(grad_map['x'] / np.std(grad_map['x']))
            # ax_events.relim()

endtime = time.time()
print("Done")
print("Elapsed time: {} seconds".format(endtime - time_0))

# Display in separate figure
print("Total summed Events # {}".format(iEv))
idx_pos = pol_events_batch > 0
idx_neg = pol_events_batch < 0

trace_map = grad_map_covar["xx"] + grad_map_covar["yy"]

import pickle

pickle_out = open("grad_map.pickle", "wb")
pickle.dump(grad_map, pickle_out)
pickle_out.close()

pickle_out = open("trace_map.pickle", "wb")
pickle.dump(trace_map, pickle_out)
pickle_out.close()

grad_map_clip = {}
grad_map_clip["x"] = grad_map["x"]
grad_map_clip["y"] = grad_map["y"]
mask = trace_map > 0.05  # % reconstruct only gradients with small covariance

grad_map_clip["x"][mask] = 0
grad_map_clip["y"][mask] = 0

# grad_map_clip.to_csv('grad_map_clip.csv')
#
# grad_map_clip['x'] = grad_map_clip['x'].loc[1750:2000]
# grad_map_clip['y'] = grad_map_clip['y'].loc[400:700]

rec_image = integration_methods.frankotchellappa(grad_map_clip["x"], grad_map_clip["y"])
rec_image = rec_image - np.mean(rec_image)

rec_image_normalized = rec_image / np.max(np.abs(rec_image))
fig_normalized = plt.figure(1)
plt.imshow(rec_image_normalized, cmap=plt.cm.binary)
plt.title("Reconstructed image (log)")
plt.savefig(os.path.join(images_dir, "reconstructed_log.pdf"), dpi=350)
plt.show()
#
rec_image_exp = np.exp(0.001 + rec_image)
fig_normalized_linear = plt.figure(2)
plt.imshow(rec_image_exp, cmap=plt.cm.binary)
plt.title("Reconstructed image (linear)")
plt.savefig(os.path.join(images_dir, "reconstructed_linear.pdf"), dpi=350)
plt.show()

fig_gradientx = plt.figure(3)
h_gx = plt.imshow(
    grad_map["x"] / np.std(grad_map["x"]), cmap=plt.cm.binary, vmin=-5, vmax=5
)
plt.title("Gradient in X")
plt.savefig(os.path.join(images_dir, "gradient_x.pdf"), dpi=350)
plt.show()

fig_gradienty = plt.figure(4)
h_gx = plt.imshow(
    grad_map["y"] / np.std(grad_map["y"]), cmap=plt.cm.binary, vmin=-5, vmax=5
)
plt.title("Gradient in Y")
plt.savefig(os.path.join(images_dir, "gradient_y.pdf"), dpi=350)
plt.show()

fig_tracemap = plt.figure(5)
h_gx = plt.imshow(trace_map / np.max(trace_map), cmap=plt.cm.binary, vmin=0, vmax=1)
plt.title("Trace of Covariance")
plt.savefig(os.path.join(images_dir, "trace.pdf"), dpi=350)
plt.show()

# g_ang = -1*np.arctan2(grad_map['y'], grad_map['x'])
# g_grad = np.sqrt(np.power(grad_map['x'], 2) + np.power(grad_map['y'], 2))
# g_grad_unit = g_grad/1.
# g_grad_unit[g_grad_unit > 1.0] = 1.0
# g_ang_unit = g_ang/360. + 0.5

np.save("intensity_map.npy", rec_image)


import topbnv_tools as tbt

import numpy as np
import matplotlib.pylab as plt

import ROOT
import sys

import lichen.lichen as lch

import pickle

################################################################################
def weights_for_histos(weights,vals):
    hweights = []
    for w,d in zip(weights,vals):
        print('weighting: ',w,len(d))
        hweights.append(w*np.ones(len(d)))
    return hweights

################################################################################
def main():

    lumi_file_name = 'lumi_info.pkl'
    lumi_info = pickle.load( open( lumi_file_name, "rb" ))

    filenames = sys.argv[1:]

    MCfiles = []
    DATAfiles = []
    for filename in filenames:
        filename2 = filename.split('/')[-1]
        if filename2.split('DATASET')[0] == 'MC_':
            #print('**********************************')
            MCfiles.append(filename)
        else:
            DATAfiles.append(filename)
    '''
    for filename in MCfiles:
        print('MC: ', filename)
    for filename in DATAfiles:
        print('DATA: ', filename)
    '''
    mcdata = []
    #print("Will open files:")
    mcInfo = tbt.csvtodict("MCinfo.csv")
    weights = []
    mcEvents = []
    crosssection = []

    data,tot_lumi = tbt.chain_pickle_files(DATAfiles,lumi_info)
    print("tot_lumi: ",tot_lumi)

    for f in MCfiles:
        #print('f', f)
        mcdataTMP, tot_lumiMC = tbt.chain_pickle_files(f)
        mcdata.append(mcdataTMP)
        fnew = f.split('DATASET_crab_')[1].split('_NFILES')[0]
        nfiles = int(f.split('NFILES_')[1].split('_')[1].split('.')[0]) - int(f.split('NFILES_')[1].split('_')[0])
        crosssection = 1000*(float(mcInfo[fnew]['cross_section'])) # Convert from pb to fb
        mcEvents = (float(mcInfo[fnew]['total_events']))
        print("crosssection: ",crosssection)
        print("MCEvents/crosssection: ",mcEvents/crosssection)
        print("MCEvents: ",mcEvents)
        print("tot_lumi: ",tot_lumi)
        print("nfiles: ",nfiles)
        print("nfiles_tot: ",mcInfo[fnew]['nfiles'])
        weights.append((crosssection * tot_lumi) / (nfiles*mcEvents/float(mcInfo[fnew]['nfiles'])))

    topmassDATA = data['topmass']
    wmassDATA  = data['wmass']
    csvsDATA  = data['csvs']
    anglesDATA = data['angles']
    dRsDATA  = data['dRs']
    #njets = data['njets']
    njetsDATA  = data['njets']
    leadmuptDATA  = data['leadmupt']
    leadmuetaDATA  = data['leadmueta']

    topmassMC = []
    wmassMC  = []
    csvsMC  = []
    anglesMC = []
    dRsMC  = []
    #njets = []
    njetsMC  = []
    leadmuptMC  = []
    leadmuetaMC  = []

    for mc in mcdata:
        topmassMC.append(mc['topmass'])
        wmassMC.append(mc['wmass'])
        csvsMC.append(mc['csvs'])
        anglesMC.append(mc['angles'])
        dRsMC.append(mc['dRs'])
        #njets.append(mc['njets'])
        njetsMC.append(mc['njets'])
        leadmuptMC.append(mc['leadmupt'])
        leadmuetaMC.append(mc['leadmueta'])

    print('Weights', weights)

    bins = 100

    hweights = []
    for w,d in zip(weights,[topmassMC[0],topmassMC[1]]):
        print('weighting: ',w,len(d))
        hweights.append(w*np.ones(len(d)))

    plt.figure()
    vals = [topmassMC[0],topmassMC[1]]
    hw = weights_for_histos(weights,vals)
    plt.hist(vals, bins, range=(0,1600), weights=hw,stacked=True)
    lch.hist_err(topmassDATA, bins, range=(0,1600))

    plt.figure()
    vals = [leadmuptMC[0],leadmuptMC[1]]
    hw = weights_for_histos(weights,vals)
    plt.hist(vals, bins, range=(0,200), weights=hw,stacked=True)
    lch.hist_err(leadmuptDATA, bins, range=(0,200))

    plt.figure()
    vals = [leadmuetaMC[0],leadmuetaMC[1]]
    hw = weights_for_histos(weights,vals)
    plt.hist(vals, bins, range=(-3,3), weights=hw,stacked=True)
    lch.hist_err(leadmuetaDATA, bins, range=(-3,3))

    print(mcdata[0].keys())
    plt.figure()
    plt.subplot(2,2,1)
    plt.hist(mcdata[0]["trig_HLT_IsoMu24_accept"])
    plt.subplot(2,2,2)
    plt.hist(mcdata[0]["trig_HLT_IsoTkMu24_accept"])
    plt.subplot(2,2,3)
    plt.hist(mcdata[0]["trig_HLT_IsoMu22_eta2p1_accept"])
    plt.subplot(2,2,4)
    plt.hist(mcdata[0]["trig_HLT_IsoTkMu22_eta2p1_accept"])

    '''
    #lch.hist_err(topmass,bins=100,range=(0,600),color='k')
    plt.xlabel('Top Mass (GeV)')

    plt.subplot(3,3,2)
    #lch.hist_err(wmass,bins=100,range=(0,300),color='k')
    plt.xlabel('W Mass (GeV)')

    plt.subplot(3,3,3)
    #lch.hist_err(csvs,bins=110,range=(0,1.1),color='k')
    plt.xlabel('Isolation Variable')

    plt.subplot(3,3,4)
    #lch.hist_err(angles,bins=100,range=(0, 3.2),color='k')
    plt.xlabel('Angles')

    plt.subplot(3,3,5)
    #plt.plot(wmass,angles,'.',markersize=0.5,alpha=0.2)
    #lch.hist_2D(wmass,angles,xbins=100,ybins=100,xrange=(0,300),yrange=(0,3.14))
    plt.xlim(50,150)
    plt.ylim(0, 3.2)
    plt.xlabel('W Mass')
    plt.ylabel('Angles')

    plt.subplot(3,3,6)
    #lch.hist_err(dRs,bins=100,range=(0, 3.2),color='k')
    plt.xlabel('dRs')

    plt.subplot(3,3,7)
    #lch.hist_2D(dRs,angles,xbins=100,ybins=100,xrange=(0,6.28),yrange=(0,3.14))
    plt.xlabel('dRs')
    plt.ylabel('Angles')
    '''

    ################################################################################
    # Cut on the wmass
    #index = wmass>70.0
    #index *= wmass<95.0
    '''
    plt.figure()
    plt.title('W Mass Cuts')
    plt.subplot(3,3,1)
    #lch.hist_err(topmass[index],bins=100,range=(0,600),color='k')
    plt.xlabel('Top Mass (GeV)')

    plt.subplot(3,3,2)
    #lch.hist_err(wmass[index],bins=100,range=(0,300),color='k')
    plt.xlabel('W Mass (GeV)')

    plt.subplot(3,3,4)
    #lch.hist_err(angles[index],bins=100,range=(0,3.2),color='k')
    plt.xlabel('Angles')

    plt.subplot(3,3,5)
    #plt.plot(wmass[index],angles[index],'.',markersize=0.5,alpha=0.2)
    #lch.hist_2D(wmass[index],angles[index],xbins=100,ybins=100,xrange=(0,300),yrange=(0,3.14))
    plt.xlim(50,150)
    plt.ylim(0, 3.2)
    plt.xlabel('W Mass')
    plt.ylabel('Angles')

    plt.subplot(3,3,6)
    #lch.hist_err(dRs[index],bins=100,range=(0, 3.2),color='k')
    plt.xlabel('dRs')

    plt.subplot(3,3,7)
    #lch.hist_2D(dRs[index],angles[index],xbins=100,ybins=100,xrange=(0,6.28),yrange=(0,3.14))
    plt.xlabel('dRs')
    plt.ylabel('Angles')

    # For talk
    plt.figure()
    #lch.hist_err(topmass[index],bins=100,range=(0,600),color='k')
    plt.hist(topmass[index],bins=100,range=(0,600),color='grey',alpha=0.2)
    plt.xlabel(r'Top candidate with W-mass cut (GeV/c$^2$)',fontsize=14)
    plt.tight_layout()
    plt.savefig('top.png')

    plt.figure()
    #lch.hist_err(wmass,bins=100,range=(0,300),color='k')
    plt.hist(wmass,bins=100,range=(0,300),color='grey',alpha=0.2)
    plt.xlabel(r'W candidate (GeV/c$^2$)',fontsize=14)
    plt.tight_layout()
    plt.savefig('W.png')

    plt.figure()
    #lch.hist_err(csvs,bins=110,range=(0,1.1),color='k')
    plt.hist(csvs,bins=100,range=(0,1.1),color='grey',alpha=0.2)
    plt.xlabel(r'CSVv2 variable',fontsize=14)
    plt.tight_layout()
    plt.savefig('csvv2.png')

    '''
    plt.show()

    return data

################################################################################
if __name__=="__main__":
    main()