# Import modules
import enum
import itertools
import logging
import numpy as np
import regions
from rig import machine

# Import classes
from collections import defaultdict
from pacman.model.graphs.machine.impl.machine_vertex \
    import MachineVertex
from pacman.model.resources.resource_container import ResourceContainer
from pacman.model.resources.sdram_resource import SDRAMResource
from spinn_front_end_common.abstract_models.abstract_has_associated_binary \
    import AbstractHasAssociatedBinary
from spinn_front_end_common.abstract_models.abstract_provides_n_keys_for_partition \
    import AbstractProvidesNKeysForPartition
from spinn_front_end_common.abstract_models.abstract_starts_synchronized \
    import AbstractStartsSynchronized
from utils import Args

# Import functions
from six import iteritems
from utils import (calc_bitfield_words, calc_slice_bitfield_words,
                   get_model_executable_filename, load_regions, split_slice)

logger = logging.getLogger("pynn_spinnaker")

# ----------------------------------------------------------------------------
# Regions
# ----------------------------------------------------------------------------
class Regions(enum.IntEnum):
    """Region names, corresponding to those defined in `ensemble.h`"""
    system = 0
    neuron = 1
    synapse = 2
    input_buffer = 3
    back_prop_output = 4
    flush = 5
    intrinsic_plasticity = 6
    spike_recording = 7
    analogue_recording_0 = 8
    analogue_recording_1 = 9
    analogue_recording_2 = 10
    analogue_recording_3 = 11
    analogue_recording_start = analogue_recording_0
    analogue_recording_end = analogue_recording_3 + 1
    profiler = analogue_recording_end
    statistics = analogue_recording_end + 1

# ----------------------------------------------------------------------------
# Vertex
# ----------------------------------------------------------------------------
class Vertex(MachineVertex, AbstractHasAssociatedBinary,
             AbstractProvidesNKeysForPartition, AbstractStartsSynchronized):
    def __init__(self, neuron_slice, vert_index, sdram, app_name):
        self.neuron_slice = neuron_slice

        self.input_verts = []
        self.back_prop_out_buffers = None
        self.region_memory = None
        self.app_name = app_name
        self.vert_index = vert_index

        # Superclass
        # **NOTE** as vertex partitioning is already done,
        # only SDRAM is required for subsequent placing decisions
        MachineVertex.__init__(
            self, label="<neuron slice:%s>" % str(self.neuron_slice),
            resources_required=ResourceContainer(sdram=SDRAMResource(sdram)))

    # ------------------------------------------------------------------------
    # AbstractProvidesNKeysForPartition methods
    # ------------------------------------------------------------------------
    def get_n_keys_for_partition(self, partition, graph_mapper):
        # Each neuron requires two keys
        # **HACK** allocate fixed keyspace for each neuron cluster:
        # means a fixed bit can always be used to signify flushing
        return 2 * 1024

    # ------------------------------------------------------------------------
    # AbstractHasAssociatedBinary methods
    # ------------------------------------------------------------------------
    def get_binary_file_name(self):
        return self.app_name

    # ------------------------------------------------------------------------
    # Public methods
    # ------------------------------------------------------------------------
    def get_back_prop_in_buffer(self, post_slice):
        # Check the slices involved overlap and that this
        # neuron vertex actually has back propagation buffers
        assert post_slice.overlaps(self.neuron_slice)
        assert self.back_prop_out_buffers is not None

        # Calculate start and end bit in neuron id-space
        neuron_start_bit = max(post_slice.start, self.neuron_slice.start)
        neuron_end_bit = min(post_slice.stop, self.neuron_slice.stop)
        logger.debug("\t\t\tNeuron start bit:%u, Neuron end bit:%u",
                     neuron_start_bit, neuron_end_bit)

        # Calculate where in the buffer post_slice starts
        buffer_start_bit = neuron_start_bit - self.neuron_slice.start
        assert buffer_start_bit >= 0

        # Seperate where the buffer starts in words and bits
        buffer_start_word = buffer_start_bit // 32
        buffer_start_bit -= (buffer_start_word * 32)
        buffer_end_bit = (neuron_end_bit - neuron_start_bit) + buffer_start_bit
        buffer_num_words = calc_bitfield_words(buffer_end_bit)
        logger.debug("\t\t\tBuffer start word:%u, Buffer start bit:%u, Buffer end bit:%u, Buffer num words:%u",
                     buffer_start_word, buffer_start_word,
                     buffer_end_bit, buffer_num_words)

        # Return offset pointers into out buffers
        return (
            [b + (buffer_start_word * 4) for b in self.back_prop_out_buffers],
            buffer_num_words, buffer_start_bit, buffer_end_bit)

    def get_spike_tx_key(self, routing_info):
        # Routing key is used for spike transmission
        return self.get_routing_key(routing_info)

    def get_flush_tx_key(self, routing_info):
        # **YUCK** flush events are transmitted with fixed bit set
        return self.get_routing_key(routing_info) | (1 << 10)

    def get_routing_key(self, routing_info):
        vert_routing = routing_info.get_routing_info_from_pre_vertex(self, 0)
        return 0xFFFFFFFF if vert_routing is None else vert_routing.first_key

    def get_routing_mask(self, routing_info):
        vert_routing = routing_info.get_routing_info_from_pre_vertex(self, 0)
        return vert_routing.first_mask

# -----------------------------------------------------------------------------
# NeuralCluster
# -----------------------------------------------------------------------------
class NeuralCluster(object):
    # Tag names, corresponding to those defined in neuron_processor.h
    profiler_tag_names = {
        0:  "Synapse shape",
        1:  "Update neurons",
        2:  "Apply buffer",
    }

    # Names of statistics
    statistic_names = (
        "task_queue_full",
        "timer_event_overflows",
    )

    def __init__(self, pop_id, cell_type, parameters, initial_values,
                 sim_timestep_ms, timer_period_us, sim_ticks,
                 record_sample_interval, indices_to_record, config,
                 frontend, post_synaptic_width, requires_back_prop, pop_size):
        # Create standard regions
        self.regions = {}
        self.regions[Regions.system] = regions.System(
            timer_period_us, sim_ticks)
        self.regions[Regions.neuron] = cell_type._neuron_region_class(
            cell_type, parameters, initial_values, sim_timestep_ms, pop_size)
        self.regions[Regions.back_prop_output] = regions.SDRAMBackPropOutput(
            requires_back_prop)
        self.regions[Regions.flush] = regions.Flush(config.flush_time,
                                                    sim_timestep_ms)
        self.regions[Regions.spike_recording] = regions.SpikeRecording(
            indices_to_record, sim_timestep_ms, sim_ticks)
        self.regions[Regions.statistics] = regions.Statistics(len(self.statistic_names))

        # If cell type has any receptors i.e. any need for synaptic input
        if len(cell_type.receptor_types) > 0:
            # Add a synapse region and an input buffer
            self.regions[Regions.synapse] = regions.ParameterSpace(
                cell_type._synapse_mutable_param_map,
                cell_type._synapse_immutable_param_map,
                parameters, initial_values, pop_size,
                sim_timestep_ms=sim_timestep_ms)

            self.regions[Regions.input_buffer] = regions.InputBuffer()

         # If cell type has an intrinsic plasticity parameter map
        if hasattr(cell_type, "intrinsic_plasticity_param_map"):
            self.regions[Regions.intrinsic_plasticity] =\
                regions.HomogeneousParameterSpace(
                    cell_type._intrinsic_plasticity_param_map,
                    parameters,
                    sim_timestep_ms)

        # Assert that there are sufficient analogue
        # recording regions for this celltype's needs
        num_analogue_rec_regions = Regions.analogue_recording_end -\
            Regions.analogue_recording_start
        assert num_analogue_rec_regions >= (len(cell_type.recordable) - 1)

        # Loop through cell's non-spike recordables
        # and create analogue recording regions
        # **HACK** this assumes the first entry is spike
        for i, v in enumerate(cell_type.recordable[1:]):
            self.regions[Regions(Regions.analogue_recording_start + i)] =\
                regions.AnalogueRecording(indices_to_record, v,
                                          record_sample_interval,
                                          sim_timestep_ms, sim_ticks)

        # Add profiler region if required
        if config.num_profile_samples is not None:
            self.regions[Regions.profiler] =\
                regions.Profiler(config.num_profile_samples)

        # Split population slice
        neuron_slices = split_slice(pop_size, post_synaptic_width)

        # Get neuron executable name
        neuron_app = get_model_executable_filename(
            "neuron_", cell_type, config.num_profile_samples is not None)

        logger.debug("\t\tNeuron application:%s", neuron_app)
        logger.debug("\t\t%u neuron vertices", len(neuron_slices))

        # Build neuron vertices for each slice
        self.verts = []
        for vert_id, neuron_slice in enumerate(neuron_slices):
            # Create vertex
            vert = Vertex(neuron_slice, vert_id,
                          self._estimate_sdram(neuron_slice), neuron_app)

            # Add to frontend and verts list
            frontend.add_machine_vertex(vert)
            self.verts.append(vert)

    # --------------------------------------------------------------------------
    # Public methods
    # --------------------------------------------------------------------------
    def allocate_out_buffers(self, placements, transceiver, app_id):
        # Loop through vertices
        for v in self.verts:
            # Get placement
            placement = placements.get_placement_of_vertex(v)

            logger.debug("\t\tVertex %s (%u, %u, %u)",
                         v, placement.x, placement.y, placement.p)

            # **TODO** zero memory - waiting on https://github.com/SpiNNakerManchester/SpiNNMan/issues/59
            # If back propagation is enabled, allocate two back
            # propagation out buffers for this neuron vertex
            if self.regions[Regions.back_prop_output].enabled:
                back_prop_buffer_bytes =\
                    calc_slice_bitfield_words(v.neuron_slice) * 4
                v.back_prop_out_buffers = [
                    transceiver.malloc_sdram(placement.x, placement.y,
                                             back_prop_buffer_bytes,
                                             app_id=app_id)
                    for _ in range(2)]

    def load(self, routing_info, placements, transceiver, app_id):
        # Loop through vertices
        for v in self.verts:
            # Get placement and allocation
            placement = placements.get_placement_of_vertex(v)

            # Use routing info to get spike and flush TX keys
            spike_tx_key = v.get_spike_tx_key(routing_info)
            flush_tx_key = v.get_flush_tx_key(routing_info)

            logger.debug("\t\t\tVertex %s (%u, %u, %u): Spike key:%08x, Flush key:%08x",
                            v, placement.x, placement.y, placement.p,
                            spike_tx_key, flush_tx_key)

            # Get the input buffers from each synapse vertex
            in_buffers = [
                s.get_in_buffer(v.neuron_slice)
                for s in v.input_verts]

            # Get regiona arguments
            region_arguments = self._get_region_arguments(
                spike_tx_key, flush_tx_key, v.neuron_slice,
                in_buffers, v.back_prop_out_buffers)

            # Load regions
            v.region_memory = load_regions(self.regions, region_arguments,
                                           placement, transceiver, app_id)

    def read_recorded_spikes(self):
        # Loop through all neuron vertices and read spike times into dictionary
        spike_times = {}
        region = self.regions[Regions.spike_recording]
        for v in self.verts:
            region_mem = v.region_memory[Regions.spike_recording]
            spike_times.update(region.read_spike_times(v.neuron_slice,
                                                       region_mem))
        return spike_times

    def read_recorded_signal(self, channel):
        # Get index of channelread_profile
        region_index = Regions(Regions.analogue_recording_start + channel)
        region = self.regions[region_index]

        # Loop through all neuron vertices and read signal
        signal = {}
        for v in self.verts:
            region_mem = v.region_memory[region_index]
            signal.update(region.read_signal(v.neuron_slice, region_mem))

        return signal

    def read_profile(self):
        # Get the profile recording region and
        region = self.regions[Regions.profiler]

        # Return profile data for each vertex that makes up population
        return [(v.neuron_slice.python_slice,
                 region.read_profile(v.region_memory[Regions.profiler],
                                     self.profiler_tag_names))
                for v in self.verts]

    def read_statistics(self):
        # Get the statistics recording region
        region = self.regions[Regions.statistics]

        # Read stats from all vertices
        return region.read_stats(
            [v.region_memory[Regions.statistics] for v in self.verts],
            self.statistic_names)

    # --------------------------------------------------------------------------
    # Private methods
    # --------------------------------------------------------------------------
    def _estimate_sdram(self, vertex_slice):
        # Begin with size of spike recording region
        sdram = self.regions[Regions.spike_recording].sizeof(vertex_slice);

        # Add on size of neuron region
        sdram += self.regions[Regions.neuron].sizeof(vertex_slice)

        # If profiler region exists, add its size
        if Regions.profiler in self.regions:
            sdram += self.regions[Regions.profiler].sizeof()

        # Loop through possible analogue recording regions
        for t in range(Regions.analogue_recording_start,
                       Regions.analogue_recording_end):
            # If region exists, add its size to total
            if Regions(t) in self.regions:
                sdram += self.regions[Regions(t)].sizeof(vertex_slice)

        return sdram

    def _get_region_arguments(self, spike_tx_key, flush_tx_key, vertex_slice,
                              in_buffers, back_prop_out_buffers):
        region_arguments = defaultdict(Args)

        analogue_recording_regions = range(Regions.analogue_recording_start,
                                           Regions.analogue_recording_end)
        # Add vertex slice to regions that require it
        for r in itertools.chain((Regions.neuron,
                                  Regions.synapse,
                                  Regions.spike_recording),
                                 analogue_recording_regions):
            region_arguments[r] = Args(vertex_slice)

        # Add kwargs for regions that require them
        region_arguments[Regions.system].kwargs["application_words"] =\
            [spike_tx_key, flush_tx_key, len(vertex_slice)]
        region_arguments[Regions.input_buffer].kwargs["in_buffers"] =\
            in_buffers
        region_arguments[Regions.back_prop_output].kwargs["out_buffers"] =\
            back_prop_out_buffers
        return region_arguments


from .models import *
from django.views.generic.base import TemplateView

class PortfolioView(TemplateView):
    title = "What I have done!"
    template_name = 'portfolio/portfolio.djhtml'
    link_slug = 'portfolio'
    models = {
              'portfolio': Portfolio
              }

    def get_object(self):
        object = {}
        for k, v in self.models.iteritems():
            object[k] = v.objects.all().order_by('-id')
        return object

    def get_context_data(self, **kwargs):
        context = super(PortfolioView, self).get_context_data(**kwargs)
        object = self.get_object()
        for k, v in object.iteritems():
            context[k] = v
        context['title'] = self.title
        context['link_slug'] = self.link_slug
        return context


import time
from .utils import *
import pymongo
from .course_logger import logger

class MoocPipeline:
    data_path = 'H:/course/'
    collection_name = "course"

    outline = {}
    play_list = []
    subtitles = []
    WORK_DIR = ''

    def parse_resource(self, resource, counter):
        post_data = {'callCount': '1', 'scriptSessionId': '${scriptSessionId}190',
                     'httpSessionId': '5531d06316b34b9486a6891710115ebc', 'c0-scriptName': 'CourseBean',
                     'c0-methodName': 'getLessonUnitLearnVo', 'c0-id': '0', 'c0-param0': 'number:' + resource[0],
                     'c0-param1': 'number:' + resource[1], 'c0-param2': 'number:0',
                     'c0-param3': 'number:' + resource[2], 'batchId': str(int(time.time()) * 1000)}
        res = requests.post('https://www.icourse163.org/dwr/call/plaincall/CourseBean.getLessonUnitLearnVo.dwr',
                            params=post_data).text
        file_name = resource[3].strip()
        index_dunhao = file_name.find("、")
        if index_dunhao != -1:
            file_name = "{}{}".format(counter, file_name.split("、")[1])
        else:
            file_name = "{}{}".format(counter, file_name)
        index = file_name.find(".")
        if index != -1:
            file_name = file_name.replace(".", "-")
        resolutions = ['Shd', 'Hd', 'Sd']
        play_info = {}
        for sp in resolutions:
            video_info = re.search(r'(?P<ext>mp4)%sUrl="(?P<url>.*?\.(?P=ext).*?)"' % sp, res)

            if video_info:
                url, ext = video_info.group('url', 'ext')
                ext = '.' + ext
                play_info['name'] = file_name
                play_info['video_url'] = url

                logger.info("正在下载： {} ".format(file_name + ext))
                try:
                    r = requests.get(url)
                    if not os.path.exists(self.data_path + r'\video\\' + file_name + ext):
                        with open(self.data_path + r'\video\\' + file_name + ext, 'wb')as f:
                            f.write(r.content)
                    play_info['video_path'] = self.data_path + "/video/" + file_name + ext
                    logger.info("下载成功！")
                    break
                except Exception as e:
                    logger.erro("下载失败：{}  {}".format(file_name, url), e)

        subtitles = re.findall(r'name="(.+)";.*url="(.*?)"', res)
        for subtitle in subtitles:
            if len(subtitles) == 1:
                sub_name = file_name + '.srt'
            else:
                subtitle_lang = subtitle[0].encode('utf_8').decode('unicode_escape')
                sub_name = file_name + '_' + subtitle_lang + '.srt'

            try:
                r = requests.get(subtitle[1])
                play_info['sub_url'] = subtitle[1]
                if not os.path.exists(self.data_path + r'\video\\' + sub_name):
                    with open(self.data_path + r'\video\\' + sub_name, 'wb')as f:
                        f.write(r.content)
                play_info['sub_path'] = self.data_path + "/video/" + sub_name
                logger.info("下载成功！")
            except Exception as e:
                logger.erro("下载失败：{}  {}".format(sub_name, subtitle[1]), e)
        if play_info:
            self.play_list.append(play_info)

    def save_to_mongo(self, data):
        if self.db[self.collection_name].insert(data):
            logger.info("SAVE SUCCESS", data)
            return True
        return False

    def __init__(self, mongo_uri, mongo_db):
        self.mongo_uri = mongo_uri
        self.mongo_db = mongo_db

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            mongo_uri=crawler.settings.get('MONGO_URI'),
            mongo_db=crawler.settings.get('MONGO_DATABASE', 'items')
        )

    def open_spider(self, spider):
        self.client = pymongo.MongoClient(self.mongo_uri)
        self.db = self.client[self.mongo_db]

    def close_spider(self, spider):
        self.client.close()

    def get_resource(self, term_id):
        counter = Counter()
        CONFIG = {}
        pdf_list = []
        video_list = []
        post_data = {'callCount': '1', 'scriptSessionId': '${scriptSessionId}190', 'c0-scriptName': 'CourseBean',
                     'c0-methodName': 'getMocTermDto', 'c0-id': '0', 'c0-param0': 'number:' + term_id,
                     'c0-param1': 'number:0', 'c0-param2': 'boolean:true', 'batchId': str(int(time.time()) * 1000)}
        res = requests.post('https://www.icourse163.org/dwr/call/plaincall/CourseBean.getMocTermDto.dwr',
                            params=post_data).text.encode('utf_8').decode('unicode_escape')

        chapters = re.findall(r'homeworks=\w+;.+id=(\d+).+name="(.+)";', res)
        for chapter in chapters:
            counter.add(0)
            lessons = re.findall(r'chapterId=' + chapter[0] + r'.+contentType=1.+id=(\d+).+name="(.+)".+test', res)

            lesson_list = []
            for lesson in lessons:
                counter.add(1)
                lesson_list.append(lesson[1])
                self.outline[chapter[1]] = lesson_list

                # 视频
                videos = re.findall(r'contentId=(\d+).+contentType=(1).+id=(\d+).+lessonId=' +
                                    lesson[0] + r'.+name="(.+)"', res)
                for video in videos:
                    counter.add(2)
                    if video:
                        try:
                            os.mkdir(self.data_path + r'\video')
                        except FileExistsError:
                            pass
                        self.parse_resource(video, counter)
                    video_list.append(video[3])

                # 课件
                pdfs = re.findall(r'contentId=(\d+).+contentType=(3).+id=(\d+).+lessonId=' +
                                  lesson[0] + r'.+name="(.+)"', res)
                for pdf in pdfs:
                    counter.add(2)

    def process_item(self, item, spider):

        self.data_path = 'H:/course/'
        self.outline = {}

        if spider.name == 'course':
            self.data_path = os.path.join(self.data_path, item['title'])
            try:
                os.mkdir(self.data_path)
            except FileExistsError:
                pass
            self.get_resource(item["term_id"])
        else:
            return

        item['outline'] = self.outline
        item['play_list'] = self.play_list
        self.save_to_mongo(item)
        return item


from progressbar import update_progress

class ReadSequenceFile:

    # TODO:  Write method to handle nexus fromat sequence files.

    def fasta(self, input_file):

        with open(input_file) as f:
            content = f.readlines()
            f.close()

        # Strip new line ('\n')
        content = [line.strip('\n') for line in content]

        seq_dict = {}
        seq_name = ""
        concat_sequence = ""

        for line in content:
            if ">" in line:
                seq_dict[seq_name] = concat_sequence

                # Reset seq_name to current sequence and rest concat_sequence to empty.
                line = line.translate({ord(c): None for c in '>'})
                seq_name = line
                concat_sequence = ""
            else:
                concat_sequence += line

        # Add last key value pair to dictionary
        seq_dict[seq_name] = concat_sequence
        del seq_dict[""]

        return seq_dict

class WriteSequenceFile:

    # TODO: Write method for writing nexus format sequence files.

    def fasta(self, output_filename, input_seqs):

        i = 0

        # Write fasta format sequence files with proper formating.
        with open(output_filename, 'w') as f:
            while i < len(input_seqs):
                seq_name = str(i)
                seq_name_write = ">"+ seq_name + "\n"
                f.write(seq_name_write)
                seq_data = input_seqs[i] + "\n"
                f.write(seq_data)
                i += 1

            f.close()

        print("Done.")

class ParseSequences:

    # TODO:  Write site_count_protein method for handling protein sequences.

    def remove_bad_seqs_DNA(self, input_seqs):

        # If passed a dict convert to list
        converter = TypeConvert()
        v = converter.dict_to_list(input_seqs)
        initial_count = len(v)

        i = 0

        # If sequences/strings containing any of the following characters, remove them from the sequence list.
        while i < len(v):
            line = v[i]
            bad_chars = set('UWSMKRYBDHVNuwsmkrybdhvn')

            if any((c in bad_chars) for c in line):
                v.pop(i)
                print(line + " removed " + "index: " + str(i))
                i -=1
            else:
                pass

            i += 1

        # Calculate the total number of sequences that have been removed
        final_count = len(v)
        removed_count = initial_count - final_count

        print(str(removed_count) + ' DNA sequences removed')
        print('Sequences in dataset: ', final_count)

        return v

    def remove_bad_seqs_protein(self, input_seqs):

        # TODO:  bad_chars should be dynamic, the set should be defined as a user input string?  Write catch for duplicates since a set can't take duplicates.

        # If passed a dict convert to list
        converter = TypeConvert()
        v = converter.dict_to_list(input_seqs)
        initial_count = len(v)

        i = 0

        # If sequences/strings containing any of the following characters, remove them from the sequence list.
        while i < len(v):
            line = v[i]
            bad_chars = set('*Xx')

            if any((c in bad_chars) for c in line):
                v.pop(i)
                print(line + " removed " + "index: " + str(i))
                i -= 1
            else:
                pass

            i = i + 1

        final_count = len(v)
        removed_count = initial_count - final_count

        print(str(removed_count) + ' protein sequences removed')
        print('Sequences in dataset: ', final_count)

        return v

    def site_count_DNA(self, input_seqs, site_number):

        # If passed a dict convert to list
        converter = TypeConvert()
        v = converter.dict_to_list(input_seqs)
        initial_count = len(v)

        nuc_pos = int(site_number) - 1
        i = 0

        nuc_A = 0
        nuc_G = 0
        nuc_C = 0
        nuc_T = 0

        while i < len(v):
            line = v[i]
            char = line[nuc_pos]

            if "A" == char:
                nuc_A += 1
            elif "G" == char:
                nuc_G += 1
            elif "C" == char:
                nuc_C += 1
            elif "T" == char:
                nuc_T += 1

            i += 1

        input_seq_total = len(v)

        nuc_A_percent = 100 * (nuc_A/input_seq_total)
        nuc_G_percent = 100 * (nuc_G/input_seq_total)
        nuc_C_percent = 100 * (nuc_C/input_seq_total)
        nuc_T_percent = 100 * (nuc_T/input_seq_total)

        print("A: " + str(nuc_A) + " ," + str(nuc_A_percent) + " %")
        print("G: " + str(nuc_G) + " ," + str(nuc_G_percent) + " %")
        print("C: " + str(nuc_C) + " ," + str(nuc_C_percent) + " %")
        print("T: " + str(nuc_T) + " ," + str(nuc_T_percent) + " %")

    def remove_duplicates(self, input_seqs):

        # If passed a dict convert to list
        converter = TypeConvert()
        v = converter.dict_to_list(input_seqs)
        initial_count = len(v)

        # Extract uniques and place them into a new list
        raw_list = v
        uniques_list = []
        [uniques_list.append(x) for x in raw_list if x not in uniques_list]

        # Count the number of duplicate sequences removed
        initial_count = len(raw_list)
        updated_count = len(uniques_list)
        removed_count = initial_count - updated_count

        print (str(removed_count) + " duplicate sequences removed")
        print ("Dataset size: ", len(uniques_list))

        return uniques_list

    def sort_seqs(self, input_seqs):

        # If passed a dict convert to list
        converter = TypeConvert()
        v = converter.dict_to_list(input_seqs)
        initial_count = len(v)

        seq_pos = int(input("Enter nucleotide or amino acid position: "))
        seq_pos -= 1
        seq_val = input("Enter nucleotide or amino acid character to sort: ")

        sorted_list = []
        excluded_list = []

        i = 0

        while i < len(v):
            line = v[i]

            if line[seq_pos] == seq_val:
                sorted_list.append(line)
            else:
                excluded_list.append(line)
            i += 1
            update_progress(i/len(v))

        print(str(len(sorted_list)) + " sequences in new dataset containing \'" + seq_val + "\'" + " at position " + str(seq_pos + 1) + ".")

        return sorted_list, excluded_list

class TypeConvert:

    def dict_to_list(self, input_data):

         # Checks if input data is a list or dict.  If a dict, convert to a list.  If a list, pass the data on.
        if isinstance(input_data, dict):
            v = list(input_data.values())
        else:
            v = input_data

        return v


# BSD 3-Clause License; see https://github.com/scikit-hep/awkward-1.0/blob/main/LICENSE

import awkward as ak

def is_tuple(array):
    """
    Args:
        array (#ak.Array, #ak.Record, #ak.layout.Content, #ak.layout.Record, #ak.ArrayBuilder, #ak.layout.ArrayBuilder):
            Array or record to check.

    If `array` is a record, this returns True if the record is a tuple.
    If `array` is an array, this returns True if the outermost record is a tuple.
    """
    with ak._v2._util.OperationErrorContext(
        "ak._v2.is_tuple",
        dict(array=array),
    ):
        return _impl(array)

def _impl(array):
    layout = ak._v2.to_layout(array, allow_record=True)

    return layout.is_tuple


# -*- coding: utf-8 -*-
import argparse
import tensorflow as tf

if __name__ == '__main__':
    argparser = argparse.ArgumentParser(description=__doc__)
    argparser.add_argument(
        '-ver', '--version',
        type=int,
        default='None',
        help='File name for the pbx file')
    argparser.add_argument(
        '-sp', '--savepath',
        default='None',
        help='Path directory of checkpoint file')
    argparser.add_argument(
        '-mn', '--modelname',
        default='None',
        help='File name for the pbx file')
    args = argparser.parse_args()

    SERVE_PATH = './serve/{}/{}'.format(args.modelname, args.version)

    checkpoint = tf.train.latest_checkpoint(args.savepath)

    tf.reset_default_graph()

    with tf.Session() as sess:
        saver = tf.train.import_meta_graph(checkpoint + '.meta')
        saver.restore(sess, checkpoint)
        graph = tf.get_default_graph()
        inputs = graph.get_tensor_by_name('input/x:0')
        prediction = graph.get_tensor_by_name('accuracy/prediction:0')

        builder = tf.saved_model.builder.SavedModelBuilder(SERVE_PATH)

        # Create tensor info
        model_input = tf.saved_model.utils.build_tensor_info(inputs)
        model_output = tf.saved_model.utils.build_tensor_info(prediction)

        # Build signature definition
        prediction_signature = \
            tf.saved_model.signature_def_utils.build_signature_def(
                inputs={'images': model_input},
                outputs={'scores': model_output},
                method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME)

        signature_def_key = \
            tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY

        builder.add_meta_graph_and_variables(
            sess, [tf.saved_model.tag_constants.SERVING],
            signature_def_map={
                signature_def_key:
                    prediction_signature
            },
            main_op=tf.tables_initializer())

        builder.save()

    print('Done exporting')


import datetime

cnt = 0
StartButton = find (Pattern("1458783675490.png").similar(0.96).targetOffset(-9,4)) #시작
#popup("start")
StopButton=find(Pattern("1458783675490.png").similar(0.96).targetOffset(-13,25))
#popup("stop")
while True:
    while True:
        if cnt==1: # 모든서비스 정지 상태
            if exists("1458783762689.png"):
                click(Pattern("1458783531404.png").similar(0.85).targetOffset(-68,7))
                click(StartButton) #시작
                cnt = 0
                wait(180)
                break
            else:
                wait(1)

        else:      #모든 서비스 동작상태
            if exists(Pattern("1458784172959.png").similar(0.84)):
                click(Pattern("1458783531404.png").similar(0.85).targetOffset(-68,7))
                click(StopButton) #종료
                cnt = 1
                wait(180)
                break

            else:
                wait(1)

    while True:
        count = 1
        f = open("C:/a/b.txt",'a')
        if exists(Pattern("1458782824822.png").similar(0.85)):
            data = s + "    icon 1ea \n"
            f.write(data)

            if exists(Pattern("1458782869295.png").similar(0.96)):
#                f = open("C:/a/b.txt",'a')
                data = s+ "    icon 2ea \n"
                f.write(data)
#                f.close()
                if exists(Pattern("1458782906005.png").similar(0.96)):
#                    f = open("C:/a/b.txt",'a')
                    data = s + "    icon 3ea \n"
                    f.write(data)
#                    f.close()
                    if exists(Pattern("1458782928103.png").exact()):
#                        f = open("C:/a/b.txt",'a')
                        today = datetime.datetime.now()
                        s = str(today)
                        data = s + "    icon 4ea complete! \n"

                        f.write(data)

                        f.close()
                        count = count +1
                        break
                    else:
                        wait(1)
#                        f = open("C:/a/b.txt",'a')
                        today = datetime.datetime.now()
                        s = str(today)
                        data = s + "    Final fail \n"
                        f.write(data)
                        f.close()
                else:
                    wait(1)
#                    f = open("C:/a/b.txt",'a')
                    today = datetime.datetime.now()
                    s = str(today)
                    data = s + "    4fail \n"
                    f.write(data)
#                    f.close()
            else:
                wait(1)
#                f = open("C:/a/b.txt",'a')
                today = datetime.datetime.now()
                s = str(today)
                data = s + "    3fail \n"
                f.write(data)
#                f.close()
        else:
            wait(1)
#            f = open("C:/a/b.txt",'a')
            today = datetime.datetime.now()
            s = str(today)
            data = s + "    2fail \n"

            f.write(data)
#            f.close()


# -*- coding: utf-8 -*-
import simplejson as json

from django.contrib.auth.decorators import permission_required
from django.core.exceptions import ValidationError
from django.http import HttpResponseBadRequest, JsonResponse, QueryDict
from django.views import View
from django.utils import timezone
from django.utils.decorators import method_decorator

from app.models import Donor, Donation, Item

class DonorView(View):
    """DonorView
    - GET: Return JSON serialized Donor object
    - POST: Insert and return that Donor object
    - PUT: Update and return that Donor object
    - DELETE: Delete and return HTTP status code
    """

    def get(self, request):
        try:
            donor = Donor.objects.get(id=request.GET["id"])
            return JsonResponse(donor.camel_serialize(), status=200)
        except ValidationError as e:
            return _error_response(e)
        except Exception as e:
            print(e.args)
            return HttpResponseBadRequest()

    @method_decorator(permission_required("app.can_add_donor"))
    def post(self, request):
        try:
            donor = Donor.objects.create(
                donor_name=request.POST["donorName"],
                email=request.POST["email"],
                want_receipt="wantReceipt" in request.POST,
                telephone_number=request.POST["telephoneNumber"],
                mobile_number=request.POST["mobileNumber"],
                address_line=request.POST["addressLine"],
                city=request.POST["city"],
                province=request.POST["province"],
                postal_code=request.POST["postalCode"],
                customer_ref=request.POST["customerRef"],
                verified="verified" in request.POST
            )
            return JsonResponse(donor.camel_serialize(), status=201)
        except ValidationError as e:
            return _error_response(e)
        except Exception as e:
            print(e.args)
            return HttpResponseBadRequest()

    @method_decorator(permission_required("app.can_change_donor"))
    def put(self, request):
        try:
            request.PUT = QueryDict(request.body)
            donor = Donor.objects.get(id=request.PUT["id"])
            donor.donor_name = request.PUT["donorName"]
            donor.email = request.PUT["email"]
            donor.want_receipt = "wantReceipt" in request.PUT
            donor.telephone_number = request.PUT["telephoneNumber"]
            donor.mobile_number = request.PUT["mobileNumber"]
            donor.address_line = request.PUT["addressLine"]
            donor.city = request.PUT["city"]
            donor.province = request.PUT["province"]
            donor.postal_code = request.PUT["postalCode"]
            donor.customer_ref = request.PUT["customerRef"]
            donor.verified = "verified" in request.PUT
            donor.save()
            return JsonResponse(donor.camel_serialize(), status=200)
        except ValidationError as e:
            return _error_response(e)
        except Exception as e:
            print(e.args)
            return HttpResponseBadRequest()

    @method_decorator(permission_required("app.can_delete_donor"))
    def delete(self, request):
        try:
            request.DELETE = QueryDict(request.body)
            donor = Donor.objects.get(id=request.DELETE["id"])
            donor.delete()
            return JsonResponse({}, status=200)
        except ValidationError as e:
            return _error_response(e)
        except Exception as e:
            print(e.args)
            return HttpResponseBadRequest()

class DonationView(View):
    """DonationView
    - GET: Return Donation object based on tax_receipt_no
    - GET: Return JSON serialized Donation objects based on donor id
    - POST: Insert and return that Donation object
    - PUT: Update and return that Donation object
    - DELETE: Delete and return HTTP status code
    """

    def get(self, request):
        try:
            donation = Donation.objects.get(
                tax_receipt_no=request.GET["taxReceiptNo"]
            )
            return JsonResponse(donation.camel_serialize(), status=200)
        except ValidationError as e:
            return _error_response(e)
        except Exception as e:
            print(e.args)
            return HttpResponseBadRequest()

    @method_decorator(permission_required("app.can_add_donation"))
    def post(self, request):
        try:
            donation = Donation.objects.create(
                donor=Donor.objects.get(id=request.POST["donorId"]),
                donate_date=timezone.localtime().strptime(
                    request.POST["donateDate"], "%Y-%m-%d").date(),
                verified="verified" in request.POST,
                pick_up=request.POST["pickUp"]
            )
            return JsonResponse(donation.camel_serialize(), status=200)
        except ValidationError as e:
            return _error_response(e)
        except Exception as e:
            print(e.args)
            return HttpResponseBadRequest()

    @method_decorator(permission_required("app.can_change_donation"))
    def put(self, request):
        try:
            request.PUT = QueryDict(request.body)
            donation = Donation.objects.get(
                tax_receipt_no=request.PUT["taxReceiptNo"])
            donation.donate_date = request.PUT["donateDate"]
            donation.verified = "verified" in request.PUT
            donation.pick_up = request.PUT["pickUp"]
            donation.save()
            return JsonResponse(donation.camel_serialize(), status=200)
        except ValidationError as e:
            return _error_response(e)
        except Exception as e:
            print(e.args)
            return HttpResponseBadRequest()

    @method_decorator(permission_required("app.can_delete_donation"))
    def delete(self, request):
        try:
            request.DELETE = QueryDict(request.body)
            donation = Donation.objects.get(
                tax_receipt_no=request.DELETE["taxReceiptNo"])
            donation.delete()
            return JsonResponse({}, status=200)
        except ValidationError as e:
            return _error_response(e)
        except Exception as e:
            print(e.args)
            return HttpResponseBadRequest()

class ItemView(View):
    """ItemView
    - GET: Return JSON serialized Item objects based on donation id
    - POST: Insert and return Item object
    - PUT: Update and return Item object
    - DELETE: Delete and return HTTP status code
    """

    def get(self, request):
        try:
            item = Item.objects.get(id=request.GET["id"])
            return JsonResponse(item.camel_serialize(), status=200)
        except ValidationError as e:
            return _error_response(e)
        except Exception as e:
            print(e.args)
            return HttpResponseBadRequest()

    @method_decorator(permission_required("app.can_add_item"))
    def post(self, request):
        try:
            item = Item.objects.create(
                donation=Donation.objects.get(
                    tax_receipt_no=request.POST["taxReceiptNo"]),
                description=request.POST["description"],
                particulars=request.POST["particulars"],
                manufacturer=request.POST["manufacturer"],
                model=request.POST["model"],
                quantity=_safe_cast(request.POST["quantity"], float, 0),
                working="working" in request.POST,
                condition=request.POST["condition"],
                quality=request.POST["quality"],
                status=request.POST["status"],
                batch=request.POST["batch"],
                value=_safe_cast(request.POST["value"], float, 0.0),
                verified="verified" in request.POST
            )
            return JsonResponse(item.camel_serialize(), status=200)
        except ValidationError as e:
            return _error_response(e)
        except Exception as e:
            print(e.args)
            return HttpResponseBadRequest()

    @method_decorator(permission_required("app.can_change_item"))
    def put(self, request):
        try:
            request.PUT = QueryDict(request.body)
            item = Item.objects.get(id=request.PUT["id"])
            item.description = request.PUT["description"]
            item.particulars = request.PUT["particulars"]
            item.manufacturer = request.PUT["manufacturer"]
            item.model = request.PUT["model"]
            item.quantity = _safe_cast(request.PUT["quantity"], float, 0)
            item.working = "working" in request.PUT
            item.condition = request.PUT["condition"]
            item.quality = request.PUT["quality"]
            item.status = request.PUT["status"],
            item.batch = request.PUT["batch"]
            item.value = _safe_cast(request.PUT["value"], float, 0.0)
            item.verified = "verified" in request.PUT
            item.save()
            return JsonResponse(item.camel_serialize(), status=200)
        except ValidationError as e:
            return _error_response(e)
        except Exception as e:
            print(e.args)
            return HttpResponseBadRequest()

    @method_decorator(permission_required("app.can_delete_item"))
    def delete(self, request):
        try:
            request.DELETE = QueryDict(request.body)
            item = Item.objects.get(id=request.DELETE["id"])
            item.delete()
            return JsonResponse({}, status=200)
        except ValidationError as e:
            return _error_response(e)
        except Exception as e:
            print(e.args)
            return HttpResponseBadRequest()

"""
Private Methods
"""

def _safe_cast(val, to_type, default=None):
    try:
        return to_type(val)
    except (ValueError, TypeError):
        return default

def _error_response(err):
    return JsonResponse({"errors": err.messages}, status=400)


#!/usr/bin/env python
# --------------------------------------------------------
#       Main class for Rate Pixel Analysis
# created some time in 2016 by D. Sanz (sandiego@phys.ethz.ch), maintained by M. Reichmann (remichae@phys.ethz.ch)
# --------------------------------------------------------

from __future__ import print_function

from collections import Counter

from ROOT import TFormula, THStack, TProfile2D, TPie, gRandom, TH3F, TMultiGraph
from numpy import corrcoef, ceil

from dut_analysis import *
from pix_cut import CutPix

class PixAnalysis(DUTAnalysis):
    def __init__(self, run_number, dut, test_campaign=None, tree=True, t_vec=None, verbose=False, prnt=True):

        DUTAnalysis.__init__(self, run_number, dut, test_campaign, tree, t_vec, verbose, prnt)

        # Main
        self.Dut = dut + 3
        self.PX, self.PY = self.Run.PixelSize

        if self.Tree:
            self.Cut = CutPix.from_parent(self.Cut)
            self.IsAligned = self.check_alignment()

            # Pulse Height Calibrations
            self.Fit = TF1('ErFit', '[3] * (TMath::Erf((x - [0]) / [1]) + [2])', -500, 255 * 7)
            if self.check_calibration_files():
                self.Parameters = self.load_calibration_fitpars()
                self.Vcals = self.load_vcals()
                self.Points = self.load_calibration_points()

        self.print_finished(prnt=prnt)

    def __del__(self):
        for c in gROOT.GetListOfCanvases():
            c.Close()

    # ----------------------------------------
    # region INIT
    def update_config(self):
        self.Config.read(join(self.Dir, 'Configuration', self.TCString, 'PixelConfig.ini'))

    def load_calibration_files(self, fits=False):
        calibration_list_dir = join(self.Run.Converter.TrackingDir, 'calibration_lists', 'GKCalibrationList_Telescope{n}.txt'.format(n=self.Run.Converter.TelescopeID))
        with open(calibration_list_dir) as f:
            calibration_dir = join(self.Run.Converter.TrackingDir, f.readline().strip('./\n'))
            return [join(calibration_dir, line.strip('\n ') if fits else 'phCalibration_C{}.dat'.format(i)) for i, line in enumerate(f.readlines())]

    def check_calibration_files(self):
        files = concatenate([self.load_calibration_files(), self.load_calibration_files(fits=True)])
        for f in files:
            if not file_exists(f):
                log_warning('Calibration file {} does not exist...'.format(basename(f)))
                return False
        return True

    def load_calibration_fitpars(self, redo=False):
        def f():
            self.check_calibration_files()
            split_at = arange(self.Bins.NRows, self.Bins.NCols * self.Bins.NRows, self.Bins.NRows)  # split at every new column (after n_rows)
            return array([split(genfromtxt(filename, skip_header=3, usecols=arange(4)), split_at) for filename in self.load_calibration_files(fits=True)])
        return do_pickle(self.make_pickle_path('Calibration', 'Pars', run=self.Run.Converter.TelescopeID), f, redo=redo)

    def load_vcals(self, redo=False):
        def func():
            return array([concatenate([genfromtxt(f, 'i2', skip_header=1, max_rows=1)[2:], 7 * genfromtxt(f, 'i2', skip_header=2, max_rows=1)[2:]]) for f in self.load_calibration_files()])
        return do_pickle(self.make_pickle_path('Calibration', 'Vcal', run=self.Run.Converter.TelescopeID), func, redo=redo)

    def load_calibration_points(self, redo=False):
        def func():
            split_at = arange(self.Bins.NRows, self.Bins.NCols * self.Bins.NRows, self.Bins.NRows)
            return [array(split(genfromtxt(f, 'i2', skip_header=4, usecols=arange(self.Vcals[i].size)), split_at)) for i, f in enumerate(self.load_calibration_files())]
        return do_pickle(self.make_pickle_path('Calibration', 'Points', run=self.Run.Converter.TelescopeID), func, redo=redo)
    # endregion INIT
    # ----------------------------------------

    # ----------------------------------------
    # region GET
    def get_ph_str(self):
        return 'cluster_charge[{}]'.format(self.Dut)

    def get_pulse_height(self, bin_size=None, cut=None, redo=False, corr=None):
        suffix = '{bins}_{c}'.format(bins=self.Bins.BinSize if bin_size is None else bin_size, c=self.Cut(cut).GetName())
        picklepath = self.make_pickle_path('Ph_fit', 'Fit', self.RunNumber, self.DUT.Number, suf=suffix)

        def f():
            p, fit_pars = self.draw_pulse_height(bin_size=bin_size, cut=self.Cut(cut), show=False)
            return fit_pars

        return make_ufloat(do_pickle(picklepath, f, redo=redo), par=0)

    def get_thresholds(self, cols=None, pix=None, vcal=True):
        columns, rows = split(array(self.Cut.CutConfig['local_fiducial']), 2) if self.Cut.CutConfig['local_fiducial'] is not None else [0, self.Bins.NCols - 1], [0, self.Bins.NRows - 1]
        columns = array([cols]).flatten() if cols is not None else columns
        columns, rows = (full(2, pix[0]), full(2, pix[1])) if pix is not None else (columns, rows)
        dic = {}
        for col in xrange(columns[0], columns[1] + 1):
            for row in xrange(rows[0], rows[1] + 1):
                self.Fit.SetParameters(*self.Parameters[self.Dut][col][row])
                dic[(col, row)] = self.Fit.GetX(0) * (self.Bins.VcalToEl if not vcal else 1)
        return dic

    def get_vcal(self, redo=False):
        h = self.draw_vcal_distribution(show=False, redo=redo)
        return ufloat(h.GetMean(), h.GetMeanError())
    # endregion GET
    # ----------------------------------------

    # ----------------------------------------
    # region OCCUPANCY
    def draw_occupancy(self, roc=None, name=None, cluster=True, tel_coods=False, cut='', show=True):
        """ draw hitmap or cluster map """
        name = self.DUT.Name if roc is None else name
        roc = self.Dut if roc is None else roc
        return self._draw_occupancy(roc, name, cluster, tel_coods, cut, show)

    def draw_time_occupancy(self, cut=None, fid=False, binning=None, show=True, show_vid=False):
        self.Bins.set_bin_size(binning)
        cut_string = self.Cut.generate_custom(exclude='fiducial' if not fid else [], cluster=False) if cut is None else TCut(cut)
        h = TH3F('h_to', 'to', *(self.Bins.get_time() + self.Bins.get_pixel()))
        self.Tree.Draw('row:col:{}>>h_to'.format(self.get_t_var()), cut_string, 'goff')
        format_histo(h, y_tit='col', z_tit='row')
        gx, gy = [self.make_tgrapherrors('g_to{}'.format(i), 'Mean {}'.format(i.title())) for i in ['x', 'y']]
        for ibin in xrange(h.GetNbinsX() - 1):
            h.GetXaxis().SetRange(ibin, ibin + 1)
            t = h.GetXaxis().GetBinCenter(ibin)
            p = h.Project3D('zy')
            px, py = p.ProfileX(), p.ProfileY()
            for g, p in zip([gx, gy], [px, py]):
                g.SetPoint(ibin, t, p.GetMean())
                g.SetPointError(ibin, 0, p.GetMeanError())
            if show_vid:
                self.draw_histo(p, draw_opt='colz') if not ibin else p.Draw('samecolz')
                c = gROOT.GetListOfCanvases()[-1]
                c.Update()
                c.Modified()
                sleep(.5)
        set_root_output(show)
        c = self.make_canvas('cto', 'Hit Map Evolution', divide=(2, 1), x=2)
        for i, g in enumerate([gx, gy], 1):
            p = c.cd(i)
            p.SetLeftMargin(.13)
            format_histo(g, x_tit='Time [hh:mm]', y_tit='Pixel Position', t_ax_off=0, y_off=1.8)
            g.Draw('ap')
        return h

    # endregion OCCUPANCY
    # ----------------------------------------

    # ----------------------------------------
    # region DISTRIBUTIONS
    def draw_adc_distribution(self, cut=None, show=True, col=None, pix=None):
        h = TH1I('h_adc', 'ADC Distribution {d}'.format(d=self.DUT.Name), *self.Bins.get_adc())
        cut_string = self.Cut(cut) + self.Cut.generate_masks(col=col, pixels=pix, exclude=False)()
        self.format_statbox(entries=True)
        self.Tree.Draw('adc>>h_adc', cut_string, 'goff')
        format_histo(h, x_tit='adc', y_tit='Number of Entries', y_off=1.4, fill_color=self.FillColor)
        self.save_histo(h, 'ADCDisto', show, lm=.13, logy=True)
        return h

    def draw_vcal_distribution(self, cut=None, mcol=None, mpix=None, vcal=True, show=True, redo=False):
        def f():
            h1 = TH1F('h_vcal', 'vcal Distribution {d}'.format(d=self.DUT.Name), *self.Bins.get_ph(vcal))
            cut_string = self.Cut(cut) + self.Cut.generate_masks(col=mcol, pixels=mpix, exclude=False)() + TCut('n_hits[{}] == 1'.format(self.Dut))
            n = self.Tree.Draw('col:row:adc', cut_string, 'goff')
            cols, rows, adcs = self.Run.get_root_vecs(n, 3, dtype=int)
            for i, (col, row, adc) in enumerate(zip(cols, rows, adcs)):
                self.Fit.SetParameters(*self.Parameters[self.Dut][col][row])
                h1.Fill(self.Fit.GetX(adc) * (self.Bins.VcalToEl if not vcal else 1))
            return h1
        h = do_pickle(self.make_simple_pickle_path(sub_dir='VCAL'), f, redo=redo)
        if show:
            self.format_statbox(all_stat=True)
            format_histo(h, x_tit='Pulse Height [{u}]'.format(u='vcal' if vcal else 'e'), y_tit='Number of Entries', y_off=1.4, fill_color=self.FillColor)
            self.save_histo(h, '{p}Disto'.format(p='Ph' if not vcal else 'Vcal'), show, lm=0.13)
        return h

    def draw_signal_distribution(self, cut=None, show=True, prnt=True, roc=None, vcal=False, redo=False, draw_thresh=False):
        roc = self.Dut if roc is None else roc
        cut_string = self.Cut.generate_custom(exclude='masks') if cut is None else TCut(cut)
        pickle_path = self.make_pickle_path('PulseHeight', run=self.RunNumber, suf='{}_{}_{}'.format(roc, cut_string.GetName(), int(vcal)))

        def f():
            set_root_output(False)
            h1 = TH1F('h_phd', 'Pulse Height Distribution - {d}'.format(d=self.DUT.Name), *self.Bins.get_ph(vcal))
            self.Tree.Draw('cluster_charge[{d}]{v}>>h_phd'.format(d=self.Dut, v='/{}'.format(self.Bins.VcalToEl) if vcal else ''), cut_string, 'goff')
            return h1

        h = do_pickle(pickle_path, f, redo=redo)
        x_range = [h.GetXaxis().GetXmin(), h.GetBinCenter(h.FindLastBinAbove(2)) * 1.2]
        self.format_statbox(all_stat=True, x=.92, w=.25)
        format_histo(h, x_tit='Pulse Height [{u}]'.format(u='vcal' if vcal else 'e'), y_tit='Number of Entries', y_off=1.8, fill_color=self.FillColor, x_range=x_range)
        self.draw_histo(h, show=show, lm=.13, rm=.06)
        if draw_thresh:
            self.draw_y_axis(1500, h.GetYaxis().GetXmin(), h.GetMaximum(), 'threshold #approx {}e  '.format(1500), off=.3, line=True, opt='-L')
        self.save_plots('PulseHeightDisto', prnt=prnt)
        return h

    def draw_cluster_disto(self, n=1, cut=None, redo=False, show=True, prnt=True):
        cut_string = TCut(str(n), 'n_clusters[{r}] == {n}'.format(r=self.Dut, n=n)) + self.Cut(cut)
        return self.draw_signal_distribution(cut=cut_string, redo=redo, show=show, prnt=prnt)

    def draw_cluster_ph_distos(self, cut=None, redo=False, show=True):
        hs = OrderedDict([(str(n), self.draw_cluster_disto(n, redo=redo, show=False, prnt=False)) for n in xrange(1, 4)])
        hs['>3'] = self.draw_signal_distribution(cut=TCut('g3', 'n_clusters[{r}] > 3'.format(r=self.Dut)) + self.Cut(cut), show=False, prnt=False, redo=redo)
        stack = THStack('h_cph', 'Pulser Height per Cluster Size')
        l1 = self.make_legend(y2=.88, nentries=5, x1=.7)
        for i, (name, h) in enumerate(hs.iteritems()):
            format_histo(h, color=self.Colors[i], normalise=True, fill_color=self.Colors[i], fill_style=3002, rebin=5)
            l1.AddEntry(h, name, 'fl')
            stack.Add(h)
        x_max = max(h.GetBinCenter(h.FindLastBinAbove(0)) for h in hs.itervalues())
        format_histo(stack, x_tit='Pulse Height [e]', y_tit='Number of Entries', y_off=1.4, draw_first=True, x_range=[self.Bins.MinPH, x_max])
        self.save_histo(stack, 'ClusterPulseHeight', show, lm=.13, leg=l1, draw_opt='nostackhist', gridy=True)
        return hs
    # endregion DISTRIBUTIONS
    # ----------------------------------------

    # ----------------------------------------
    # region PULSE HEIGHT
    def draw_pulse_height(self, cut=None, bin_size=30000, show=True, adc=False):
        """ Pulse height analysis vs event for a given cut. If no cut is provided it will take all. """
        h = TProfile('hpht', '{} -  {}'.format('ADC' if adc else 'Pulse Height', self.DUT.Name), *self.Bins.get_time(bin_size))
        self.format_statbox(only_fit=True, w=.3)
        self.Tree.Draw('{}:{} >> hpht'.format(self.get_ph_str() if not adc else 'adc', self.get_t_var()), self.Cut(cut), 'goff')
        self.draw_histo(h, show=show)
        fit_par = h.Fit('pol0', 'qs')
        format_histo(h, name='Fit Result', y_off=1.9, y_tit='Pulse Height [e]', x_tit='Time [hh:mm]', markersize=.6, t_ax_off=self.Run.StartTime)
        self.save_histo(h, '{}Time'.format('ADC' if adc else 'PulseHeight'), show, lm=.16, draw_opt='e1', save=show, canvas=get_last_canvas())
        return h, FitRes(fit_par)

    def draw_ph_pull(self, event_bin_width=None, fit=True, bin_width=100, save=True, show=True, adc=False):
        return self._draw_ph_pull(event_bin_width, fit, bin_width, bins=self.Bins.get_ph(adc=adc, bin_width=bin_width), show=show, save=save)

    def draw_adc_vs_event(self, cut=None, show=True):
        return self.draw_pulse_height(cut, show=show, adc=True)

    def draw_adc_pull(self, event_bin_width=None, fit=True, save=True, show=True):
        return self.draw_ph_pull(event_bin_width, fit, show=show, save=save, adc=True)

    # endregion PULSE HEIGHT
    # ----------------------------------------

    # ----------------------------------------
    # region 2D DISTRIBUTIONS
    def get_2d_coods(self, roc=None, global_=False, native=False, mm=True):
        fac = '*10' if mm else ''
        coods = 'cluster_ypos_tel[{r}-4]{f}:cluster_xpos_tel[{n}-4]{f}' if global_ else 'row:col' if native else 'dia_track_y_local[{r}]{f}:dia_track_x_local[{r}]{f}'
        return coods.format(r=self.Dut if roc is None else roc, f=fac)

    def draw_adc_map(self, show=True, cut=None, zero_ratio=False):
        cut_string = self.Cut(cut) + self.Cut.generate_masks(cluster=False)()
        set_root_output(False)
        h = TProfile2D('p_am', 'ADC Map', *self.Bins.get_pixel())
        self.Tree.Draw('adc{}:row:col >> p_am'.format('<0' if zero_ratio else ''), cut_string, 'goff')
        self.format_statbox(x=.81, entries=True)
        format_histo(h, x_tit='col', y_tit='row', z_tit='ADC', y_off=1.3, z_off=1.5, z_range=[0, 1] if zero_ratio else None)
        self.save_histo(h, 'ADCMap', show, rm=.17, lm=.13, draw_opt='colz')

    def draw_threshold_map(self, vcal=True, cols=None, show=True):
        h = TProfile2D('p_tm', 'Artificial Threshold Map', *self.Bins.get_pixel())
        for (col, row), thresh in self.get_thresholds(cols, vcal=vcal).iteritems():
            h.Fill(col, row, thresh if vcal else thresh / 1000.)
        format_histo(h, x_tit='col', y_tit='row', z_tit='Artificial Treshold [{u}]'.format(u='vcal' if vcal else 'ke'), y_off=1.3, z_off=1.7, stats=0)
        self.save_histo(h, 'ThresholdMap', show, rm=.17, lm=.13, draw_opt='colz')
        return h

    def draw_adc_fixed_vcal_map(self, roc=None, vcal=200, show=True):
        roc = self.Dut if roc is None else roc
        h = TProfile2D('p_pm', 'ADC Map for Vcal {v}'.format(v=vcal), *self.Bins.get_pixel())
        cols, rows = split(array(self.Cut.CutConfig['local_fiducial']), 2) if self.Cut.CutConfig['local_fiducial'] is not None else [0, self.Bins.NCols - 1], [0, self.Bins.NRows - 1]
        for col in xrange(cols[0], cols[1] + 1):
            for row in xrange(rows[0], rows[1] + 1):
                self.Fit.SetParameters(*self.Parameters[roc][col][row])
                h.Fill(col, row, self.Fit(vcal))
        format_histo(h, x_tit='col', y_tit='row', z_tit='Pulse Height [adc]', y_off=1.3, z_off=1.5, stats=0)
        self.save_histo(h, 'ADCMap{v}'.format(v=vcal), show, rm=.17, lm=.13, draw_opt='colz')

    def draw_sig_map_disto(self, res=None, cut=None, fid=True, x_range=None, redo=False, normalise=False, ret_value=False, show=True, save=True):
        return self._draw_sig_map_disto(res, cut, fid, x_range, redo, normalise, ret_value, ph_bins=self.Bins.get_ph(), show=show, save=save)
    # endregion 2D DISTRIBUTIONS
    # ----------------------------------------

    # ----------------------------------------
    # region CALIBRATION
    def draw_calibration_fit(self, col, row, show=True, roc=None):
        roc = self.Dut if roc is None else roc
        self.Fit.SetParameters(*self.Parameters[roc][col][row])
        format_histo(self.Fit, title='Calibration Fit for Pix {c} {r}'.format(c=col, r=row), x_tit='vcal', y_tit='adc', y_off=1.4, color=632, lw=2)
        gr = TGraph(len(self.Vcals[roc]), array(self.Vcals[roc], 'd'), array(self.Points[roc][col][row], 'd'))
        format_histo(gr, marker=20, name='gr_fp', title='Calibration Fit for Pix {c} {r}'.format(c=col, r=row), x_tit='vcal', y_tit='adc', y_off=1.4)
        self.draw_histo(gr, lm=.12, draw_opt='ap')
        self.Fit.Draw('same')
        self.save_plots('CalFit{c}{r}'.format(c=col, r=row), show=show)

    def fit_erf(self, col, row, roc=None, show=True):
        roc = self.Dut if roc is None else roc
        # fit.SetParameters(*self.Parameters[roc][col][row])
        points = OrderedDict(sorted({vcal: value for vcal, value in zip(self.Vcals, self.Points[roc][col][row])}.iteritems()))
        good_points = OrderedDict(sorted({vcal: value for vcal, value in zip(self.Vcals, self.Points[roc][col][row]) if value}.iteritems()))
        start = 0
        for vcal, value in points.iteritems():
            if value:
                start = vcal
                break
        if len(good_points) > 3:
            fit = deepcopy(self.Fit)
            fit.SetParameters(309.2062, 112.8961, 1.022439, 35.89524)
        else:
            fit = TF1('fit', 'pol1', 0, 3000)
        if len(good_points) > 1:
            gr = TGraph(len(points), array(points.keys(), 'd'), array(points.values(), 'd'))
            gr.Fit(fit, 'q', '', start, 3000 if len(good_points) > 3 else 1000)
            format_histo(gr, marker=20, x_tit='vcal', y_tit='adc', y_off=1.3, title='Calibration Fit for Pix {c} {r}'.format(c=col, r=row))
            self.draw_histo(gr, draw_opt='ap', show=show, lm=.12)
            return fit
    # endregion CALIBRATION
    # ----------------------------------------

    # ----------------------------------------
    # region EFFICIENCY
    def get_efficiency_cut(self, trig_phase=True):
        return self.Cut.generate_custom(include=['fiducial', 'rhit', 'tracks', 'chi2_x', 'chi2_y', 'aligned', 'event_range', 'beam_interruptions'] + (['trigger_phase'] if trig_phase else []))

    def get_hit_efficiency(self, roc=None, cut=None):
        cut_string = self.get_efficiency_cut() if cut is None else TCut(cut)
        roc = self.Dut if roc is None else roc
        n = self.Tree.Draw('n_hits[{r}]>0'.format(r=roc), cut_string, 'goff')
        values = self.Run.get_root_vec(n, dtype=bool)
        return calc_eff(values=values)

    def draw_eff_vs_chi2(self, step=10, show=True):
        g = self.make_tgrapherrors('gec2', 'Efficiency vs Chi2')
        for i, chi2 in enumerate(arange(step, 100 + step / 100., step)):
            self.Cut.set_chi2(chi2)
            eff = self.get_hit_efficiency()
            g.SetPoint(i, chi2, eff.n)
            g.SetPointError(i, 0, eff.s)
        format_histo(g, x_tit='#chi^{2} [%quantile]', y_tit='Efficiency [%]', y_off=1.3)
        self.draw_histo(g, draw_opt='ap', lm=.12, show=show)

    def draw_hit_efficiency(self, save=True, cut=None, vs_time=True, bin_width=5000, n=1e9, start=0, show=True):
        set_root_output(False)
        h = TProfile('h_he', 'Hit Efficiency {s}'.format(s=self.Dut), *self.Bins.get(bin_width, vs_time))
        cut_string = self.Cut.generate_custom(exclude=['masks']) if cut is None else TCut(cut)
        x_var = self.get_t_var() if vs_time else 'event_number'
        self.Tree.Draw('(n_hits[{r}]>0)*100:{x} >> h_he'.format(r=self.Dut, x=x_var), cut_string, 'goff', int(n), start)
        g = self.make_graph_from_profile(h)
        fit = fix_chi2(g, .01, show)
        format_histo(g, x_tit='Time [hh:mm]' if vs_time else 'Event Number', y_tit='Efficiency [%]', y_off=1.4, y_range=[-5, 115], stats=0,
                     t_ax_off=self.Run.StartTime if vs_time else None, markersize=1.2, draw_first=True)
        self.draw_histo(g, show=show, lm=.13, gridy=True, draw_opt='apz', bm=.2)
        self.draw_stats(fit, width=.35, y2=.35, names=['Efficiency'])
        self.draw_preliminary()
        self.save_plots('HitEfficiencyROC{n}'.format(n=self.Dut), save=save, show=show)
        return fit if fit.Parameter(0) is not None else 0

    def draw_efficiency_map(self, res=None, cut='all', show=True):
        cut_string = TCut(cut) + self.Cut.CutStrings['tracks']
        cut_string = self.Cut.generate_custom(exclude=['masks', 'fiducial']) if cut == 'all' else cut_string
        p = TProfile2D('p_em', 'Efficiency Map {d}'.format(d=self.DUT.Name), *self.Bins.get_global(res_fac=res, mm=True))
        self.Tree.Draw('(n_hits[{r}]>0)*100:{}:{}>>p_em'.format(r=self.Dut, *self.Cut.get_track_vars(self.Dut - 4, mm=True)), cut_string, 'goff')
        self.format_statbox(entries=True, x=.81)
        format_histo(p, x_tit='Track Position X [mm]', y_tit='Track Position Y [mm]', z_tit='Efficiency [%]', y_off=1.4, z_off=1.5)
        self.draw_histo(p, 'Efficiency Map', show, lm=.13, rm=.17, draw_opt='colz')
        self.draw_fid_cut(scale=10)
        self.draw_preliminary()
        self.save_plots('Efficiency Map')

    def get_fiducial_cell(self, n):
        x1, x2, y1, y2 = self.Cut.CutConfig['fiducial']
        nx = int(round((x2 - x1) / self.PX))
        # ny = int(round((y2 - y1) / .010))
        return round(x1 + self.PX * (n % nx), 4), round(y1 + self.PY * (n / nx), 4)

    def draw_cell_efficiency(self, cell=0, res=2, show=True):
        # x, y = self.get_fiducial_cell(cell)
        cut_string = self.Cut.generate_custom(exclude=['masks'])
        p = TProfile2D('pce', 'Efficiency for Fiducial Cell {}'.format(cell), res, 0, self.PX, res, 0, self.PY)
        n = self.Tree.Draw('(n_hits[{r}]>0)*100:dia_track_y_local[{r1}]:dia_track_x_local[{r1}]>>pce'.format(r=self.Dut, r1=self.Dut - 4), cut_string, 'goff')
        effs = [self.Tree.GetV1()[i] for i in xrange(n)]
        x_vals = [self.Tree.GetV3()[i] for i in xrange(n)]
        y_vals = [self.Tree.GetV2()[i] for i in xrange(n)]
        for e, x, y in zip(effs, x_vals, y_vals):
            p.Fill(x % self.PX, y % self.PY, e)
        self.format_statbox(entries=True, x=.81)
        format_histo(p, x_tit='Track x [cm]', y_tit='Track y [cm]', z_tit='Efficiency [%]', y_off=1.4, z_off=1.5)
        self.draw_histo(p, show=show, lm=.13, rm=.17, draw_opt='colz')

    def draw_efficiency_vs_trigphase(self, show=True):
        n = self.Tree.Draw('n_hits[{}]>0:trigger_phase[1]'.format(self.Dut), self.get_efficiency_cut(trig_phase=False), 'goff')
        hits, tp = self.Run.get_root_vecs(n, 2)
        y = [calc_eff(values=hits[where(tp == i)]) for i in arange(10)]
        g = self.make_tgrapherrors('getp', 'Efficiency per Trigger Phase', x=arange(10), y=y)
        format_histo(g, fill_color=self.FillColor, x_tit='Trigger Phase', y_tit='Efficiency [%]', y_off=1.4, x_range=[-1, 10])
        self.save_histo(g, 'EffVsTrigPhase', show, draw_opt='ba', lm=.13)
    # endregion EFFICIENCY
    # ----------------------------------------

    # ----------------------------------------
    # region TRIGGER PHASE
    def draw_trigger_phase(self, cut=None, show=True):
        return self._draw_trigger_phase(dut=True, cut=self.Cut.generate_custom(exclude='trigger_phase') if cut is None else cut, show=show)

    def draw_trigger_phase_time(self, bin_width=30000, cut=None, show=True):
        return self._draw_trigger_phase_time(dut=True, bin_width=bin_width, cut=cut, show=show)

    def draw_trigphase_offset(self, cut=None, show=True):
        h = TH1I('h_tp', 'Trigger Phase Offset', 18, -9.5, 8.5)
        self.Tree.Draw('trigger_phase[1] - trigger_phase[0]>>h_tp', self.Cut.generate_custom(exclude='trigger_phase') if cut is None else cut, 'goff')
        self.format_statbox(entries=True, y=0.88)
        format_histo(h, x_tit='Trigger Phase', y_tit='Number of Entries', y_off=1.8, fill_color=self.FillColor, ndivx=20)
        self.save_histo(h, 'TPOff', show, lm=.16)

    def draw_trigphase_off_time(self, bin_width=30000, cut=None, show=True):
        h = TProfile('htpot', 'Trigger Phase vs Time', *self.Bins.get(bin_width, vs_time=True))
        self.Tree.Draw('(trigger_phase[1] - trigger_phase[0]):{}>>htpot'.format(self.get_t_var()), self.Cut.generate_custom(exclude='trigger_phase') if cut is None else cut, 'goff')
        self.format_statbox(entries=True, y=0.88)
        format_histo(h, x_tit='Time [hh:mm]', y_tit='Trigger Phase', y_off=1.8, fill_color=self.FillColor, t_ax_off=self.Run.StartTime)
        self.save_histo(h, 'TPOffTime', show, lm=.16)
    # endregion TRIGGER PHASE
    # ----------------------------------------

    # ----------------------------------------
    # region ARTIFICIAL THRESHOLD
    def find_landau(self, aver=10, m1=2500, m2=5000, s1=500, s2=1600):
        seed = self.draw_signal_distribution(show=False)
        h = deepcopy(seed)
        m_range = range(m1, m2 + 1, 100)
        s_range = range(s1, s2 + 1, 50)
        p = TProfile2D('g_fl', 'Find Landau', len(m_range) - 1, m_range[0], m_range[-1], len(s_range) - 1, s_range[0], s_range[-1])
        self.PBar.start(len(m_range) * len(s_range) * aver)
        i = 0
        r_min, r_max = .5, 1.5
        for _ in xrange(aver):

            for m in m_range:
                for s in s_range:
                    i += 1
                    self.PBar.update(i)
                    if r_min < m / 4. / s < r_max:
                        diff = self.model_landau(seed, h, m, s, show=False, thresh=True)
                        p.Fill(m, s, diff)
        self.PBar.finish()
        self.format_statbox(entries=True, x=.82)
        format_histo(p, x_tit='MPV [e]', y_tit='Sigma [e]', z_tit='#chi^{2} to Seed Function', y_off=1.7, z_off=1.3)
        self.draw_histo(p, draw_opt='colz', lm=.13, rm=0.16)
        self.draw_ms_ratios(r_min, r_max, m1, m2, s1, s2)
        self.save_plots('FindLandau')
        self.find_working_point(p)

    def draw_ms_ratios(self, r_min, r_max, m1, m2, s1, s2, step=.1):
        ratios = arange(r_min, r_max + step, step)
        off = .01
        for i, ratio in enumerate(ratios):
            cut = TCutG('ms{n}'.format(n=i), 2, array([0, 20000], 'd'), array([0, 20000 / ratio / 4]))
            x_pos = m1 if m1 / ratio / 4 > s1 else 4 * ratio * s1
            y_pos = m1 / ratio / 4 if m1 / ratio / 4 > s1 else s1
            self.draw_tlatex(x_pos + off * (m2 - m1), y_pos + off * (s2 - s1), text='{0:3.1f}'.format(ratio), size=.02, align=11)
            cut.Draw('same')
            self.Objects.append(cut)

    @staticmethod
    def find_working_point(h):
        ps = [h.ProfileY(), h.ProfileX()]
        fits = [TF1('f{n}'.format(n=i), 'pol2', 0, 10000) for i in xrange(2)]
        for fit, p in zip(fits, ps):
            p.Fit(fit, 'qs0')
        mins = [fit.GetMinimumX() for fit in fits]
        print(mins, mins[1] / mins[0] / 4)

    def model_landau(self, seed=None, h=None, m=10000, s=1000, show=True, thresh=False):
        seed = self.draw_signal_distribution(show=False) if seed is None else seed
        h = deepcopy(seed) if h is None else h
        h.SetName('h_ml')
        n = seed.GetEntries()
        h.Reset()
        thresholds = [47 * i for i in [150, 160, 170]]
        for _ in xrange(int(n)):
            v = gRandom.Landau(m, s)
            threshold = thresholds[int(gRandom.Rndm() * len(thresholds))]
            h.Fill(v if v > threshold else 0 if thresh else v)
            # h.Fill(v if v > 11421 else 0 if thresh else v)
        diff = mean([(h.GetBinContent(i) - seed.GetBinContent(i)) ** 2 for i in xrange(h.GetNbinsX()) if i is not h.FindBin(0)])
        seed.SetFillColor(2)
        h.SetFillColor(self.FillColor)
        seed.SetFillStyle(3017)
        h.SetFillStyle(3018)
        l1 = self.make_legend(y2=.76)
        l1.AddEntry(h, 'Simulation', 'f')
        l1.AddEntry(seed, 'Original', 'f')
        self.draw_histo(h, show=show, leg=l1)
        self.draw_histo(seed, show=show, draw_opt='same', canvas=gROOT.GetListOfCanvases()[-1])
        seed.Draw('same')
        return diff

    def landau_vid(self, save=False, mpv=5000, sigma=820):
        h = self.draw_signal_distribution()
        h.GetYaxis().SetRangeUser(0, 2500)
        zero_bin = h.FindBin(0)
        zero_entries = int(h.GetBinContent(zero_bin))
        entries = int(h.GetEntries())
        print(entries)
        c = gROOT.GetListOfCanvases()[-1]
        thresholds = self.get_thresholds(vcal=False)
        for i in xrange(entries):
            h.SetBinContent(zero_bin, zero_entries)
            v = gRandom.Landau(mpv, sigma)
            threshold = thresholds[int(rand() * self.Bins.NCols), int(rand() * self.Bins.NRows)]
            if v < threshold:
                h.Fill(v)
                zero_entries -= 1
            if i % 100 == 0:
                c.Update()
                c.Modified()
            if i % 100 == 0 and save:
                self.save_canvas(c, name='l{i:04d}'.format(i=i), show=False, print_names=False)
    # endregion ARTIFICIAL THRESHOLD
    # ----------------------------------------

    # ----------------------------------------
    # region ALIGNMENT
    def draw_alignment(self, plane1=2, plane2=None, mode='y', binning=5000, chi2=1, show=True, vs_time=True, redo=False):
        plane2 = self.Dut if plane2 is None else plane2
        picklepath = self.make_pickle_path('Alignment', run=self.RunNumber, suf='{m}_{p1}{p2}_{b}_{t}'.format(m=mode, p1=plane1, p2=plane2, b=binning, t='Time' if vs_time else 'EvtNr'))

        def func():
            start = self.info('Checking for alignment between plane {p1} and {p2} ... '.format(p1=plane1, p2=plane2), next_line=False)
            h = TH3F('hpa', 'pa', *(self.Bins.get(binning, vs_time=vs_time) + 2 * self.Bins.get_global_cood(mode, res_fac=sqrt(12))))
            y, z_ = ('cluster_{m}pos_tel[{r}]'.format(m=mode, r=plane) for plane in [plane1, plane2])
            cut_string = TCut('n_clusters[{p1}]==1 && n_clusters[{p2}]==1'.format(p1=plane1, p2=plane2)) + self.Cut.generate_chi2(mode, chi2)()
            self.Tree.Draw('{z}:{y}:{x}>>hpa'.format(z=z_, y=y, x=self.get_t_var() if vs_time else 'Entry$'), cut_string, 'goff')
            g = self.make_tgrapherrors('g_pa', 'Plane Correlation {p1} {p2}'.format(p1=plane1, p2=plane2), marker_size=.5)
            for ibin in xrange(h.GetNbinsX() - 1):
                h.GetXaxis().SetRange(ibin, ibin + 1)
                p = h.Project3D('yz')
                g.SetPoint(ibin, h.GetXaxis().GetBinCenter(ibin), p.GetCorrelationFactor())
            format_histo(g, x_tit='Time [hh::mm]' if vs_time else 'Event Number', y_tit='Correlation Factor', y_off=1.5, y_range=[0, 1], t_ax_off=self.Run.StartTime if vs_time else None)
            self.add_to_info(start)
            return g

        gr = do_pickle(picklepath, func, redo=redo)
        self.save_histo(gr, 'PixelAligment', show, draw_opt='alp', lm=.13, prnt=show)
        return gr

    def check_alignment(self):
        pickle_path = self.make_pickle_path('Alignment', run=self.RunNumber)

        def f():
            g = self.draw_alignment(show=False)
            x, y = get_graph_vecs(g)
            return mean_sigma(y)

        m, s = do_pickle(pickle_path, f)
        if m < .4:
            log_warning('Planes are not correlated!')
        elif s > .05:
            log_warning('Large fluctuations in correlation!')
        return m > .4

    def draw_correlation(self, plane1=2, plane2=None, mode='y', chi2=None, res=.7, start=0, evts=int(1e10), cut=None, show=True):
        old_chi2 = self.Cut.CutConfig['chi2_x']
        self.Cut.set_chi2(old_chi2 if chi2 is None else chi2)
        plane2 = self.Dut if plane2 is None else plane2
        h = TH2F('h_pc', 'Plane Correlation', *self.Bins.get_global(res_fac=res))
        cut = self.Cut(cut)
        self.Tree.Draw('cluster_{m}pos_tel[{p1}]:cluster_{m}pos_tel[{p2}]>>h_pc'.format(m=mode, p1=plane1, p2=plane2), cut, 'goff', evts, start)
        self.info('Correlation Factor: {f:4.3f}'.format(f=h.GetCorrelationFactor()))
        format_histo(h, x_tit='{m} Plane {p}'.format(p=plane1, m=mode), y_tit='{m} Plane {p}'.format(p=plane2, m=mode), y_off=1.5, stats=0, z_tit='Number of Entries', z_off=1.5)
        self.save_histo(h, 'PlaneCorrelation{m}{p1}{p2}'.format(m=mode.title(), p1=plane1, p2=plane2), show, lm=.13, draw_opt='colz', rm=.17)
        self.Cut.set_chi2(old_chi2)

    def draw_event_offsets(self, evnts=1000, start=0, pnts=None, rnge=1, show=True):
        pnts = int(ceil(self.Run.NEntries / evnts)) if pnts is None else pnts
        n_graphs = 2 * rnge + 1
        graphs = [self.make_tgrapherrors('g_to{i}'.format(i=i), '', color=self.get_color(), marker_size=.5) for i in xrange(n_graphs)]
        l1 = self.make_legend(x1=.8, y2=.5, nentries=n_graphs - 1)
        self.PBar.start(pnts * evnts - start)
        p1 = self.Dut
        p2 = 2
        for k in xrange(pnts):
            x, y = OrderedDict(), OrderedDict()
            n = self.Tree.Draw('plane:row:event_number', 'n_hits[2]==1 || n_hits[{p1}]==1'.format(p1=p1), 'goff', evnts, start + k * evnts)
            planes = [int(self.Tree.GetV1()[i]) for i in xrange(n)]
            rows = [int(self.Tree.GetV2()[i]) for i in xrange(n)]
            nrs = Counter([int(self.Tree.GetV3()[i]) for i in xrange(n)])
            for ev, size in sorted(nrs.iteritems()):
                self.PBar.update(ev + 1 - start)
                plane = [planes.pop(0) for _ in xrange(size)]
                row = [rows.pop(0) for _ in xrange(size)]
                if plane.count(p1) == 1:
                    x[ev] = row[plane.index(p1)]
                if plane.count(p2) == 1:
                    y[ev] = row[plane.index(p2)]
            xts = [[] for _ in xrange(n_graphs)]
            yts = [[] for _ in xrange(n_graphs)]
            for i in xrange(-rnge, rnge + 1):
                j = i + rnge
                for ev, row in x.iteritems():
                    if ev + i in y:
                        xts[j].append(row)
                        yts[j].append(y[ev + i])
                corr = corrcoef(xts[j], yts[j])[0][1]
                graphs[j].SetPoint(k, k * evnts + start, corr)
        self.PBar.finish()
        mg = TMultiGraph('m_eo', 'Correlations')
        for i, gr in enumerate(graphs):
            l1.AddEntry(gr, 'offset: {i}'.format(i=i - rnge), 'pl')
            mg.Add(gr, 'pl')
        format_histo(mg, x_tit='Event Number', y_tit='Correlation Factor', y_off=1.5, y_range=[-.2, 1], draw_first=True)
        self.save_histo(mg, 'EventOffsets', show, draw_opt='ap', leg=l1, lm=.13)
        self.reset_colors()
    # endregion ALIGNMENT
    # ----------------------------------------

    # ----------------------------------------
    # region DRAW
    def draw_dut_hits(self, dut1=4, dut2=5, cut=None, show=True):
        h = TH2F('h_hd', 'Number of Hits', 40, 0, 40, 40, 0, 40)
        self.Tree.Draw('n_hits[{}]:n_hits[{}]>>h_hd'.format(dut2, dut1), self.Cut(cut), 'goff')
        self.format_statbox(entries=True, x=.81)
        x_tit, y_tit = ('Number of Hits in {}'.format(self.Run.DUT.Names[dut - 4]) for dut in [dut1, dut2])
        format_histo(h, x_tit=x_tit, y_tit=y_tit, y_off=1.3, z_tit='Number of Entries', z_off=1.4, x_range=[0, h.FindLastBinAbove(0, 1) + 1], y_range=[0, h.FindLastBinAbove(0, 2) + 1])
        self.save_histo(h, 'HitsDiaSil', show, draw_opt='colz', rm=0.17, lm=.13, logz=True)

    def draw_hit_pie(self, d2=5):
        zero, dia, dut2, both = (self.Tree.GetEntries(s.format(d2)) for s in ['n_hits[4]==0&&n_hits[{}]==0', 'n_hits[4]>0&&n_hits[{}]==0', 'n_hits[4]==0&&n_hits[{}]>0', 'n_hits[4]>0&&n_hits[{}]>0'])
        names = ['No Hits', '{} Hit'.format(self.DUT.Name), '{} Hit'.format(self.Run.DUT.Names[d2 - 4]), 'Both Hits']
        pie = TPie('pie', 'Hit Contributions', 4, array([zero, dia, dut2, both], 'f'), array(self.get_colors(4), 'i'))
        for i, label in enumerate(names):
            pie.SetEntryRadiusOffset(i, .05)
            pie.SetEntryLabel(i, label.title())
        format_pie(pie, h=.04, r=.2, text_size=.025, angle3d=70, angle_off=250)
        self.draw_histo(pie, draw_opt='3drsc')
        self.reset_colors()

    def draw_cluster_size(self, cut='', show=True):
        return self._draw_cluster_size(self.Dut, self.DUT.Name, cut, show)

    def draw_residuals(self, mode=None, cut=None, show=True, x_range=None):
        return self._draw_residuals(self.Dut, mode=mode, cut=cut, show=show, x_range=x_range)

    def draw_slope(self, show=True):
        with open(join(self.Dir, 'data', 'vcalCalibration.txt')) as f:
            d = load(f)
            h = TH1F('ho', 'Vcal Calibration Slopes', 30, 35, 65)
            for value in d['slope']:
                h.Fill(value)
            self.format_statbox(fit=True, all_stat=True, w=.3)
            format_histo(h, x_tit='Slope [e]', y_tit='Number of Entries', y_off=1.2, fill_color=self.FillColor)
            self.draw_histo(h, show=show)
            h.Fit('gaus', 'q')

    def draw_offset(self, show=True):
        with open(join(self.Dir, 'data', 'vcalCalibration.txt')) as f:
            d = load(f)
            h = TH1F('ho', 'Vcal Calibration Offsets', 15, -200, 400)
            for value in d['offset']:
                h.Fill(value)
            self.format_statbox(fit=True, all_stat=True, w=.3)
            format_histo(h, x_tit='Offset [e]', y_tit='Number of Entries', y_off=1.2, fill_color=self.FillColor)
            self.format_statbox(fit=True, all_stat=True)
            self.draw_histo(h, show=show)
            h.Fit('gaus', 'q')
    # endregion DRAW
    # ----------------------------------------

    def do_analysis(self, do_cut_dist=False, do_res_ana=False, do_cut_ana=False, do_occupancy=True, do_pulse_height=True):
        """ Does automatic analysis with the selected options """
        TFormula.SetMaxima(1000000, 10000, 10000000)  # (1000,1000,1000)

        if do_cut_dist:
            self.Cut.do_cuts_distributions()
        if do_res_ana:
            self.Cut.do_res_analysis()
        if do_cut_ana:
            self.do_cuts_analysis(do_occupancy, do_pulse_height, True)

    def do_cuts_analysis(self, do_occupancy, do_pulse_height, normalize_ph_plots=False):
        """ Calls do_cut_analysis in the Cut method """
        self.Cut.do_cuts_analysis(do_occupancy, do_pulse_height, normalize_ph_plots)

if __name__ == '__main__':

    pargs = init_argparser(run=139, tc='201810', dut=1, has_verbose=True, tree=True)
    z = PixAnalysis(pargs.run, pargs.dut, pargs.testcampaign, pargs.tree, pargs.verbose)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
import django.db.models.deletion
import methodology.models
import config.storage_backends
from django.conf import settings

class Migration(migrations.Migration):

    dependencies = [
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
    ]

    operations = [
        migrations.CreateModel(
            name='Methodology',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('created_at', models.DateTimeField(auto_now_add=True)),
                ('date_decision', models.DateTimeField()),
                ('date_deleted', models.DateTimeField(blank=True, null=True)),
                ('upload', models.FileField(upload_to=methodology.models.get_upload_to, storage=config.storage_backends.AnalyticalMediaStorage())),
            ],
            options={
                'ordering': ('category', '-date_decision'),
            },
        ),
        migrations.CreateModel(
            name='MethodologyCategory',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('name', models.CharField(max_length=100)),
            ],
        ),
        migrations.AddField(
            model_name='methodology',
            name='category',
            field=models.ForeignKey(related_name='methodology_methodology_category', to='methodology.MethodologyCategory', on_delete=django.db.models.deletion.PROTECT),
        ),
        migrations.AddField(
            model_name='methodology',
            name='uploaded_by',
            field=models.ForeignKey(related_name='methodology_methodology_user', blank=True, to=settings.AUTH_USER_MODEL, null=True, on_delete=django.db.models.deletion.PROTECT),
        ),
    ]


import os
import json
import numpy as np

###################################################################################################

def getObjTemplate(objid, pos, color_palette):

    filepath = os.path.join(FILE_PATH, objid[:-2], objid+'.stl')
    metapath = os.path.join(META_PATH, objid[:-2], objid+'.json')

    angle = np.random.randint(conf['objects']['angle-min'], conf['objects']['angle-max'])
    #scale = np.random.choice(conf['objects']['scales'])
    color = np.random.choice(color_palette)
    z = 0.

    if 'DRG' in objid:
        scale = 1.0
        z = 0.02

    if 'MON' in objid:
       scale = 1.5

    if 'MYT' in objid:
       scale = 1.6

    if 'ADV' in objid:
       scale = 1.3

    if 'ANM' in objid:
       scale = 1.5

    obj = {}
    obj['filepath'] = filepath
    obj['metapath'] = metapath
    obj['properties'] = {
        'loc' :  pos[ np.random.randint(len(pos)) ] + [z],
        'rot' :  int(angle),
        'scale': float(scale),
        'color': color
    }

    return obj

def getObjList(group):

    num_obj = np.random.randint(conf['scene']['objcount-min'], conf['scene']['objcount-max'])
    objs = np.random.choice(np.array(group['ids'], dtype=object), size=num_obj)
    objids = [ np.random.choice(a)+'_'+str(inx) for inx, a in enumerate(objs) ]
    return objids

###################################################################################################

confpath = './SceneGenerator-conf_train.json'
with open(confpath, 'r') as fp:
    conf = json.load(fp)

with open(conf['meta']['groupinfo'],'r') as fp:
    groups = json.load(fp)

group = groups[conf['meta']['groupname']]
seed = conf['meta']['seed']
FILE_PATH = conf['meta']['fileglobalpath']
META_PATH = conf['meta']['metaglobalpath']

np.random.seed(seed)
for num_scene in range(group['samples']):

    scene = {"objects":{}, "meta":{}}
    objects = getObjList(group)
    pos = conf['objects']['positions'].copy()
    color_palette = np.random.choice( conf['objects']['colors'],
                                      size=conf['objects']['colorcount'], replace=False)

    for objid in objects:
        scene['objects'][objid] = getObjTemplate(objid[:-2], pos, color_palette)
        pos.remove(scene['objects'][objid]['properties']['loc'][:2])

    persp_max, persp_min = conf['scene']['presp-angle-max'], conf['scene']['presp-angle-min']
    scene['objects']['Camera'] = { 'rot': np.random.randint(persp_min, persp_max) }

    savepath = os.path.join( conf['meta']['savepath'],
                             'group{}scene{}.json'.format(conf['meta']['groupname'], num_scene) )
    with open(savepath, 'w') as fp:
        json.dump(scene, fp, indent=4)


import argparse
import torch
from torch.autograd import Variable
import numpy as np
import time, math, glob
import scipy.io as sio
import matplotlib.pyplot as plt
import cv2
import os
import pandas as pd
from skimage.measure import compare_ssim
from util.measure import PSNR, SSIM
from PIL import Image

if __name__ == '__main__':
    cuda = True
    if cuda and not torch.cuda.is_available():
        raise Exception("No GPU found, please run without --cuda")

    model_path = '/home/server/LihuaJian/result'
    model_name = 'best_99'
    model = torch.load(model_path+'/'+model_name + '.pth')["model"]
    model.eval()

    datasets_path = '/media/server/80SSD/LihuaJian/test/10/Input'
    GTs_path = '/media/server/80SSD/LihuaJian/test/10/GT'
    datasets = ['10']

    result_path = '/media/server/80SSD/LihuaJian/result' + '/' + model_name
    results = {}

    for dataset in datasets:
        path = datasets_path + '/' + dataset
        GT_path = GTs_path+'/'+dataset

        save_path = result_path+'/'+dataset
        if not os.path.exists(save_path):  # 如果路径不存在
            os.makedirs(save_path)
        image_list = glob.glob(path + "/*.*")
        gt_list = glob.glob(GT_path + '/*.*')
        assert len(gt_list) == len(image_list)

        avg_elapsed_time = 0.0
        avg_psnr_predicted = 0.0
        avg_ssim_predicted = 0.0
        gray_image = False

        for idx, image_name in enumerate(image_list):
            print("Processing ", image_name)
            im_input = plt.imread(image_name)
            gt = plt.imread(gt_list[idx])

            if len(im_input.shape) == 2:
                gray_image = True
                im_input = np.reshape(im_input, [im_input.shape[0], im_input.shape[1], 1])
                im_input = np.concatenate((im_input, im_input, im_input), 2)

            im_input = np.transpose(im_input, [2, 0, 1])
            im_input = im_input[np.newaxis, :]

            im_input = torch.from_numpy(im_input).float()
            if torch.max(im_input) > 10:
                im_input = im_input/255.

            if cuda:
                model = model.cuda()
                im_input = im_input.cuda()
            else:
                model = model.cpu()

            start_time = time.time()
            gt_o = model(im_input)
            elapsed_time = time.time() - start_time
            avg_elapsed_time += elapsed_time

            gt_o = gt_o.cpu()
            gt_o = gt_o.data[0].numpy().astype(np.float32)

            gt_o = gt_o * 255.
            gt_o[gt_o < 0] = 0
            gt_o[gt_o > 255.] = 255.
            gt_o = gt_o.astype('uint8')

            gt = gt * 255.
            gt[gt < 0] = 0
            gt[gt > 255.] = 255.
            gt = gt.astype('uint8')

            gt_o = np.transpose(gt_o, [1, 2, 0])
            if gray_image:
                gt_o = Image.fromarray(gt_o)
                gt_o = gt_o.convert('L')
                gt_o = np.asarray(gt_o)

            psnr_predicted = PSNR(gt, gt_o, shave_border=0)
            avg_psnr_predicted += psnr_predicted

            ssim_predicted = SSIM(gt, gt_o, shave_border=0)
            avg_ssim_predicted += ssim_predicted

            name = image_name.split('/')[-1]
            name = name.split('.')[0]+'.png'

            gt_o = Image.fromarray(gt_o)
            gt_o.save(save_path + '/' + name)

        print("Dataset=", dataset)
        print("PSNR_predicted=", avg_psnr_predicted / len(image_list))
        print("SSIM predicted=", avg_ssim_predicted / len(image_list))
        print("It takes average {}s for processing".format(avg_elapsed_time / len(image_list)))

        results[dataset] = [avg_psnr_predicted / len(image_list)]
        results[dataset].append(avg_ssim_predicted / len(image_list))
    pd_results = pd.DataFrame(results, index=['PSNR', 'SSIM'], columns=datasets)
    pd_results.to_csv(result_path + '/' + 'result.csv', sep=',')