text:
def master_creation(primary_data_entry, secondary_data_entry, master_entry):

	import numpy as np
	import os
	from astropy.io import fits

	im_total = np.zeros((1,1))
	mean_prime = 0
	mean_secondary = 0
	mean_pair = [mean_prime, mean_secondary]
	data_entry = [primary_data_entry, secondary_data_entry]

	stage = 0

	for stage in range(0,2):
		if os.path.isfile(data_entry[stage].data_filedata):
			im_fit = fits.open(data_entry[stage].data_filedata)
			im_data = im_fit[0].data
			mean_pair[stage] = im_data
		else:
			counter = 0
			for root, dirs, files in os.walk(data_entry[stage].data_filedata):
				for filename in files:
					if filename.endswith(".fits"):
						if filename.startswith(str(data_entry[stage].data_type)):
							full_filename = os.path.join(root, filename)
							counter += 1
							im_fit = fits.open(full_filename)
							im_data = im_fit[0].data
							im_total = np.add(im_total, im_data)
			if counter == 0:
				print("Counter zero!")
				return(1)
			else:
				mean_pair[stage] = np.divide(im_total, counter)
		stage += 1

	master_im = np.subtract(mean_pair[0], mean_pair[1])
	hdu = fits.PrimaryHDU()
	hdu.data = master_im

	file_name = "Master_"+str(primary_data_entry.data_type)+".fits"

	master_entry.set_master_filename(file_name)
	hdu.writeto(file_name, overwrite=True)

	return(0)

max_stars_repo_name: cjduffy/HighRes

max_issues_repo_name: cjduffy/HighRes

max_forks_repo_name: cjduffy/HighRes

reason: import numpy,from astropy


text:
"""Mixin helper to create DeepImageJ compatible metadata."""
import datetime
import os
from functools import reduce

import numpy as np
from tifffile import imsave as tifffile_imsave

from ...common import distributed

def serialize_value(value):
    """
    Serialize a value in an DeepImageJ XML compatible manner.

    :param value:
    :return:
    """
    if isinstance(value, bool):
        return str(value).lower()
    else:
        return str(value)

def serialize_xml(the_dict):
    """
    Serialize a dictionary into a XML structure.

    :param the_dict:
    :return:
    """

    def _inner(inner_dict, level=0):
        return ''.join(
            reduce(
                lambda a, b: a + b,
                (
                    (
                        [('\t' * level) + '<%s>' % (key,) + '\n']
                        + [_inner(value, level + 1)]
                        + [('\t' * level) + '</%s>' % (key,) + '\n']
                    )
                    if isinstance(value, dict)
                    else (
                        [
                            ('\t' * level)
                            + '<%s>%s</%s>'
                            % (
                                key,
                                serialize_value(value),
                                key,
                            )
                            + '\n'
                        ]
                    )
                    for key, value in inner_dict.items()
                ),
            )
        )

    return '<?xml version="1.0" encoding="UTF-8" standalone="no"?>\n' + _inner(the_dict)

def write_to(file_name, data):
    """
    Write text to a file.

    :param file_name:
    :param data:
    :return:
    """
    with open(file_name, 'w+') as fp:
        fp.write(data)

class DeepImageJCompatibilityMixin:
    """Mixin to give DeepImageJ metadata writing."""

    def save_model(self):  # noqa: D102
        if not distributed.is_rank_zero():  # only rank zero may save
            return

        super().save_model()

        # signature_to_export = self.signatures[self.PREDICTION_SIGNATURE]

        # name_input = signature_to_export.input_signature[0].name
        # name_output = signature_to_export.get_concrete_function(
        # ).outputs[0].name  # no?

        now = datetime.datetime.now().isoformat()
        name = os.path.basename(self.model_path)
        base = {
            'Model': {
                'ModelInformation': {
                    'Name': name,
                    'Author': 'A JUNN User',
                    'URL': 'https://modsim.github.io/junn',
                    'Credit': 'A JUNN User',
                    'Version': 1,
                    'Date': now,
                    'Reference': 'Personal communication.',
                },
                'ModelTest': {  # dummy values
                    'InputSize': '1024x1024',
                    'OutputSize': '1024x1024',
                    'MemoryPeak': '1024.0 Mb',
                    'Runtime': '1.0 s',
                    'PixelSize': '1.0µmx1.0µm',
                },
                'ModelCharacteristics': {
                    'ModelTag': 'tf.saved_model.tag_constants.SERVING',
                    'SignatureDefinition': 'predict',
                    'InputTensorDimensions': ',-1,-1,-1,',
                    'NumberOfInputs': 1,
                    'InputNames0': 'image',
                    'InputOrganization0': 'HWC',
                    'NumberOfOutputs': 1,
                    'OutputNames0': 'output_0',
                    'OutputOrganization0': 'HWC',
                    'Channels': 1,  # does this have to be dynamic?
                    'FixedPatch': False,
                    'MinimumSize': 1,
                    'PatchSize': 1024,
                    'FixedPadding': False,
                    'Padding': 1,
                    'PreprocessingFile': 'preprocessing.txt',
                    'PostprocessingFile': 'postprocessing.txt',
                    'slices': 1,
                },
            }
        }

        configuration = base  # update it afterwards

        def p(file_name):
            return os.path.join(self.model_path, file_name)

        # write necessary files
        write_to(p('config.xml'), serialize_xml(configuration))

        write_to(p('preprocessing.txt'), 'run("32-bit");')

        write_to(p('postprocessing.txt'), '')

        tifffile_imsave(
            p('exampleImage.tiff'),
            np.zeros(
                (
                    32,
                    32,
                ),
                dtype=np.float32,
            ),
        )

        tifffile_imsave(
            p('resultImage.tiff'),
            np.zeros(
                (
                    32,
                    32,
                ),
                dtype=np.float32,
            ),
        )

ext: py

max_stars_repo_name: modsim/junn

max_issues_repo_name: modsim/junn

max_forks_repo_name: modsim/junn

reason: import numpy


text:
import time
import numpy as np
import math

import torch

from torch import nn
from torch.nn import functional as F
from torchvision.models import resnet
from torch.nn.modules.batchnorm import _BatchNorm

from det3d.torchie.cnn import constant_init, kaiming_init, xavier_init
from det3d.torchie.trainer import load_checkpoint
from det3d.models.utils import Empty, GroupNorm, Sequential
from det3d.models.utils import change_default_args

from .. import builder
from ..registry import NECKS
from ..utils import build_norm_layer

@NECKS.register_module
class RPN(nn.Module):
    def __init__(
        self,
        layer_nums,
        ds_layer_strides,
        ds_num_filters,
        us_layer_strides,
        us_num_filters,
        num_input_features,
        norm_cfg=None,
        name="rpn",
        logger=None,
        **kwargs
    ):
        super(RPN, self).__init__()
        self._layer_strides = ds_layer_strides
        self._num_filters = ds_num_filters
        self._layer_nums = layer_nums
        self._upsample_strides = us_layer_strides
        self._num_upsample_filters = us_num_filters
        self._num_input_features = num_input_features

        if norm_cfg is None:
            norm_cfg = dict(type="BN", eps=1e-3, momentum=0.01)
        self._norm_cfg = norm_cfg

        assert len(self._layer_strides) == len(self._layer_nums)
        assert len(self._num_filters) == len(self._layer_nums)
        assert len(self._num_upsample_filters) == len(self._upsample_strides)

        self._upsample_start_idx = len(self._layer_nums) - len(self._upsample_strides)

        must_equal_list = []
        for i in range(len(self._upsample_strides)):
            # print(upsample_strides[i])
            must_equal_list.append(
                self._upsample_strides[i]
                / np.prod(self._layer_strides[: i + self._upsample_start_idx + 1])
            )

        for val in must_equal_list:
            assert val == must_equal_list[0]

        in_filters = [self._num_input_features, *self._num_filters[:-1]]
        blocks = []
        deblocks = []

        for i, layer_num in enumerate(self._layer_nums):
            block, num_out_filters = self._make_layer(
                in_filters[i],
                self._num_filters[i],
                layer_num,
                stride=self._layer_strides[i],
            )
            blocks.append(block)
            if i - self._upsample_start_idx >= 0:
                stride = (self._upsample_strides[i - self._upsample_start_idx])
                if stride > 1:
                    deblock = Sequential(
                        nn.ConvTranspose2d(
                            num_out_filters,
                            self._num_upsample_filters[i - self._upsample_start_idx],
                            stride,
                            stride=stride,
                            bias=False,
                        ),
                        build_norm_layer(
                            self._norm_cfg,
                            self._num_upsample_filters[i - self._upsample_start_idx],
                        )[1],
                        nn.ReLU(),
                    )
                else:
                    stride = np.round(1 / stride).astype(np.int64)
                    deblock = Sequential(
                        nn.Conv2d(
                            num_out_filters,
                            self._num_upsample_filters[i - self._upsample_start_idx],
                            stride,
                            stride=stride,
                            bias=False,
                        ),
                        build_norm_layer(
                            self._norm_cfg,
                            self._num_upsample_filters[i - self._upsample_start_idx],
                        )[1],
                        nn.ReLU(),
                    )
                deblocks.append(deblock)
        self.blocks = nn.ModuleList(blocks)
        self.deblocks = nn.ModuleList(deblocks)

        logger.info("Finish RPN Initialization")

    @property
    def downsample_factor(self):
        factor = np.prod(self._layer_strides)
        if len(self._upsample_strides) > 0:
            factor /= self._upsample_strides[-1]
        return factor

    def _make_layer(self, inplanes, planes, num_blocks, stride=1):

        block = Sequential(
            nn.ZeroPad2d(1),
            nn.Conv2d(inplanes, planes, 3, stride=stride, bias=False),
            build_norm_layer(self._norm_cfg, planes)[1],
            nn.ReLU(),
        )

        for j in range(num_blocks):
            block.add(nn.Conv2d(planes, planes, 3, padding=1, bias=False))
            block.add(
                build_norm_layer(self._norm_cfg, planes)[1],
            )
            block.add(nn.ReLU())

        return block, planes

    # default init_weights for conv(msra) and norm in ConvModule
    def init_weights(self):
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                xavier_init(m, distribution="uniform")

    def forward(self, x):
        ups = []
        for i in range(len(self.blocks)):
            x = self.blocks[i](x)
            if i - self._upsample_start_idx >= 0:
                ups.append(self.deblocks[i - self._upsample_start_idx](x))
        if len(ups) > 0:
            x = torch.cat(ups, dim=1)

        return x

@NECKS.register_module
class PointModule(nn.Module):
    def __init__(
        self,
        num_input_features,
        layers=[1024, 128,],
        norm_cfg=None,
        name="rpn",
        logger=None,
        **kwargs
    ):
        super(PointModule, self).__init__()

        if norm_cfg is None:
            norm_cfg = dict(type="BN", eps=1e-3, momentum=0.01)
        self._norm_cfg = norm_cfg

        blocks = [
            nn.Conv2d(num_input_features, layers[0], 1, bias=False),
            build_norm_layer(self._norm_cfg, layers[0])[1],
            nn.ReLU(),
            nn.Conv2d(layers[0], layers[1], 1, bias=False),
            build_norm_layer(self._norm_cfg, layers[1])[1],
            nn.ReLU(),
        ]
        self.pn = nn.ModuleList(blocks)
        self.out = nn.MaxPool1d(3, stride=1, padding=1)

    def forward(self, x):
        x = x.flatten(1, -1)
        x = x.view(x.shape[0], x.shape[1], 1, 1)

        for l in self.pn:
            x = l(x)

        x = x.view(x.shape[0], 1, x.shape[1])
        x = self.out(x).view(x.shape[0], x.shape[2], 1, 1)

        return x


text:
# Copyright (c) 2018, Ionel Gog
import csv

from absl import app, flags
from matplotlib.patches import Patch

import matplotlib
matplotlib.use('agg')
import matplotlib.pyplot as plt

import numpy as np
import pandas as pd
import seaborn as sns

from utils import plot_box_and_whiskers, plot_cdf, plot_timeline, generate_ticks, set_paper_rcs
from pylot_utils import ProfileEvent, ProfileEvents, read_end_to_end_runtimes, \
    converter, get_timestamps_with_obstacles

FLAGS = flags.FLAGS
flags.DEFINE_enum('graph_type', 'cdf', ['cdf', 'box_and_whiskers', 'timeline'],
                  'Type of graph to plot')
flags.DEFINE_list('events', [], 'Events to select')
flags.DEFINE_bool('time_to_decision', False, 'True to plot time to decision')
flags.DEFINE_bool('lateral_acceleration', False,
                  'True to plot lateral acceleration')
flags.DEFINE_bool('longitudinal_acceleration', False,
                  'True to plot longitudinal acceleration')
flags.DEFINE_bool('lateral_jerk', False, 'True to plot lateral jerk')
flags.DEFINE_bool('longitudinal_jerk', False, 'True to plot longitudinal jerk')
flags.DEFINE_bool('obstacle_distance', False,
                  'True to plot distance to obstacle')
flags.DEFINE_bool('lane_deviation', False,
                  'True to plot cumulative deviation from lane center')
flags.DEFINE_bool('lane_invasion', False,
                  'True to plot cdf of time in lane invasion')
flags.DEFINE_bool('crosstrack_error', False, 'True to plot crossrack error')
flags.DEFINE_bool('heading_error', False, 'True to plot heading error')
flags.DEFINE_bool('small_paper_mode', False, 'Adjusts the size of the plots.')
flags.DEFINE_bool('stretched', False, 'Adjusts the size of the plots.')
flags.DEFINE_bool('paper_mode', False, 'Adjusts the size of the plots.')
flags.DEFINE_bool('slide_mode', False, 'Adjusts the size of the plots.')
flags.DEFINE_bool('poster_mode', False, 'Adjusts the size of the plots.')
flags.DEFINE_string('file_name', 'pylot', 'File to write plot to.')
flags.DEFINE_string('file_format', 'png', 'File type of the output plot.')
flags.DEFINE_string('xlabel', '', 'The xlabel to give the graph.')
flags.DEFINE_string('ylabel', '', 'The ylabel to give the graph.')
flags.DEFINE_list('labels', [], 'Labels of plotted data.')
flags.DEFINE_list('log_file_names', '', 'Pylot log files')
flags.DEFINE_list('csv_file_names', '', 'Pylot csv files')
flags.DEFINE_list('profile_file_names', '', 'Pylot profile files')
flags.DEFINE_integer('ignore_first_sim_time_ms', 1000,
                     'Ignore data from the first simulation time ms.')
flags.DEFINE_float('min_x_val', None, 'x axis starts from.')
flags.DEFINE_float('max_x_val', None, 'x axis goes up to.')
flags.DEFINE_float('min_y_val', None, 'y axis starts from.')
flags.DEFINE_float('max_y_val', None, 'y axis goes up to.')
flags.DEFINE_integer('x_ticks_increment', None,
                     'x axis increment. None if default should be used')
flags.DEFINE_integer('y_ticks_increment', None,
                     'y axis increment. None if default should be used')
flags.DEFINE_bool('legend_outside_graph', False,
                  'True to put legend outside box')
flags.DEFINE_bool('filter_only_timestamps_with_obstacles', False,
                  "True to only plot data for timestamps with obstacles")

def read_jerk_stats(csv_file_path):
    first_sim_time = None
    csv_file = open(csv_file_path)
    csv_reader = csv.reader(csv_file)
    lateral_jerk = []
    lateral_jerk_times = []
    longitudinal_jerk = []
    longitudinal_jerk_times = []
    for row in csv_reader:
        sim_time = int(row[1])
        if first_sim_time is None:
            first_sim_time = sim_time
        sim_time -= first_sim_time + FLAGS.ignore_first_sim_time_ms
        if (row[2] == 'jerk' and sim_time >= 0):
            jerk_type = row[3]
            if jerk_type == 'lateral':
                jerk = float(row[4])
                lateral_jerk.append(jerk)
                lateral_jerk_times.append(sim_time)
            elif jerk_type == 'longitudinal':
                jerk = float(row[4])
                longitudinal_jerk.append(jerk)
                longitudinal_jerk_times.append(sim_time)
            else:
                raise ValueError('Unexpected value in {}'.format(row))
    return (lateral_jerk, lateral_jerk_times, longitudinal_jerk,
            longitudinal_jerk_times)

def read_acceleration_stats(csv_file_path):
    first_sim_time = None
    csv_file = open(csv_file_path)
    csv_reader = csv.reader(csv_file)
    lateral_acceleration = []
    lateral_acc_times = []
    longitudinal_acceleration = []
    longitudinal_acc_times = []
    for row in csv_reader:
        sim_time = int(row[1])
        if not first_sim_time:
            first_sim_time = sim_time
        sim_time -= first_sim_time + FLAGS.ignore_first_sim_time_ms
        if (row[2] == 'acceleration' and sim_time >= 0):
            acc_type = row[3]
            if acc_type == 'lateral':
                acceleration = float(row[4])
                lateral_acceleration.append(acceleration)
                lateral_acc_times.append(sim_time)
            elif acc_type == 'longitudinal':
                acceleration = float(row[4])
                longitudinal_acceleration.append(acceleration)
                longitudinal_acc_times.append(sim_time)
            else:
                raise ValueError('Unexpected value in {}'.format(row))
    return (lateral_acceleration, lateral_acc_times, longitudinal_acceleration,
            longitudinal_acc_times)

def read_infraction_stats(csv_file_path):
    first_sim_time = None
    csv_file = open(csv_file_path)
    csv_reader = csv.reader(csv_file)
    for row in csv_reader:
        proc_time = int(row[0])
        sim_time = int(row[1])
        if not first_sim_time:
            first_sim_time = sim_time
        sim_time -= first_sim_time + FLAGS.ignore_first_sim_time_ms
        event_name = row[2]
        if event_name == 'collision':
            actor_type = row[3]
            intensity = float(row[4])
            pass
        elif event_name == 'invasion':
            # red_light
            # sidewalk
            # lane
            invasion_type = row[3]
    # TODO: Implement.

def read_control_stats(csv_file_path):
    first_sim_time = None
    csv_file = open(csv_file_path)
    csv_reader = csv.reader(csv_file)
    sim_times = []
    crosstrack_err = []
    heading_err = []
    for row in csv_reader:
        sim_time = int(row[1])
        if not first_sim_time:
            first_sim_time = sim_time
        sim_time -= first_sim_time + FLAGS.ignore_first_sim_time_ms
        if (row[2] == 'control' and sim_time >= 0):
            sim_times.append(sim_time)
            crosstrack_err.append(float(row[3]))
            heading_err.append(float(row[4]))
    return (sim_times, crosstrack_err, heading_err)

def read_prediction_stats(csv_file_path):
    first_sim_time = None
    csv_file = open(csv_file_path)
    csv_reader = csv.reader(csv_file)
    sim_times = []
    msd = []
    ade = []
    fde = []
    person_msd = []
    person_ade = []
    person_fde = []
    vehicle_msd = []
    vehicle_ade = []
    vehicle_fde = []
    for row in csv_reader:
        sim_time = int(row[1])
        if not first_sim_time:
            first_sim_time = sim_time
        sim_time -= first_sim_time + FLAGS.ignore_first_sim_time_ms
        if (row[2] == 'prediction' and sim_time >= 0):
            sim_times.append(sim_time)
            if row[3] == 'MSD':
                msd.append(float(row[4]))
            elif row[3] == 'ADE':
                ade.append(float(row[4]))
            elif row[3] == 'FDE':
                fde.append(float(row[4]))
            elif row[3] == 'person-MSD':
                person_msd.append(float(row[4]))
            elif row[3] == 'person-ADE':
                person_ade.append(float(row[4]))
            elif row[3] == 'person-FDE':
                person_fde.append(float(row[4]))
            elif row[3] == 'vehicle-MSD':
                vehicle_msd.append(float(row[4]))
            elif row[3] == 'vehicle-ADE':
                vehicle_ade.append(float(row[4]))
            elif row[3] == 'vehicle-FDE':
                vehicle_fde.append(float(row[4]))
            else:
                raise ValueError('Unexpected row {}'.format(row))
    return (sim_times, msd, ade, fde, person_msd, person_ade, person_fde,
            vehicle_msd, vehicle_ade, vehicle_fde)

def get_obstacle_distances(filename, ignore_first_sim_time):
    df = pd.read_csv(
        filename,
        names=["timestamp", "ms", "log_label", "label_info", "label_value"])
    df = df.dropna()
    df['label_value'] = df['label_value'].str.replace(" ", ", ")
    df['label_value'] = df['label_value'].apply(converter)

    obstacles = df[df['log_label'] == 'obstacle']
    obstacles = obstacles.set_index('ms')
    pose = df[df['log_label'] == 'pose']

    timestamps = []
    distances = []
    first_timestamp = df["ms"].min()
    for t, p in pose[["ms", "label_value"]].values:
        adjusted_t = t - first_timestamp
        if t not in obstacles.index or adjusted_t <= ignore_first_sim_time:
            continue
        obs = obstacles.loc[t]['label_value']
        if isinstance(obs, list):
            obs = [obs]
        else:
            obs = obs.values
        min_dist = float('inf')
        for o in obs:
            dist = np.linalg.norm(np.array(p) - np.array(o))
            if dist > 0:
                min_dist = min(min_dist, dist)
        if min_dist != float('inf'):
            timestamps.append(adjusted_t - ignore_first_sim_time)
            distances.append(min_dist)

    return timestamps, distances

def get_lane_deviation(filename, lane_center, ignore_first_sim_time):
    df = pd.read_csv(
        filename,
        names=["timestamp", "ms", "log_label", "label_info", "label_value"])
    df = df.dropna()
    df['label_value'] = df['label_value'].str.replace(" ", ", ")
    df['label_value'] = df['label_value'].apply(converter)
    first_timestamp = df["ms"].min()
    df["ms"] = df["ms"] - first_timestamp
    df = df[df["ms"] > ignore_first_sim_time]
    first_timestamp = df["ms"].min()

    poses = df[df['log_label'] == 'pose']
    timestamps = (poses["ms"] - first_timestamp).to_numpy()
    poses = poses['label_value'].to_numpy()
    poses = np.array([np.array(x) for x in poses])
    y = poses[:-2, 1]
    deviation = np.cumsum(np.abs(y - lane_center))
    return timestamps[:-2], deviation

def get_lane_invasions(filename, lane_boundary, ignore_first_sim_time):
    df = pd.read_csv(
        filename,
        names=["timestamp", "ms", "log_label", "label_info", "label_value"])
    df = df.dropna()
    collision = len(df[df["log_label"] == "collision"]) > 0
    df['label_value'] = df['label_value'].str.replace(" ", ", ")
    df['label_value'] = df['label_value'].apply(converter)
    first_timestamp = df["ms"].min()
    df["ms"] = df["ms"] - first_timestamp
    df = df[df["ms"] > ignore_first_sim_time]
    first_timestamp = df["ms"].min()

    poses = df[df['log_label'] == 'pose']
    p = poses['label_value'].to_numpy()
    p = np.array([np.array(x) for x in p])
    y = p[:-2, 1]

    timestamps = poses.iloc[:-2][y > lane_boundary]["ms"]
    time_spent = 25
    if len(timestamps) > 1:
        time_spent = timestamps.iloc[-1] - timestamps.iloc[0]

    if collision:
        color = 'r'
    else:
        color = 'g'

    return time_spent, color

def main(argv):
    all_profile_events = [
        ProfileEvents(profile_file_name)
        for profile_file_name in FLAGS.profile_file_names
    ]
    if FLAGS.time_to_decision:
        all_ts_w_obstacles = []
        for csv_file_name in FLAGS.csv_file_names:
            ts = get_timestamps_with_obstacles(csv_file_name,
                                               obstacle_distance_threshold=20)
            all_ts_w_obstacles.append(ts)
        all_sim_times = []
        all_end_to_end_runtimes = []
        for ts, csv_file_name in zip(all_ts_w_obstacles, FLAGS.csv_file_names):
            if not FLAGS.filter_only_timestamps_with_obstacles:
                ts = None
            (sim_times, end_to_end_runtimes) = read_end_to_end_runtimes(
                csv_file_name, unit='ms', timestamps_with_obstacles=ts)
            all_sim_times.append(sim_times)
            all_end_to_end_runtimes.append(end_to_end_runtimes)
        if FLAGS.graph_type == 'cdf':
            plot_cdf(FLAGS,
                     FLAGS.file_name,
                     all_end_to_end_runtimes,
                     r'End-to-end runtime [$ms$]',
                     'CDF of end-to-end runtime',
                     FLAGS.labels,
                     show_legend=True,
                     legend_outside_box=FLAGS.legend_outside_graph,
                     x_ticks_increment=FLAGS.x_ticks_increment)
        elif FLAGS.graph_type == 'timeline':
            plot_timeline(FLAGS,
                          FLAGS.file_name,
                          all_sim_times,
                          all_end_to_end_runtimes,
                          r'Simulation Time [$s$]',
                          'End-to-end runtime',
                          FLAGS.labels,
                          show_legend=True,
                          legend_outside_box=FLAGS.legend_outside_graph,
                          cut_x_min=FLAGS.min_x_val,
                          cut_x_max=FLAGS.max_x_val,
                          x_ticks_increment=FLAGS.x_ticks_increment,
                          y_ticks_increment=FLAGS.y_ticks_increment,
                          cut_y_min=FLAGS.min_y_val,
                          cut_y_max=FLAGS.max_y_val,
                          scale_down_x_ticks=1000)

    if FLAGS.lateral_acceleration or FLAGS.longitudinal_acceleration:
        all_lateral_acc = []
        all_lateral_acc_times = []
        all_longitudinal_acc = []
        all_longitudinal_acc_times = []
        for csv_file_name in FLAGS.csv_file_names:
            (lateral_acc, lateral_acc_times, longitudinal_acc,
             longitudinal_acc_times) = read_acceleration_stats(csv_file_name)
            all_lateral_acc.append(lateral_acc)
            all_lateral_acc_times.append(lateral_acc_times)
            all_longitudinal_acc.append(longitudinal_acc)
            all_longitudinal_acc_times.append(longitudinal_acc_times)
        if FLAGS.lateral_acceleration:
            plot_timeline(FLAGS,
                          FLAGS.file_name,
                          all_lateral_acc_times,
                          all_lateral_acc,
                          r'Simulation Time [$s$]',
                          r'Lateral acceleration [$\frac{m}{s^2}$]',
                          FLAGS.labels,
                          show_legend=True,
                          legend_outside_box=FLAGS.legend_outside_graph,
                          cut_x_min=FLAGS.min_x_val,
                          cut_x_max=FLAGS.max_x_val,
                          x_ticks_increment=FLAGS.x_ticks_increment,
                          y_ticks_increment=FLAGS.y_ticks_increment,
                          cut_y_min=FLAGS.min_y_val,
                          cut_y_max=FLAGS.max_y_val,
                          line_width=1.0,
                          scale_down_x_ticks=1000)
        if FLAGS.longitudinal_acceleration:
            plot_timeline(FLAGS,
                          FLAGS.file_name,
                          all_longitudinal_acc_times,
                          all_longitudinal_acc,
                          r'Simulation Time [$s$]',
                          r'Longitudinal acceleration [$\frac{m}{s^2}$]',
                          FLAGS.labels,
                          show_legend=True,
                          legend_outside_box=FLAGS.legend_outside_graph,
                          cut_x_min=FLAGS.min_x_val,
                          cut_x_max=FLAGS.max_x_val,
                          x_ticks_increment=FLAGS.x_ticks_increment,
                          y_ticks_increment=FLAGS.y_ticks_increment,
                          cut_y_min=FLAGS.min_y_val,
                          cut_y_max=FLAGS.max_y_val,
                          line_width=1.0,
                          scale_down_x_ticks=1000)

    if FLAGS.lateral_jerk or FLAGS.longitudinal_jerk:
        all_lateral_jerk = []
        all_lateral_jerk_times = []
        all_longitudinal_jerk = []
        all_longitudinal_jerk_times = []
        for csv_file_name in FLAGS.csv_file_names:
            (lateral_jerk, lateral_jerk_times, longitudinal_jerk,
             longitudinal_jerk_times) = read_jerk_stats(csv_file_name)
            all_lateral_jerk.append(lateral_jerk)
            all_lateral_jerk_times.append(lateral_jerk_times)
            all_longitudinal_jerk.append(longitudinal_jerk)
            all_longitudinal_jerk_times.append(longitudinal_jerk_times)

        if FLAGS.lateral_jerk:
            import brewer2mpl
            bmap = brewer2mpl.get_map('Set2', 'qualitative', 7)
            colors = bmap.mpl_colors
            jerk_data = []
            set_paper_rcs()
            plt.figure(figsize=(2.4, 1.66))
            print(all_lateral_jerk)
            for i in range(3):
                jerk_data.append(
                    pd.DataFrame({
                        'jerk': [abs(x) for x in all_lateral_jerk[i]],
                        'deadline': FLAGS.labels[i]
                    }))
            data = pd.concat(jerk_data)
            ax = sns.boxplot(
                y='jerk',
                x='deadline',
                data=data,
                width=0.7,
                showfliers=True,
                color=colors[6],
                fliersize=0.9,
                whis=(5, 95),
                saturation=1,
            )
            plt.ylim(0, 141)
            plt.yticks([x for x in range(0, 121, 40)],
                       [str(x) for x in range(0, 121, 40)])
            plt.ylabel("Comfort\nAbs. Lateral Jerk [$m/s^3$]")
            plt.xlabel(r"Planning runtime [ms]")

            #showfliers=True,
            #fliersize=0.7)
            plt.savefig('planning-absolute-jerk.{}'.format(FLAGS.file_format),
                        bbox_inches='tight')

            plot_timeline(FLAGS,
                          FLAGS.file_name,
                          all_lateral_jerk_times,
                          all_lateral_jerk,
                          r'Simulation Time [$s$]',
                          r'Lateral jerk [$m/s^3$]',
                          FLAGS.labels,
                          show_legend=True,
                          legend_outside_box=FLAGS.legend_outside_graph,
                          cut_x_min=FLAGS.min_x_val,
                          cut_x_max=FLAGS.max_x_val,
                          colors=colors,
                          x_ticks_increment=FLAGS.x_ticks_increment,
                          y_ticks_increment=FLAGS.y_ticks_increment,
                          cut_y_min=FLAGS.min_y_val,
                          cut_y_max=FLAGS.max_y_val,
                          line_width=0.5,
                          scale_down_x_ticks=1000)
        if FLAGS.longitudinal_jerk:
            plot_timeline(FLAGS,
                          FLAGS.file_name,
                          all_longitudinal_jerk_times,
                          all_longitudinal_jerk,
                          r'Simulation Time [$s$]',
                          r'Longitudinal jerk [$\frac{m}{s^3}$]',
                          FLAGS.labels,
                          show_legend=True,
                          legend_outside_box=FLAGS.legend_outside_graph,
                          cut_x_min=FLAGS.min_x_val,
                          cut_x_max=FLAGS.max_x_val,
                          x_ticks_increment=FLAGS.x_ticks_increment,
                          y_ticks_increment=FLAGS.y_ticks_increment,
                          cut_y_min=FLAGS.min_y_val,
                          cut_y_max=FLAGS.max_y_val,
                          line_width=1.0,
                          scale_down_x_ticks=1000)

    if FLAGS.obstacle_distance:
        all_ts = []
        all_dists = []
        for csv_file_name in FLAGS.csv_file_names:
            ts, dist = get_obstacle_distances(csv_file_name,
                                              FLAGS.ignore_first_sim_time_ms)
            all_ts.append(ts)
            all_dists.append(dist)
        plot_timeline(FLAGS,
                      FLAGS.file_name,
                      all_ts,
                      all_dists,
                      r'Simulation Time [$s$]',
                      r'Distance to closest obstacle [$m$]',
                      FLAGS.labels,
                      show_legend=True,
                      legend_outside_box=FLAGS.legend_outside_graph,
                      cut_x_min=FLAGS.min_x_val,
                      cut_x_max=FLAGS.max_x_val,
                      x_ticks_increment=FLAGS.x_ticks_increment,
                      y_ticks_increment=FLAGS.y_ticks_increment,
                      cut_y_min=FLAGS.min_y_val,
                      cut_y_max=FLAGS.max_y_val,
                      line_width=1.0,
                      scale_down_x_ticks=1000)

    if FLAGS.lane_deviation:
        all_ts = []
        all_deviation = []
        for csv_file_name in FLAGS.csv_file_names:
            ts, deviation = get_lane_deviation(csv_file_name, 326.61,
                                               FLAGS.ignore_first_sim_time_ms)
            all_ts.append(ts)
            all_deviation.append(deviation)
        plot_timeline(FLAGS,
                      FLAGS.file_name,
                      all_ts,
                      all_deviation,
                      r'Simulation Time [$s$]',
                      r'Cumulative Deviation from Lane Center [$m$]',
                      FLAGS.labels,
                      show_legend=True,
                      legend_outside_box=FLAGS.legend_outside_graph,
                      cut_x_min=FLAGS.min_x_val,
                      cut_x_max=FLAGS.max_x_val,
                      x_ticks_increment=FLAGS.x_ticks_increment,
                      y_ticks_increment=FLAGS.y_ticks_increment,
                      cut_y_min=FLAGS.min_y_val,
                      cut_y_max=FLAGS.max_y_val,
                      line_width=1.0,
                      scale_down_x_ticks=1000)

    if FLAGS.lane_invasion:
        all_invasion = []
        all_collisions = []
        for csv_file_name in FLAGS.csv_file_names:
            invasion, collision = get_lane_invasions(
                csv_file_name, 327, FLAGS.ignore_first_sim_time_ms)
            all_invasion.append(invasion)
            all_collisions.append(collision)
        plt.barh(y=[i for i in range(len(all_invasion))],
                 width=all_invasion,
                 height=0.5,
                 color=all_collisions,
                 tick_label=FLAGS.labels)
        plt.xlabel(r"Total Time in Oncoming Lane [$s$]")
        plt.xlim(FLAGS.min_x_val, FLAGS.max_x_val)
        ts, tls = generate_ticks(FLAGS.min_x_val, FLAGS.max_x_val,
                                 FLAGS.x_ticks_increment, 1000)
        plt.xticks(ts, tls)
        ax = plt.gca()
        legend_elements = [
            Patch(facecolor='red', edgecolor='r', label='Failure'),
            Patch(facecolor='green', edgecolor='g', label='Success')
        ]
        ax.legend(handles=legend_elements)
        plt.savefig("{}.{}".format(FLAGS.file_name, FLAGS.file_format),
                    format=FLAGS.file_format,
                    bbox_inches="tight")

    if FLAGS.crosstrack_error or FLAGS.heading_error:
        for csv_file_name in FLAGS.csv_file_names:
            (sim_times, crosstrack_err,
             heading_err) = read_control_stats(csv_file_name)
        raise NotImplementedError

    if FLAGS.graph_type == 'cdf' and len(FLAGS.events) > 0:
        # assert len(FLAGS.labels) == len(FLAGS.events) * len(
        #     all_profile_events), 'All events must be labelled'
        all_ts_w_obstacles = []
        for csv_file_name in FLAGS.csv_file_names:
            ts = get_timestamps_with_obstacles(csv_file_name,
                                               obstacle_distance_threshold=20)
            all_ts_w_obstacles.append(ts)
        all_runtimes = []
        for ts, profile_events in zip(all_ts_w_obstacles, all_profile_events):
            if not FLAGS.filter_only_timestamps_with_obstacles:
                ts = None
            for event_name in FLAGS.events:
                runtimes = profile_events.get_runtimes(
                    event_name, unit='us', timestamps_with_obstacles=ts)
                all_runtimes.append(runtimes)
        plot_cdf(FLAGS,
                 FLAGS.file_name,
                 all_runtimes,
                 FLAGS.xlabel,
                 FLAGS.ylabel,
                 FLAGS.labels,
                 show_legend=True,
                 legend_outside_box=FLAGS.legend_outside_graph,
                 x_ticks_increment=FLAGS.x_ticks_increment,
                 scale_x_axis=1000)
    elif FLAGS.graph_type == 'timeline' and len(FLAGS.events) > 0:
        assert len(FLAGS.labels) == len(FLAGS.events) * len(
            all_profile_events), 'All events must be labelled'
        all_sim_times = []
        all_runtimes = []
        for profile_events in all_profile_events:
            for event_name in FLAGS.events:
                sim_times, runtimes = profile_events.get_timeline(event_name,
                                                                  unit='ms')
            all_sim_times.append(sim_times)
            all_runtimes.append(runtimes)
        plot_timeline(FLAGS,
                      FLAGS.file_name,
                      all_sim_times,
                      all_runtimes,
                      FLAGS.xlabel,
                      FLAGS.ylabel,
                      FLAGS.labels,
                      show_legend=True,
                      legend_outside_box=FLAGS.legend_outside_graph,
                      cut_x_min=FLAGS.min_x_val,
                      cut_x_max=FLAGS.max_x_val,
                      x_ticks_increment=FLAGS.x_ticks_increment,
                      cut_y_min=FLAGS.min_y_val,
                      cut_y_max=FLAGS.max_y_val)

if __name__ == '__main__':
    app.run(main)

max_stars_repo_name: erdos-project/erdos-experiments

max_issues_repo_name: erdos-project/erdos-experiments

max_forks_repo_name: erdos-project/erdos-experiments


text:
# this file is used to create the initial array that will be used for displaying our effects
import fixedsizes as fixed
import numpy as np
import pandas as pd

#array containing the mapping of channel to position
hex_lp = np.array([[-1,1,7,-1],[3,0,4,5],[21,2,6,11],[20,22,10,8],[23,18,14,9],[17,16,12,15],[-1,19,13,-1]],dtype=int)

#map for a rotation of the hexagon by 60 degrees counterclockwise
shift_map = np.array([[0,7,11,3],[2,6,15,19],[1,10,14,23],[5,9,18,22],[4,13,17,26],[8,12,21,25],[24,16,20,27]],dtype=int)

#functions to apply rotation to a suiting array
def rotate_hex_by_one(array):
    if (array.shape!=(fixed.HEX_Y,fixed.HEX_X)):
        return array
    new_array = np.zeros((fixed.HEX_Y,fixed.HEX_X),dtype=int)
    for i in range(shift_map.size):
        np.put(new_array, i, array.flat[shift_map.flat[i]])
    array = new_array
    return array

def rotate_hex_by_n(array, n):
    for i in range(n):
        array = rotate_hex_by_one(array)
    return array

def initiate_array(raw_info):
    # ensure correct input
    if (raw_info.shape!=(5,)):
        return np.zeros((fixed.HEX_Y,fixed.HEX_X),dtype=int)
    # copy template array
    array = hex_lp.copy()
    # advance all contained channels along according to position in chain
    np.putmask(array, array>=0, array+raw_info[0]*fixed.TLC5947_CHANNELS)
    # rotate array to correct position
    array = rotate_hex_by_n(array, raw_info[2])
    array = conform_array_to_type(array, raw_info[1])
    return array

# cut arrays down to size according to type
def conform_array_to_type(array, type):
    if (array.shape!=(fixed.HEX_Y,fixed.HEX_X)) or (type==0):
        return array
    elif type==1:
        return array[4::1, :fixed.HEX_X]
    elif type==2:
        return array[0:4:1, :fixed.HEX_X]
    elif type==3:
        return array[:fixed.HEX_Y, 2::1]
    elif type==4:
        return array[:fixed.HEX_Y, 0:2:1]
    else:
        return array

# insert a smaller array into a larger one
def insert_at_position(dest_array, src_array, x, y):
    for i_y in range(np.size(src_array,0)):
        for i_x in range(np.size(src_array,1)):
            if (src_array[i_y, i_x]!=-1):
                dest_array[y+i_y, x+i_x] = src_array[i_y, i_x]
    return dest_array

def static_array_from_xlsx(file):
    # reading the controller placement info from .xlsx file
    # format: nr_of_controller type_of_hexagon rotation origin_x origin_y
    raw_excel = pd.read_excel(file, skiprows=0)
    controller_info = np.array(raw_excel)

    static_array = np.zeros((fixed.DISPLAY_Y,fixed.DISPLAY_X),dtype=int)

    for i in range(fixed.DRIVER_COUNT):
        hexagon = initiate_array(controller_info[i])
        static_array = insert_at_position(static_array, hexagon, controller_info[i, 3], controller_info[i, 4])

    #print(static_array)
    return static_array


text:
import numpy as np
import torch
from gym.spaces import Discrete
from torch.distributions import Categorical

from games.graphon_mfg import FiniteGraphonMeanFieldGame

class InvestmentGraphon(FiniteGraphonMeanFieldGame):
    """
    Models the Investment game.
    """

    def __init__(self, graphon = (lambda x,y: 1-max(x,y)), time_steps: int = 50, num_qualities: int = 10,
                 payoff_quality: float = 0.3, cost_investment: float = 2):
        self.graphon = graphon
        self.payoff_quality = payoff_quality
        self.cost_investment = cost_investment
        self.num_qualities = num_qualities

        def initial_state_distribution(x):
            return Categorical(probs=torch.tensor([1] + [0] * (num_qualities-1)))
        agent_observation_space = Discrete(num_qualities)
        agent_action_space = Discrete(2)
        super().__init__(agent_observation_space, agent_action_space, time_steps, initial_state_distribution, graphon)

    def transition_probs_g(self, t, x, u, g):
        if u == 0 or x[1] == (self.num_qualities - 1):
            probs = np.zeros(self.num_qualities)
            probs[x[1]] = 1
            return probs
        elif u == 1:
            probs = np.zeros(self.num_qualities)
            probs[x[1]] = (1 + x[1]) / self.num_qualities
            probs[x[1] + 1] = (self.num_qualities - x[1] - 1) / self.num_qualities
            return probs

    def reward_g(self, t, x, u, g):
        average_quality = g.evaluate_integral(t, lambda dy: dy[1])
        return self.payoff_quality * x[1] / (1 + average_quality) \
               - self.cost_investment * u

max_stars_repo_name: tudkcui/gmfg-learning

max_issues_repo_name: tudkcui/gmfg-learning

max_forks_repo_name: tudkcui/gmfg-learning


text:
# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
#
# NVIDIA CORPORATION and its licensors retain all intellectual property
# and proprietary rights in and to this software, related documentation
# and any modifications thereto.  Any use, reproduction, disclosure or
# distribution of this software and related documentation without an express
# license agreement from NVIDIA CORPORATION is strictly prohibited.

"""Generate images using pretrained network pickle."""
import glob
import os
import re
from typing import List, Optional

import click
import dnnlib
import numpy as np
import PIL.Image
import torch

import legacy
import base58

@click.command()
@click.pass_context
@click.option('--network', 'network_pkl', help='Network pickle filename', required=True)
@click.option('--count', 'total_count', type=int, default=32, help='Number of images to generate')
@click.option('--conf', 'min_confidence', type=float, default=-1.0, help='Minimum confidence of images to keep')
@click.option('--batch', 'batch_size', type=int, default=32, help='Batch size.')
@click.option('--zvec', 'zvec_infile', type=str, help='Path to b58 vector file.')
@click.option('--zperm', 'zvec_noise_perm', type=float, default=0.01, help='Amount of noise to permute zvec.')
@click.option('--trunc', 'truncation_psi', type=float, help='Truncation psi', default=1, show_default=True)
@click.option('--noise-mode', help='Noise mode', type=click.Choice(['const', 'random', 'none']), default='const', show_default=True)
@click.option('--projected-w', help='Projection result file', type=str, metavar='FILE')
@click.option('--outdir', help='Where to save the output images', type=str, required=True, metavar='DIR')
def generate_images(
    ctx: click.Context,
    total_count : int,
    min_confidence : float,
    network_pkl: str,
    batch_size : int,
    truncation_psi: float,
    noise_mode: str,
    outdir: str,
    projected_w: Optional[str],
    zvec_infile : str,
    zvec_noise_perm : float,
):
    print('Loading networks from "%s"...' % network_pkl)
    device = torch.device('cuda')
    with dnnlib.util.open_url(network_pkl) as f:
        loaded = legacy.load_network_pkl(f)
        G = loaded['G_ema'].to(device) # type: ignore
        D = loaded['D'].to(device) # type: ignore

    zvec_in = None
    if zvec_infile:
        zvec_bytes = base58.b58decode(open(zvec_infile, 'rt').read())
        zvec_in = np.fromstring(zvec_bytes, "<f4")
        zvec_in = torch.tensor(zvec_in).to(device)

    os.makedirs(outdir, exist_ok=True)

    label = torch.zeros([1, G.c_dim], device=device)

    while total_count > 0:
        z = torch.randn((batch_size, G.z_dim)).to(device)
        # z = z / torch.norm(z, 2, dim=-1, keepdim=True)
        if zvec_in is not None:
            z = z * zvec_noise_perm + zvec_in[None, ...]

        with torch.no_grad():
            images = G(z, label, truncation_psi=truncation_psi, noise_mode=noise_mode)
            logits = D(images, label)
            z = z.cpu().numpy()

        images = (images.permute(0, 2, 3, 1) * 127.5 + 128).clamp(0, 255).to(torch.uint8)

        logits = logits[:, 0]
        odds = torch.exp(logits).cpu().numpy()
        probs = 1.0 / (1.0 + odds)

        for i, prob in enumerate(probs):
            prob = float(prob)
            if prob < min_confidence:
                continue

            img, zvec = images[i], z[i]

            dircount = len(glob.glob(f"{outdir}/*.png"))
            outpath = f'{outdir}/img_{dircount:04d}.png'
            print(f"Saving: {outpath} (prob={prob*100:.02f}%)")
            PIL.Image.fromarray(img.cpu().numpy(), 'RGB').save(outpath)

            zvec_b58 = zvec.astype("<f4").tostring()
            zvec_b58 = base58.b58encode(zvec_b58)
            with open(outpath.replace('.png', '_zvec.b58'), 'wt') as fptr:
                fptr.write(zvec_b58.decode('utf-8'))

            total_count -= 1
            if total_count <= 0:
                break

    print("Done!")

#----------------------------------------------------------------------------

if __name__ == "__main__":
    generate_images() # pylint: disable=no-value-for-parameter

#----------------------------------------------------------------------------


text:
import numpy as np

from .mask import Mask
from ..data import Spectrum

class MaskEnds(Mask):
    """Masks the ends of a spectrum.

    This class, when called, creates a mask that masks the N pixels at each end of the given spectrum.
    """

    def __init__(self, npixels=10, *args, **kwargs):
        """Initializes a new mask masking the ends of a spectrum.

        Args:
            npixels: Number of pixels to mask at each end of the spectrum.
        """
        Mask.__init__(self, *args, **kwargs)
        self._npixels = npixels

        # npixels cannot be negative
        if npixels < 0:
            raise ValueError('Number of pixels to mask at each end of the spectrum cannot be negative.')

    def __call__(self, spectrum: Spectrum, filename: str = None) -> np.ndarray:
        """Creates a new mask for the given spectrum masking only the N pixels at each end.

        Args:
            spectrum: Spectrum to create mask for.
            filename: Name of file containing spectrum to create mask for (unused).

        Returns;
            Boolean array containing good pixel mask for given spectrum.
        """

        # create empty mask
        mask = np.ones((len(spectrum.wave)), dtype=np.bool)

        # if npixels is zero, just return mask
        if self._npixels == 0:
            return mask

        # mask ends and return mask
        mask[:self._npixels] = False
        mask[-self._npixels:] = False
        self.log.info('Masked %d pixels at both ends of the spectrum.', self._npixels)
        return mask

__all__ = ['MaskEnds']

max_stars_repo_name: thusser/spexxy

max_issues_repo_name: thusser/spexxy

max_forks_repo_name: thusser/spexxy


text:
import cv2
import numpy as np

patch_sizes = np.arange(3, 12, 2)

mask_dict = {}

for ps in patch_sizes:
    radius = ps // 2
    mask = np.zeros((ps, ps), dtype=np.uint8)
    mask = cv2.circle(
        img=mask,
        center=(radius, radius),
        radius=radius,
        color=1,
        thickness=-1
    )
    mask_dict[ps] = mask

np.save("circle_masks.npy", mask_dict)


text:
from unittest import TestCase
import numpy as np
from bfdc.feature import check_limits

class TestCheckLimits(TestCase):
    def test_check_limits_2d(self):
        img = np.zeros((512,512))
        peak = (-1,550)
        expected = (0,511)
        out = check_limits(img,peak)
        self.assertEqual(out,expected)

    def test_check_limits_linear(self):
        img = np.zeros(12)
        peak = 5
        expected = 5
        out = check_limits(img,peak)
        self.assertEqual(out,expected)

    def test_check_limits_linear0(self):
        img = np.zeros(12)
        peak = 0
        expected = 0
        out = check_limits(img,peak)
        self.assertEqual(out,expected)

    def test_check_limits_linear_neg(self):
        img = np.zeros(12)
        peak = -1
        expected = 0
        out = check_limits(img,peak)
        self.assertEqual(out,expected)

    def test_check_limits_linear_pos(self):
        img = np.zeros(12)
        peak = 15
        expected = 11
        out = check_limits(img,peak)
        self.assertEqual(out,expected)


text:
import os
import h5py
import numpy as np
from ..obs import Obs, CObs
from ..correlators import Corr

def _get_files(path, filestem, idl):
    ls = os.listdir(path)

    # Clean up file list
    files = list(filter(lambda x: x.startswith(filestem), ls))

    if not files:
        raise Exception('No files starting with', filestem, 'in folder', path)

    def get_cnfg_number(n):
        return int(n[len(filestem) + 1:-3])

    # Sort according to configuration number
    files.sort(key=get_cnfg_number)

    cnfg_numbers = []
    filtered_files = []
    for line in files:
        no = get_cnfg_number(line)
        if idl:
            if no in list(idl):
                filtered_files.append(line)
                cnfg_numbers.append(no)
        else:
            filtered_files.append(line)
            cnfg_numbers.append(no)

    # Check that configurations are evenly spaced
    dc = np.unique(np.diff(cnfg_numbers))
    if np.any(dc < 0):
        raise Exception("Unsorted files")
    if len(dc) == 1:
        idx = range(cnfg_numbers[0], cnfg_numbers[-1] + dc[0], dc[0])
    else:
        raise Exception('Configurations are not evenly spaced.')

    return filtered_files, idx

def read_meson_hd5(path, filestem, ens_id, meson='meson_0', tree='meson', idl=None):
    """Read hadrons meson hdf5 file and extract the meson labeled 'meson'

    Parameters
    -----------------
    path : str
        path to the files to read
    filestem : str
        namestem of the files to read
    ens_id : str
        name of the ensemble, required for internal bookkeeping
    meson : str
        label of the meson to be extracted, standard value meson_0 which
        corresponds to the pseudoscalar pseudoscalar two-point function.
    tree : str
        Label of the upmost directory in the hdf5 file, default 'meson'
        for outputs of the Meson module. Can be altered to read input
        from other modules with similar structures.
    idl : range
        If specified only configurations in the given range are read in.
    """

    files, idx = _get_files(path, filestem, idl)

    corr_data = []
    infos = []
    for hd5_file in files:
        file = h5py.File(path + '/' + hd5_file, "r")
        raw_data = list(file[tree + '/' + meson + '/corr'])
        real_data = [o[0] for o in raw_data]
        corr_data.append(real_data)
        if not infos:
            for k, i in file[tree + '/' + meson].attrs.items():
                infos.append(k + ': ' + i[0].decode())
        file.close()
    corr_data = np.array(corr_data)

    l_obs = []
    for c in corr_data.T:
        l_obs.append(Obs([c], [ens_id], idl=[idx]))

    corr = Corr(l_obs)
    corr.tag = r", ".join(infos)
    return corr

class Npr_matrix(np.ndarray):

    def __new__(cls, input_array, mom_in=None, mom_out=None):
        obj = np.asarray(input_array).view(cls)
        obj.mom_in = mom_in
        obj.mom_out = mom_out
        return obj

    @property
    def g5H(self):
        """Gamma_5 hermitean conjugate

        Uses the fact that the propagator is gamma5 hermitean, so just the
        in and out momenta of the propagator are exchanged.
        """
        return Npr_matrix(self,
                          mom_in=self.mom_out,
                          mom_out=self.mom_in)

    def _propagate_mom(self, other, name):
        s_mom = getattr(self, name, None)
        o_mom = getattr(other, name, None)
        if s_mom is not None and o_mom is not None:
            if not np.allclose(s_mom, o_mom):
                raise Exception(name + ' does not match.')
        return o_mom if o_mom is not None else s_mom

    def __matmul__(self, other):
        return self.__new__(Npr_matrix,
                            super().__matmul__(other),
                            self._propagate_mom(other, 'mom_in'),
                            self._propagate_mom(other, 'mom_out'))

    def __array_finalize__(self, obj):
        if obj is None:
            return
        self.mom_in = getattr(obj, 'mom_in', None)
        self.mom_out = getattr(obj, 'mom_out', None)

def read_ExternalLeg_hd5(path, filestem, ens_id, idl=None):
    """Read hadrons ExternalLeg hdf5 file and output an array of CObs

    Parameters
    ----------
    path : str
        path to the files to read
    filestem : str
        namestem of the files to read
    ens_id : str
        name of the ensemble, required for internal bookkeeping
    idl : range
        If specified only configurations in the given range are read in.
    """

    files, idx = _get_files(path, filestem, idl)

    mom = None

    corr_data = []
    for hd5_file in files:
        file = h5py.File(path + '/' + hd5_file, "r")
        raw_data = file['ExternalLeg/corr'][0][0].view('complex')
        corr_data.append(raw_data)
        if mom is None:
            mom = np.array(str(file['ExternalLeg/info'].attrs['pIn'])[3:-2].strip().split(' '), dtype=int)
        file.close()
    corr_data = np.array(corr_data)

    rolled_array = np.rollaxis(corr_data, 0, 5)

    matrix = np.empty((rolled_array.shape[:-1]), dtype=object)
    for si, sj, ci, cj in np.ndindex(rolled_array.shape[:-1]):
        real = Obs([rolled_array[si, sj, ci, cj].real], [ens_id], idl=[idx])
        imag = Obs([rolled_array[si, sj, ci, cj].imag], [ens_id], idl=[idx])
        matrix[si, sj, ci, cj] = CObs(real, imag)

    return Npr_matrix(matrix, mom_in=mom)

def read_Bilinear_hd5(path, filestem, ens_id, idl=None):
    """Read hadrons Bilinear hdf5 file and output an array of CObs

    Parameters
    ----------
    path : str
        path to the files to read
    filestem : str
        namestem of the files to read
    ens_id : str
        name of the ensemble, required for internal bookkeeping
    idl : range
        If specified only configurations in the given range are read in.
    """

    files, idx = _get_files(path, filestem, idl)

    mom_in = None
    mom_out = None

    corr_data = {}
    for hd5_file in files:
        file = h5py.File(path + '/' + hd5_file, "r")
        for i in range(16):
            name = file['Bilinear/Bilinear_' + str(i) + '/info'].attrs['gamma'][0].decode('UTF-8')
            if name not in corr_data:
                corr_data[name] = []
            raw_data = file['Bilinear/Bilinear_' + str(i) + '/corr'][0][0].view('complex')
            corr_data[name].append(raw_data)
            if mom_in is None:
                mom_in = np.array(str(file['Bilinear/Bilinear_' + str(i) + '/info'].attrs['pIn'])[3:-2].strip().split(' '), dtype=int)
            if mom_out is None:
                mom_out = np.array(str(file['Bilinear/Bilinear_' + str(i) + '/info'].attrs['pOut'])[3:-2].strip().split(' '), dtype=int)

        file.close()

    result_dict = {}

    for key, data in corr_data.items():
        local_data = np.array(data)

        rolled_array = np.rollaxis(local_data, 0, 5)

        matrix = np.empty((rolled_array.shape[:-1]), dtype=object)
        for si, sj, ci, cj in np.ndindex(rolled_array.shape[:-1]):
            real = Obs([rolled_array[si, sj, ci, cj].real], [ens_id], idl=[idx])
            imag = Obs([rolled_array[si, sj, ci, cj].imag], [ens_id], idl=[idx])
            matrix[si, sj, ci, cj] = CObs(real, imag)

        result_dict[key] = Npr_matrix(matrix, mom_in=mom_in, mom_out=mom_out)

    return result_dict

def read_Fourquark_hd5(path, filestem, ens_id, idl=None, vertices=["VA", "AV"]):
    """Read hadrons FourquarkFullyConnected hdf5 file and output an array of CObs

    Parameters
    ----------
    path : str
        path to the files to read
    filestem : str
        namestem of the files to read
    ens_id : str
        name of the ensemble, required for internal bookkeeping
    idl : range
        If specified only configurations in the given range are read in.
    vertices : list
        Vertex functions to be extracted.
    """

    files, idx = _get_files(path, filestem, idl)

    mom_in = None
    mom_out = None

    vertex_names = []
    for vertex in vertices:
        vertex_names += _get_lorentz_names(vertex)

    corr_data = {}

    tree = 'FourQuarkFullyConnected/FourQuarkFullyConnected_'

    for hd5_file in files:
        file = h5py.File(path + '/' + hd5_file, "r")

        for i in range(32):
            name = (file[tree + str(i) + '/info'].attrs['gammaA'][0].decode('UTF-8'), file[tree + str(i) + '/info'].attrs['gammaB'][0].decode('UTF-8'))
            if name in vertex_names:
                if name not in corr_data:
                    corr_data[name] = []
                raw_data = file[tree + str(i) + '/corr'][0][0].view('complex')
                corr_data[name].append(raw_data)
                if mom_in is None:
                    mom_in = np.array(str(file[tree + str(i) + '/info'].attrs['pIn'])[3:-2].strip().split(' '), dtype=int)
                if mom_out is None:
                    mom_out = np.array(str(file[tree + str(i) + '/info'].attrs['pOut'])[3:-2].strip().split(' '), dtype=int)

        file.close()

    intermediate_dict = {}

    for vertex in vertices:
        lorentz_names = _get_lorentz_names(vertex)
        for v_name in lorentz_names:
            if vertex not in intermediate_dict:
                intermediate_dict[vertex] = np.array(corr_data[v_name])
            else:
                intermediate_dict[vertex] += np.array(corr_data[v_name])

    result_dict = {}

    for key, data in intermediate_dict.items():

        rolled_array = np.moveaxis(data, 0, 8)

        matrix = np.empty((rolled_array.shape[:-1]), dtype=object)
        for index in np.ndindex(rolled_array.shape[:-1]):
            real = Obs([rolled_array[index].real], [ens_id], idl=[idx])
            imag = Obs([rolled_array[index].imag], [ens_id], idl=[idx])
            matrix[index] = CObs(real, imag)

        result_dict[key] = Npr_matrix(matrix, mom_in=mom_in, mom_out=mom_out)

    return result_dict

def _get_lorentz_names(name):
    assert len(name) == 2

    res = []

    if not set(name) <= set(['S', 'P', 'V', 'A', 'T']):
        raise Exception("Name can only contain 'S', 'P', 'V', 'A' or 'T'")

    if 'S' in name or 'P' in name:
        if not set(name) <= set(['S', 'P']):
            raise Exception("'" + name + "' is not a Lorentz scalar")

        g_names = {'S': 'Identity',
                   'P': 'Gamma5'}

        res.append((g_names[name[0]], g_names[name[1]]))

    elif 'T' in name:
        if not set(name) <= set(['T']):
            raise Exception("'" + name + "' is not a Lorentz scalar")
        raise Exception("Tensor operators not yet implemented.")
    else:
        if not set(name) <= set(['V', 'A']):
            raise Exception("'" + name + "' is not a Lorentz scalar")
        lorentz_index = ['X', 'Y', 'Z', 'T']

        for ind in lorentz_index:
            res.append(('Gamma' + ind + (name[0] == 'A') * 'Gamma5',
                        'Gamma' + ind + (name[1] == 'A') * 'Gamma5'))

    return res

max_stars_repo_name: JanNeuendorf/pyerrors

max_issues_repo_name: JanNeuendorf/pyerrors

max_forks_repo_name: JanNeuendorf/pyerrors


text:
from math import sqrt, pi
import bpy
import numpy as np
from blenderneuron.blender.utils import create_many_copies

class CurveContainer:

    def __init__(self, root, curve_template, smooth_sections, color, brightness,
                 recursive=True, origin_type="center", closed_ends=True,
                 container_material=None):

        self.name = root.name
        self.smooth_sections = smooth_sections
        self.closed_ends = closed_ends
        self.assigned_container_material = container_material
        self.default_color = color
        self.default_brightness = brightness

        # copy the curve template and make a new blender object out of it
        bpy.data.objects.new(self.name, curve_template.copy())

        self.linked = False
        self.material_indices = []

        # Quickly find the spline of a given section
        self.name2spline_index = {}
        self.spline_index2section = {}

        # Recursively add section splines and corresponding materials to the container
        self.add_section(root, recursive, in_top_level=True, origin_type=origin_type)

    def get_object(self):
        return bpy.data.objects.get(self.name)

    @property
    def object(self):
        return bpy.data.objects[self.name]

    @property
    def curve(self):
        ob = self.get_object()

        return ob.data

    def set_parent_object(self, parent_container):

        child = self.get_object()
        parent = parent_container.get_object()

        child.parent = parent
        child.matrix_parent_inverse = parent.matrix_world.inverted()

        # Children have their location and scale editing locked (to prevent disconnects)
        # Rotations are allowed
        child.lock_location = child.lock_scale = [True] * 3

    def remove(self):
        self.unlink()

        bl_objects = bpy.data.objects
        ob = self.get_object()
        object_name = ob.name

        # materials
        for mat in ob.data.materials:
            if mat is not None:
                # remove material animation if any
                if mat.animation_data is not None:
                    bpy.data.actions.remove(mat.animation_data.action)

                # remove material node animation
                if mat.node_tree is not None and mat.node_tree.animation_data is not None:
                    bpy.data.actions.remove(mat.node_tree.animation_data.action)

                # remove material
                bpy.data.materials.remove(mat)

        # curve
        bpy.data.curves.remove(ob.data)

    @property
    def origin(self):
        return self.get_object().location

    @origin.setter
    def origin(self, value):
        self.curve.location = value

    def diam0version(self, start, end):
        lengths = end - start

        # Versor extension
        length = sqrt(np.power(lengths,2).sum())

        # Extend in the same direction by a small amount
        if length > 0:
            versor = lengths / length
            extended = end + versor * 0.01

        # If the points are identical (should not happen), return the end point
        else:
            return end

        return extended

    def add_spline(self, coords, radii, smooth):
        curve = self.curve

        sec_spline = curve.splines.new('BEZIER')

        # This line is necessary due to a bug in Blender
        # see: https://developer.blender.org/T54112
        curve.resolution_u = curve.resolution_u

        # Subtract the container origin (as bezier points are
        # relative to the object origin)
        coords = coords - self.origin

        if self.closed_ends:
            # Add closed, 0-diam caps (to avoid open-ended cylinders)
            cap1 = self.diam0version(coords[1], coords[0])
            cap2 = self.diam0version(coords[-2], coords[-1])

            coords = np.concatenate(([cap1], coords, [cap2]))
            radii = np.concatenate(([0],radii,[0]))

        # Flatten the coords back (needed by the foreach_set() functions below)
        coords.shape = (-1)

        bezier_points = sec_spline.bezier_points

        # Allocate space for bezier points
        #bezier_points.clear() # can't clear the one initial point
        bezier_points.add(len(radii)-1)

        bezier_points.foreach_set('radius', radii)
        bezier_points.foreach_set('co', coords)

        if not smooth:
            # Fast
            bezier_points.foreach_set('handle_right', coords)
            bezier_points.foreach_set('handle_left', coords)

        else:
            # Slower
            for p in bezier_points:
                p.handle_right_type = p.handle_left_type = 'AUTO'

        return sec_spline

    @staticmethod
    def create_material(name, color, brightness):
        mat = bpy.data.materials.new(name)

        mat.diffuse_color = color
        mat.emit = brightness

        # # Ambient and back lighting
        # mat.ambient = 0.85
        # mat.translucency = 0.85
        #
        # # Raytraced reflections
        # mat.raytrace_mirror.use = True
        # mat.raytrace_mirror.reflect_factor = 0.1
        # mat.raytrace_mirror.fresnel = 2.0

        # Add Blender render and Cycles nodes
        mat.use_nodes = True

        nodes = mat.node_tree.nodes
        links = mat.node_tree.links

        links.clear()
        nodes.clear()

        # Cycles nodes
        cl_out = nodes.new('ShaderNodeOutputMaterial')
        cl_emit = nodes.new('ShaderNodeEmission')
        cl_emit.location = [-200, 0]
        cl_emit.inputs['Strength'].default_value = brightness
        cl_emit.inputs['Color'].default_value = list(mat.diffuse_color) + [1]

        # cl_trans = nodes.new('ShaderNodeBsdfTransparent')
        # cl_trans.location = [-200, 100]
        # links.new(cl_trans.outputs['BSDF'], cl_out.inputs['Surface'])

        links.new(cl_emit.outputs['Emission'], cl_out.inputs['Surface'])

        # Blender render nodes
        br_out = nodes.new('ShaderNodeOutput')
        br_out.location = [0, -200]
        br_mat = nodes.new('ShaderNodeExtendedMaterial')
        br_mat.location = [-200, -200]
        br_mat.material = mat

        links.new(br_mat.outputs['Color'], br_out.inputs['Color'])
        links.new(br_mat.outputs['Alpha'], br_out.inputs['Alpha'])

        return mat

    def add_material_to_object(self, material):
        mats = self.curve.materials
        mats.append(material)
        mat_idx = len(mats)-1
        return mat_idx

    def to_global(self, local_coords):
        """
        This function performs the fancy fast vectorized multiplication of the container object's
        world matrix (trans, rot, scale) by the local points to obtain the global
        version of the coordinates.

        :param local_coords: Local coords as returned by bezier_points.foreach_get("co")
        :return: Global version of the local_coords
        """

        # Get the world matrix
        matrix = self.get_object().matrix_world

        # Reshape coords to Nx3 matrix
        local_coords.shape = (-1, 3)

        # Add an extra 1.0s column (for matrix dot prod)
        local_coords = np.c_[local_coords, np.ones(local_coords.shape[0])]

        # Dot product matrix with the coords transpose
        # Keep the first 3 rows (x,y,z)
        # Transpose result to Nx3
        # Flatten
        global_coords = np.dot(matrix, local_coords.T)[0:3].T.reshape((-1))

        return global_coords

    def update_group_section(self, root, recursive=True):

        ob = self.get_object()

        # Find the spline that corresponds to the section
        spline_i = self.name2spline_index[root.name]

        try:
            spline = self.curve.splines[spline_i]

        except IndexError:
            print("Could not find spline with index " + str(spline_i) + " in " + self.name +
                  ". This can happen if a spline is deleted in Edit Mode.")
            raise

        point_source = spline.bezier_points

        del spline

        # Get the 3d points
        num_coords = len(point_source)

        coords = np.zeros(num_coords * 3)
        point_source.foreach_get("co", coords)

        # Adjust coords for container origin and rotation
        coords = self.to_global(coords)

        if self.closed_ends:
            # Discard the 0-radius end caps
            coords = coords[3:-3]

        root.coords = coords.tolist()

        # Get radii
        radii = np.zeros(num_coords)
        point_source.foreach_get("radius", radii)
        root.radii  = (radii[1:-1] if self.closed_ends else radii).tolist()
        del radii

        # Cleanup before recursion
        del point_source, num_coords, coords

        if recursive:
            for child in root.children:
                self.update_group_section(child, recursive=True)

    def add_section(self, root, recursive=True, in_top_level=True, origin_type="center"):
        # Reshape the coords to be n X 3 array (for xyz)
        coords = np.array(root.coords)
        coords.shape = (-1, 3)

        if in_top_level:
            self.set_origin(coords, origin_type)

        # Add section spline to the cell object
        spline = self.add_spline(coords, root.radii, self.smooth_sections)

        # If material is not provided, create one
        if self.assigned_container_material is None:
            material = CurveContainer.create_material(
                root.name,
                self.default_color,
                self.default_brightness
            )

        # If material is provided, assign it to the spline
        else:
            material = self.assigned_container_material

        mat_idx = self.add_material_to_object(material)

        # Assign the material to the new spline
        spline.material_index = mat_idx

        # Save spline index for later lookup
        # Note: In Blender, using edit-mode on a curve object, results in creation of
        # new spline instances when returning to object-mode. If references to the
        # old splines are kept, Blender usually crashes. Here we retain the spline index,
        # which is preserved (if splines are not deleted in edit-mode).
        spline_index = len(self.curve.splines) - 1
        self.name2spline_index[root.name] = spline_index
        self.spline_index2section[spline_index] = root

        # Cleanup before starting recursion
        del spline, material, mat_idx

        # Do same with the children
        if recursive:
            for child in root.children:
                self.add_section(child, recursive=True, in_top_level=False)

    def set_origin(self, coords, type = "center"):
        if type == "center":
            point_count = coords.shape[0]

            center_i = int(point_count / 2)

            # If even number of 3D points, use the mean of the
            # two points closest to the section middle
            if point_count % 2 == 0:
                center_i2 = center_i - 1
                center = (coords[center_i] + coords[center_i2]) / 2.0
            else:
                center = coords[center_i]

            self.get_object().location = center

        if type == "first":
            self.get_object().location = coords[0]

    def link(self):
        link = bpy.context.scene.objects.link
        bl_objects = bpy.data.objects

        link(self.get_object())

        self.linked = True

    def unlink(self):
        unlink_from_scene = bpy.context.scene.objects.unlink

        try:
            ob = self.get_object()
            unlink_from_scene(ob)

        except RuntimeError:
            pass  # ignore if already unlinked

        self.linked = False

max_stars_repo_name: Helveg/BlenderNEURON

max_issues_repo_name: Helveg/BlenderNEURON

max_forks_repo_name: Helveg/BlenderNEURON


text:
from __future__ import print_function

import os

import numpy as np
from keras import backend as K, models
from keras.callbacks import ModelCheckpoint, CSVLogger, EarlyStopping, ReduceLROnPlateau, TensorBoard
import matplotlib.pyplot as plt
from keras_contrib.callbacks.cyclical_learning_rate import CyclicLR
from keras.layers import Conv2D, MaxPooling2D, UpSampling2D
from keras.layers.core import Activation
from keras.layers.normalization import BatchNormalization
from keras.optimizers import Adam
from skimage.io import imsave

from data import load_train_data, load_test_data

K.set_image_data_format('channels_last')  # TF dimension ordering in this code

img_rows = 512
img_cols = 512
batch_size = 4
smooth = 1.
epochs = 500

def dice_coef(y_true, y_pred):
    y_true_f = K.flatten(y_true)
    y_pred_f = K.flatten(y_pred)
    intersection = K.sum(y_true_f * y_pred_f)
    return (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)

def dice_coef_loss(y_true, y_pred):
    return -dice_coef(y_true, y_pred)

def precision(y_true, y_pred):
    """Precision metric.

    Only computes a batch-wise average of precision.

    Computes the precision, a metric for multi-label classification of
    how many selected items are relevant.
    """
    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
    precision = true_positives / (predicted_positives + K.epsilon())
    return precision

def recall(y_true, y_pred):
    """Recall metric.

    Only computes a batch-wise average of recall.

    Computes the recall, a metric for multi-label classification of
    how many relevant items are selected.
    """
    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
    recall = true_positives / (possible_positives + K.epsilon())
    return recall

def f1score(y_true, y_pred):
    def recall(y_true, y_pred):
        """Recall metric.

        Only computes a batch-wise average of recall.

        Computes the recall, a metric for multi-label classification of
        how many relevant items are selected.
        """
        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
        recall = true_positives / (possible_positives + K.epsilon())
        return recall

    def precision(y_true, y_pred):
        """Precision metric.

        Only computes a batch-wise average of precision.

        Computes the precision, a metric for multi-label classification of
        how many selected items are relevant.
        """
        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
        precision = true_positives / (predicted_positives + K.epsilon())
        return precision

    precision = precision(y_true, y_pred)
    recall = recall(y_true, y_pred)
    return 2 * ((precision * recall) / (precision + recall))

def get_segnet():
    kernel = 3

    encoding_layers = [
        Conv2D(32, (3, 3), padding='same', input_shape=(img_rows, img_cols, 1)),
        BatchNormalization(axis=3),
        Activation('relu'),
        Conv2D(32, (kernel, kernel), padding='same'),
        BatchNormalization(axis=3),
        Activation('relu'),
        MaxPooling2D(),

        Conv2D(64, (kernel, kernel), padding='same'),
        BatchNormalization(axis=3),
        Activation('relu'),
        Conv2D(64, (kernel, kernel), padding='same'),
        BatchNormalization(axis=3),
        Activation('relu'),
        MaxPooling2D(),

        Conv2D(128, (kernel, kernel), padding='same'),
        BatchNormalization(axis=3),
        Activation('relu'),
        Conv2D(128, (kernel, kernel), padding='same'),
        BatchNormalization(axis=3),
        Activation('relu'),
        Conv2D(128, (kernel, kernel), padding='same'),
        BatchNormalization(axis=3),
        Activation('relu'),
        MaxPooling2D(),

        Conv2D(256, (kernel, kernel), padding='same'),
        BatchNormalization(axis=3),
        Activation('relu'),
        Conv2D(256, (kernel, kernel), padding='same'),
        BatchNormalization(axis=3),
        Activation('relu'),
        Conv2D(256, (kernel, kernel), padding='same'),
        BatchNormalization(axis=3),
        Activation('relu'),
        MaxPooling2D(),

        Conv2D(256, (kernel, kernel), padding='same'),
        BatchNormalization(axis=3),
        Activation('relu'),
        Conv2D(256, (kernel, kernel), padding='same'),
        BatchNormalization(axis=3),
        Activation('relu'),
        Conv2D(256, (kernel, kernel), padding='same'),
        BatchNormalization(axis=3),
        Activation('relu'),
        MaxPooling2D(),
    ]

    autoencoder = models.Sequential()
    autoencoder.encoding_layers = encoding_layers

    for l in autoencoder.encoding_layers:
        autoencoder.add(l)

    decoding_layers = [
        UpSampling2D(size=(2, 2)),
        Conv2D(256, (kernel, kernel), padding='same'),
        BatchNormalization(axis=3),
        Activation('relu'),
        Conv2D(256, (kernel, kernel), padding='same'),
        BatchNormalization(axis=3),
        Activation('relu'),
        Conv2D(256, (kernel, kernel), padding='same'),
        BatchNormalization(axis=3),
        Activation('relu'),

        UpSampling2D(size=(2, 2)),
        Conv2D(256, (kernel, kernel), padding='same'),
        BatchNormalization(axis=3),
        Activation('relu'),
        Conv2D(256, (kernel, kernel), padding='same'),
        BatchNormalization(axis=3),
        Activation('relu'),
        Conv2D(256, (kernel, kernel), padding='same'),
        BatchNormalization(axis=3),
        Activation('relu'),

        UpSampling2D(size=(2, 2)),
        Conv2D(128, (kernel, kernel), padding='same'),
        BatchNormalization(axis=3),
        Activation('relu'),
        Conv2D(128, (kernel, kernel), padding='same'),
        BatchNormalization(axis=3),
        Activation('relu'),
        Conv2D(64, (kernel, kernel), padding='same'),
        BatchNormalization(),
        Activation('relu'),

        UpSampling2D(size=(2, 2)),
        Conv2D(64, (kernel, kernel), padding='same'),
        BatchNormalization(axis=3),
        Activation('relu'),
        Conv2D(32, (kernel, kernel), padding='same'),
        BatchNormalization(axis=3),
        Activation('relu'),

        UpSampling2D(size=(2, 2)),
        Conv2D(32, (kernel, kernel), padding='same'),
        BatchNormalization(axis=3),
        Activation('relu'),

        Conv2D(1, (1, 1), padding='valid'),
        BatchNormalization(axis=3),
    ]
    autoencoder.decoding_layers = decoding_layers
    for l in autoencoder.decoding_layers:
        autoencoder.add(l)

    autoencoder.add(Activation('sigmoid'))
    autoencoder.compile(loss=dice_coef_loss, optimizer=Adam(lr=1e-3),
                        metrics=[dice_coef, 'accuracy', precision, recall, f1score])
    autoencoder.summary()

    return autoencoder

def train_and_predict(bit):
    print('-' * 30)
    print('Loading and train data (bit = ' + str(bit) + ') ...')
    print('-' * 30)
    imgs_bit_train, imgs_bit_mask_train, _ = load_train_data(bit)

    print(imgs_bit_train.shape[0], imgs_bit_mask_train.shape[0])

    imgs_bit_train = imgs_bit_train.astype('float32')
    mean = np.mean(imgs_bit_train)
    std = np.std(imgs_bit_train)

    imgs_bit_train -= mean
    imgs_bit_train /= std

    imgs_bit_mask_train = imgs_bit_mask_train.astype('float32')
    imgs_bit_mask_train /= 255.  # scale masks to [0, 1]

    print('-' * 30)
    print('Creating and compiling model (bit = ' + str(bit) + ') ...')
    print('-' * 30)
    model = get_segnet()

    callbacks = [
        EarlyStopping(
            monitor='val_loss',
            patience=30,
            verbose=1,
            min_delta=1e-5
        ),
        ReduceLROnPlateau(
            monitor='val_loss',
            factor=0.1,
            patience=5,
            verbose=1,
            epsilon=1e-5
        ),
        CyclicLR(
            base_lr=0.00001,
            max_lr=0.00006,
            step_size=2000,
            mode='triangular'
        ),
        CSVLogger(
            'log_segnet_' + str(bit) + '.csv'
        ),
        ModelCheckpoint(
            'weights_segnet_' + str(bit) + '.h5',
            monitor='val_loss',
            save_best_only=True
        ),
        TensorBoard(
            log_dir="logs/segnet",
            histogram_freq=0,
            write_grads=True,
            write_graph=True,
            write_images=True
        )
    ]

    print('-' * 30)
    print('Fitting model (bit = ' + str(bit) + ') ...')
    print('-' * 30)

    model.fit(imgs_bit_train, imgs_bit_mask_train, batch_size=batch_size, epochs=epochs, verbose=1, shuffle=True,
              validation_split=0.2,
              callbacks=callbacks)

    print('-' * 30)
    print('Loading and preprocessing test data (bit = ' + str(bit) + ') ...')
    print('-' * 30)

    imgs_bit_test, imgs_mask_test, imgs_bit_id_test = load_test_data(bit)

    imgs_bit_test = imgs_bit_test.astype('float32')
    imgs_bit_test -= mean
    imgs_bit_test /= std

    print('-' * 30)
    print('Loading saved weights...')
    print('-' * 30)
    model.load_weights('weights_segnet_' + str(bit) + '.h5')

    print('-' * 30)
    print('Predicting masks on test data (bit = ' + str(bit) + ') ...')
    print('-' * 30)
    imgs_mask_test = model.predict(imgs_bit_test, verbose=1)

    if bit == 8:
        print('-' * 30)
        print('Saving predicted masks to files...')
        print('-' * 30)
        pred_dir = 'preds_8'
        if not os.path.exists(pred_dir):
            os.mkdir(pred_dir)
        for image, image_id in zip(imgs_mask_test, imgs_bit_id_test):
            image = (image[:, :, 0] * 255.).astype(np.uint8)
            imsave(os.path.join(pred_dir, str(image_id).split('/')[-1] + '_pred_segnet.png'), image)

    elif bit == 16:
        print('-' * 30)
        print('Saving predicted masks to files...')
        print('-' * 30)
        pred_dir = 'preds_16'
        if not os.path.exists(pred_dir):
            os.mkdir(pred_dir)
        for image, image_id in zip(imgs_mask_test, imgs_bit_id_test):
            image = (image[:, :, 0] * 255.).astype(np.uint8)
            imsave(os.path.join(pred_dir, str(image_id).split('/')[-1] + '_pred_segnet.png'), image)

if __name__ == '__main__':
    train_and_predict(8)
    # train_and_predict(16)

max_stars_repo_name: maycownd/TCC-Segmentacao-Imagens

max_issues_repo_name: maycownd/TCC-Segmentacao-Imagens

max_forks_repo_name: maycownd/TCC-Segmentacao-Imagens