#!/usr/bin/env python3

# Four rooms paper helper functions.
# This script contains some useful functions for navigation data analysis.
# Author: Hannah Sheahan
# Date: 30/11/2018
# Issues: N/A

#-----------------------------------------------------------------------------#
from typing import Any, List, Mapping, Sequence, Tuple

import constants as cs
import numpy as np
import math
import json
import matplotlib.pyplot as plt
import scipy
import scipy.stats as stats
from matplotlib import cm
from matplotlib.collections import PatchCollection
from matplotlib.patches import Rectangle
from datetime import datetime, timedelta

import seaborn as sns
import copy
import pandas as pd
import sys
import random

# file i/o
import os

# for adding text description next to plots
from io import BytesIO
import base64

#-------------------------------------------------------------------------------

def find_data(data_path: str, sessions: List[str], exclusions: List[str]) -> Tuple[List[List[str]], int]:
  """Returns the Google Drive file paths of all data files in each session."""
  file_paths = []
  for session in sessions:
    session_path = os.path.join(data_path, session)
    files = os.listdir(session_path)

    # Exclude .DS_Store and subjects who's experiments crashed.
    included_files = []
    for idx, file in enumerate(files):
      include = True
      if file[0]!='.':  # Exclude .DS_Store.
        for exclude_string in exclusions:
          if exclude_string in file:
            include = False
        if include:
          included_files.append(file)
    paths = [os.path.join(session_path, file) for file in included_files]
    file_paths.append(paths)

  assert len(file_paths[0]) == len(file_paths[1])
  num_subjects = len(file_paths[0])
  print('Data files found from both sessions for {} participants.'.format(num_subjects))
  return file_paths, num_subjects

#-------------------------------------------------------------------------------

def load_data(paths: List[List[str]], num_subjects: int, sessions: List[str]) -> Tuple[List[List[Mapping[str, Any]]], Any]:
  """Loads the data from each session and matches it to the right participant.

  Args:
    paths: List of filepaths for each of the two sessions (in and out of scanner).
    num_subjects: The number of participants.
  Returns:
    All participant behavioural data, listed by session, then participant. The
    indices of each participant within each session is matched across the two
    sessions to refer to the same participant, and both are in chronological
    order.
  """
  # Use this to save the player/subject main learning data, with each element being a different subject
  all_data = [[None] * num_subjects for _ in sessions]
  file_names = [[None] * num_subjects for _ in sessions]

  for session_idx, session in enumerate(sessions):
    file_paths = paths[session_idx]
    # Sort the data in each session chronologically by participant index.
    file_order = np.argsort([a[-14:] for a in file_paths])
    file_paths = [file_paths[i] for i in file_order]
    for subject_idx in range(num_subjects):
      path = file_paths[subject_idx]
      with open(path, "r") as f:
        data = json.load(f)
      all_data[session_idx][subject_idx] = data
      file_names[session_idx][subject_idx] = path

  return all_data, file_names

#-------------------------------------------------------------------------------

def calculate_quiz_score(questionaire_data: List[Mapping[str, str]]) -> List[int]:
  """Calculate the quiz scores and order them by participant ID."""
  subject_quiz_scores = {}
  for subject_idx in range(len(questionaire_data)):
    quiz_score = 0
    sub_question_data = questionaire_data[subject_idx]
    if sub_question_data['Q1'] == 'top left':
      quiz_score += 1
    if sub_question_data['Q2'] == 'bottom right':
      quiz_score += 1
    if sub_question_data['Q3'] == 'bottom left':
      quiz_score += 1
    if sub_question_data['Q4'] == 'top right':
      quiz_score += 1
    if sub_question_data['Q5'] == 'bottom left, bottom right':
      quiz_score += 1
    if sub_question_data['Q6'] == 'top left, top right':
      quiz_score += 1
    if sub_question_data['Q7'] == 'top left, bottom left':
      quiz_score += 1
    if sub_question_data['Q8'] == 'top right, bottom right':
      quiz_score += 1
    subject_quiz_scores[sub_question_data['ID']] = quiz_score

  # Order the quiz scores by subject index.
  subject_ids = []
  for key, value in subject_quiz_scores.items():
    subject_ids.append(int(key))
  sorted_indices = sorted(range(len(subject_ids)), key=lambda k: subject_ids[k])
  subject_ordered_quiz_scores = []
  for idx in sorted_indices:
    subject_ordered_quiz_scores.append(subject_quiz_scores[str(subject_ids[idx])])

  return subject_ordered_quiz_scores

#-------------------------------------------------------------------------------

def plot_cov_training(ax, allPlayerLearningData, colours, bunch_factor=1, title=''):
    """Plot the covariance learning results for training session."""
    metric = 'correctChoices'
    whichSession = cs.SESSION_TRAINING
    nsubjects = len(allPlayerLearningData[whichSession])
    # We want the human choices only.
    allChoices = [allPlayerLearningData[whichSession][i]["roomChoicesData"][metric] for i in range(len(allPlayerLearningData[whichSession]))]
    allChoices = np.asarray(allChoices, dtype=np.float)
    allChoices = allChoices[~np.isnan(allChoices)]  # remove nans.

    # Average across adjacent trials using a bunch_factor.
    allChoices = np.reshape(allChoices, (bunch_factor, -1, nsubjects), order='F')
    allChoices = np.mean(allChoices, axis=0)

    # Mean and sem.
    meanChoices = 100 * np.nanmean(allChoices, axis=1)
    seChoices = 100 * np.divide(np.nanstd(allChoices, axis=1), np.sqrt(nsubjects) )
    ax.axhline(y=50, color='gray', linestyle=':')

    subtrials = np.linspace(1,bunch_factor*meanChoices.size, meanChoices.size)
    ax.scatter(subtrials, meanChoices, color=colours[0])
    ax.errorbar(subtrials, meanChoices, seChoices, color=colours[0])

    ax.set_ylabel('Correct first room choice (%)')
    ax.set_ylim([40,100])
    ax.set_xlabel('Trials')
    ax.set_title('Human behaviour, day 1')
    return allChoices

#-------------------------------------------------------------------------------

def plot_cov_scanner(ax, allPlayerLearningData, colours, bunch_factor=1, title=''):
    """Plot the covariance learning results for scanner session."""
    metric = 'humanCorrectChoices'
    whichSession = cs.SESSION_SCANNER
    nsubjects = len(allPlayerLearningData[whichSession])

    # We want the human choices only.
    allChoices = [allPlayerLearningData[whichSession][i]["roomChoicesData"][metric] for i in range(len(allPlayerLearningData[whichSession]))]
    allChoices = np.asarray(allChoices, dtype=np.float)
    allChoices = allChoices[~np.isnan(allChoices)]  # remove nans.

    # Average across adjacent trials using a bunch_factor.
    allChoices = np.reshape(allChoices, (bunch_factor, -1, nsubjects), order='F')
    allChoices = np.mean(allChoices, axis=0)

    # Draw context colour rectangles.
    errorboxes = [
      [Rectangle((1, 0), 32, 100)],
      [Rectangle((33, 0), 32, 100)],
      [Rectangle((33+32, 0), 32, 100)],
    ]
    edgecolor = 'white'
    facecolors = ['gold', 'gold', 'gold']
    alphas = [0.1, 0.25, 0.4]
    for box, facecolor, alpha in zip(errorboxes, facecolors, alphas):
      pc = PatchCollection(box, facecolor=facecolor, alpha=alpha,
                         edgecolor=facecolor)
      ax.add_collection(pc)

    # Mean and sem.
    meanChoices = 100 * np.nanmean(allChoices, axis=1)
    seChoices = 100 * np.divide(np.nanstd(allChoices, axis=1), np.sqrt(nsubjects) )
    ax.axhline(y=50, color='gray', linestyle=':')
    # Note that here we artificially double the trial indices to account for
    # these decisions being from humans only, as in half of trials this choice
    # is made by the computer agent.
    subtrials = np.linspace(bunch_factor/2, 2* bunch_factor*meanChoices.size, meanChoices.size)

    # Plot the learning.
    ax.scatter(subtrials, meanChoices, color=colours[1])
    ax.errorbar(subtrials, meanChoices, seChoices, color=colours[1])

    ax.set_ylim([40,100])
    ax.set_xlabel('Trials')
    ax.set_title('Human behaviour, day 2')

    return allChoices

#-------------------------------------------------------------------------------

def insert_rests(trial_maps: List[str], which_session):
  """Inserts the rests back into the sequence of trial maps."""
  if which_session == cs.SESSION_TRAINING:
    rest_idx = [0, 1, 2, 3, 4, 5, 6, 7, 8, 25, 42, 59, 76]
  else:
    rest_idx = [0, 1, 2, 3, 4, 5, 6, 23, 40, 57, 74, 91, 108]
  corrected_maps = trial_maps[:]
  for trial in rest_idx:
    corrected_maps.insert(trial, 'rest         ')
  return corrected_maps

#-------------------------------------------------------------------------------

def plot_scores_training_context(ax, allPlayerLearningData, contexts, colour, bunch_factor=1, title='', ylim=(15,30)):
    """Plot the score learning results for training session, by context."""
    metric = 'allScores'
    whichSession = cs.SESSION_TRAINING
    nsubjects = len(allPlayerLearningData[whichSession])
    # if you want the human choices only, only average across subs over those
    allChoices = [allPlayerLearningData[whichSession][i][metric] for i in range(len(allPlayerLearningData[whichSession]))]

    contextChoices = []
    for subject_idx, subjectChoices in enumerate(allChoices):
      contextMaps = allPlayerLearningData[whichSession][subject_idx]["contextMapList"]
      #print([trial_idx for trial_idx, context in enumerate(contextMaps) if context[10:] in contexts])
      contextMaps = insert_rests(contextMaps, cs.SESSION_TRAINING)
      contextTrials = [subjectChoices[trial_idx] for trial_idx, context in enumerate(contextMaps) if context[10:] in contexts]
      contextTrials = np.asarray(contextTrials, dtype=np.float)
      contextTrials = contextTrials[~np.isnan(contextTrials)]  # remove nans.
      contextChoices.append(contextTrials)

    contextChoices = np.asarray(contextChoices, dtype=np.float)
    contextChoices = np.transpose(contextChoices)

    # Average across adjacent trials using a bunch_factor
    contextChoices = np.reshape(contextChoices, (bunch_factor, -1, nsubjects), order='F')
    contextChoices = np.mean(contextChoices, axis=0)

    # mean and sem
    meanChoices = np.nanmean(contextChoices, axis=1)
    seChoices = np.divide(np.nanstd(contextChoices, axis=1), np.sqrt(nsubjects) )
    if 'peanut' in contexts: # vertical. Was tested first.
      subtrials = [list(range(16)), list(range(32, 48))]
      subtrials = [element for sublist in subtrials for element in sublist]
    else:
      subtrials = [list(range(16, 32)), list(range(48, 64))]
      subtrials = [element for sublist in subtrials for element in sublist]

    # First block in this context.
    trials = np.reshape(subtrials[:len(subtrials)//2], (bunch_factor, -1), order='F')
    trials = np.mean(trials, axis=0)
    ax.scatter(trials, meanChoices[:len(meanChoices)//2], color=colour)
    ax.errorbar(trials, meanChoices[:len(meanChoices)//2], seChoices[:len(seChoices)//2], color=colour)

    # Second block in this context.
    trials = np.reshape(subtrials[len(subtrials)//2:], (bunch_factor, -1), order='F')
    trials = np.mean(trials, axis=0)
    ax.scatter(trials, meanChoices[len(meanChoices)//2:], color=colour)
    handle = ax.errorbar(trials, meanChoices[len(meanChoices)//2:], seChoices[len(seChoices)//2:], color=colour)

    ax.set_ylabel('Trial score')
    ax.set_xlabel('Trials')
    ax.set_title('Human behaviour, day 1')
    ax.set_ylim(ylim)
    return allChoices, handle

#-------------------------------------------------------------------------------

def plot_scores_scanner_context(ax, allPlayerLearningData, contexts, colour, bunch_factor=1, title='', ylim=(15,30)):
    """Plot the score learning results for scanner session, by context.
    """
    metric = 'allScores'
    whichSession = cs.SESSION_SCANNER
    nsubjects = len(allPlayerLearningData[whichSession])
    # if you want the human choices only, only average across subs over those
    allChoices = [allPlayerLearningData[whichSession][i][metric] for i in range(len(allPlayerLearningData[whichSession]))]

    contextChoices = []
    for subject_idx, subjectChoices in enumerate(allChoices):
      contextMaps = allPlayerLearningData[whichSession][subject_idx]["contextMapList"]
      contextMaps = insert_rests(contextMaps, cs.SESSION_SCANNER)
      contextTrials = [subjectChoices[trial_idx] for trial_idx, context in enumerate(contextMaps) if context[10:] in contexts]
      contextTrials = [i for i in contextTrials if ~np.isnan(np.asarray(i, dtype=np.float))]  # remove nans.
      while len(contextTrials) != 48:
        contextTrials.append(np.nan)
      contextTrials = np.asarray(contextTrials, dtype=np.float)
      contextChoices.append(contextTrials)

    contextChoices = np.asarray(contextChoices, dtype=np.float)
    contextChoices = np.transpose(contextChoices)

    # Average across adjacent trials using a bunch_factor
    contextChoices = np.reshape(contextChoices, (bunch_factor, -1, nsubjects), order='F')
    contextChoices = np.nanmean(contextChoices, axis=0)

    # Draw context colour rectangles
    errorboxes = [
      [Rectangle((1, 0), 32, 100)],
      [Rectangle((33, 0), 32, 100)],
      [Rectangle((33+32, 0), 32, 100)],
    ]
    edgecolor = 'white'
    facecolors = ['gold', 'gold', 'gold']
    alphas = [0.1, 0.25, 0.4]
    for box, facecolor, alpha in zip(errorboxes, facecolors, alphas):
      pc = PatchCollection(box, facecolor=facecolor, alpha=alpha,
                         edgecolor=facecolor)
      ax.add_collection(pc)

    # mean and sem
    meanChoices = np.nanmean(contextChoices, axis=1)
    seChoices = np.divide(np.nanstd(contextChoices, axis=1), np.sqrt(nsubjects) )
    # Manually increase span over which trials are plotted to reflect interleaved contexts.
    subtrials = np.linspace(bunch_factor/2, 2* bunch_factor*meanChoices.size, meanChoices.size)

    # Plot the learning.
    ax.scatter(subtrials, meanChoices, color=colour)
    handle = ax.errorbar(subtrials, meanChoices, seChoices, color=colour)

    ax.set_xlabel('Trials')
    ax.set_title('Human behaviour, day 2')
    ax.set_ylim(ylim)

    return allChoices, handle

#-------------------------------------------------------------------------------

def plot_cov_training_context(ax, allPlayerLearningData, contexts, colour, bunch_factor=1, title=''):
    """Plot the covariance learning results for training session, by context."""
    metric = 'correctChoices'
    whichSession = cs.SESSION_TRAINING
    nsubjects = len(allPlayerLearningData[whichSession])
    # if you want the human choices only, only average across subs over those
    allChoices = [allPlayerLearningData[whichSession][i]["roomChoicesData"][metric] for i in range(len(allPlayerLearningData[whichSession]))]

    contextChoices = []
    for subject_idx, subjectChoices in enumerate(allChoices):
      contextMaps = allPlayerLearningData[whichSession][subject_idx]["contextMapList"]
      contextMaps = insert_rests(contextMaps, cs.SESSION_TRAINING)
      contextTrials = [subjectChoices[trial_idx] for trial_idx, context in enumerate(contextMaps) if context[10:] in contexts]
      contextTrials = np.asarray(contextTrials, dtype=np.float)
      contextTrials = contextTrials[~np.isnan(contextTrials)]  # remove nans.
      contextChoices.append(contextTrials)

    contextChoices = np.asarray(contextChoices, dtype=np.float)
    contextChoices = np.transpose(contextChoices)

    # Average across adjacent trials using a bunch_factor
    contextChoices = np.reshape(contextChoices, (bunch_factor, -1, nsubjects), order='F')
    contextChoices = np.mean(contextChoices, axis=0)

    # mean and sem
    meanChoices = 100 * np.nanmean(contextChoices, axis=1)
    seChoices = 100 * np.divide(np.nanstd(contextChoices, axis=1), np.sqrt(nsubjects) )
    ax.axhline(y=50, color='gray', linestyle=':')
    if 'peanut' in contexts: # vertical. Was tested first.
      subtrials = [list(range(16)), list(range(32, 48))]
      subtrials = [element for sublist in subtrials for element in sublist]
    else:
      subtrials = [list(range(16, 32)), list(range(48, 64))]
      subtrials = [element for sublist in subtrials for element in sublist]

    # First block in this context.
    trials = np.reshape(subtrials[:len(subtrials)//2], (bunch_factor, -1), order='F')
    trials = np.mean(trials, axis=0)
    ax.scatter(trials, meanChoices[:len(meanChoices)//2], color=colour)
    ax.errorbar(trials, meanChoices[:len(meanChoices)//2], seChoices[:len(seChoices)//2], color=colour)

    # Second block in this context.
    trials = np.reshape(subtrials[len(subtrials)//2:], (bunch_factor, -1), order='F')
    trials = np.mean(trials, axis=0)
    ax.scatter(trials, meanChoices[len(meanChoices)//2:], color=colour)
    handle = ax.errorbar(trials, meanChoices[len(meanChoices)//2:], seChoices[len(seChoices)//2:], color=colour)

    ax.set_ylabel('Correct first room choice (%)')
    ax.set_ylim([40,100])
    ax.set_xlabel('Trials')
    ax.set_title('Human behaviour, day 1')
    return contextChoices, handle

#-------------------------------------------------------------------------------

def plot_cov_scanner_context(ax, allPlayerLearningData, contexts, colour, bunch_factor=1, title=''):
    """Plot the covariance learning results for scanner session, by context."""
    metric = 'humanCorrectChoices'
    whichSession = cs.SESSION_SCANNER
    nsubjects = len(allPlayerLearningData[whichSession])
    # if you want the human choices only, only average across subs over those
    allChoices = [allPlayerLearningData[whichSession][i]["roomChoicesData"][metric] for i in range(len(allPlayerLearningData[whichSession]))]

    contextChoices = []
    for subject_idx, subjectChoices in enumerate(allChoices):
      contextMaps = allPlayerLearningData[whichSession][subject_idx]["contextMapList"]
      contextMaps = insert_rests(contextMaps, cs.SESSION_SCANNER)
      contextTrials = [subjectChoices[trial_idx] for trial_idx, context in enumerate(contextMaps) if context[10:] in contexts]
      contextTrials = [i for i in contextTrials if ~np.isnan(np.asarray(i, dtype=np.float))]  # remove nans.
      while len(contextTrials) != 24:
        contextTrials.append(np.nan)
      contextTrials = np.asarray(contextTrials, dtype=np.float)
      contextChoices.append(contextTrials)

    contextChoices = np.asarray(contextChoices, dtype=np.float)
    contextChoices = np.transpose(contextChoices)

    # Average across adjacent trials using a bunch_factor
    contextChoices = np.reshape(contextChoices, (bunch_factor, -1, nsubjects), order='F')
    contextChoices = np.nanmean(contextChoices, axis=0)

    # Draw context colour rectangles
    errorboxes = [
      [Rectangle((1, 0), 32, 100)],
      [Rectangle((33, 0), 32, 100)],
      [Rectangle((33+32, 0), 32, 100)],
    ]
    edgecolor = 'white'
    facecolors = ['gold', 'gold', 'gold']
    alphas = [0.1, 0.25, 0.4]
    for box, facecolor, alpha in zip(errorboxes, facecolors, alphas):
      pc = PatchCollection(box, facecolor=facecolor, alpha=alpha,
                         edgecolor=facecolor)
      ax.add_collection(pc)

    # mean and sem
    meanChoices = 100 * np.nanmean(contextChoices, axis=1)
    seChoices = 100 * np.divide(np.nanstd(contextChoices, axis=1), np.sqrt(nsubjects) )
    ax.axhline(y=50, color='gray', linestyle=':')
    # Manually increase span over which trials are plotted to reflect interleaved contexts.
    subtrials = np.linspace(bunch_factor/2, 2*2* bunch_factor*meanChoices.size, meanChoices.size)

    # Plot the learning.
    ax.scatter(subtrials, meanChoices, color=colour)
    handle = ax.errorbar(subtrials, meanChoices, seChoices, color=colour)

    ax.set_ylim([40,100])
    ax.set_xlabel('Trials')
    ax.set_title('Human behaviour, day 2')

    return contextChoices, handle

#-------------------------------------------------------------------------------

def moved_which_way(current_position: List[float], previous_position: List[float]) -> str:
  """Determine the direction of travel."""
  if current_position[0] > previous_position[0]: # moved to right.
    return 'right'
  elif current_position[0] < previous_position[0]: # moved left.
    return 'left'
  elif current_position[1] > previous_position[1]: # moved up.
    return 'up'
  elif current_position[1] < previous_position[1]: # moved down.
    return 'down'
  else:
    return 'no_move'

#-------------------------------------------------------------------------------

def format_room_choices(ax, allPlayerLearningData, title=''):
    """Format the covariance learning results for scanner session."""
    metric = 'humanCorrectChoices'
    whichSession = cs.SESSION_SCANNER
    nsubjects = len(allPlayerLearningData[whichSession])
    # if you want the human choices only, only average across subs over those
    allChoices = [allPlayerLearningData[whichSession][i]["roomChoicesData"][metric] for i in range(len(allPlayerLearningData[whichSession]))]
    allChoices = np.asarray(allChoices, dtype=np.float)
    allChoices = allChoices[~np.isnan(allChoices)]  # remove nans.
    allChoices = np.reshape(allChoices, (-1, nsubjects), order='F')
    return allChoices

#-------------------------------------------------------------------------------

def reformatControlStrings(controlStateTransitions):
    for trial in range(len(controlStateTransitions)):
        times = []
        controls = []
        if trial is not None:
            for entry in range(len(controlStateTransitions[trial])):
                time, control = controlStateTransitions[trial][entry].split(" ")
                time = float(time.replace(",","."))
                times.append(time)
                controls.append(control)
            controlStateTransitions[trial] = {"timestamps":times, "controlmode":controls}
    return controlStateTransitions

#-------------------------------------------------------------------------------

def defineFilepaths():
    # Specify paths for data and generated figures
    # Note that the filepath has been updated to the processed data one because otherwise we don't include the 2 manually combined subject behavioural logs
    savepath = '/Users/hsheahan/Documents/Four_rooms/2d_behavioural_data/generated_figures/'
    #filepath = '/Users/hannahsheahan/Documents/Postdoc/Experiments/MartiniTask/2D_version/Data/scanner/behaviour/originalData/'
    filepath = '/Users/hsheahan/Documents/Four_rooms/2d_behavioural_data/hannah_preprocessed/'
    filepaths = [filepath + 'session_1/', filepath + 'session_2/']

    session1 = [f for f in os.listdir(filepaths[0]) if os.path.isfile(os.path.join(filepaths[0], f))]
    session2 = [f for f in os.listdir(filepaths[1]) if os.path.isfile(os.path.join(filepaths[1], f))]
    session1 = [session1[file] for file in range(len(session1)) if ("_ignore"  not in session1[file] and "DS_"  not in session1[file])]
    session2 = [session2[file] for file in range(len(session2)) if ("_ignore"  not in session2[file] and "DS_"  not in session2[file])]
    sessions = [session1, session2]
    nSubjects = len(session1) if len(session1) > len(session2) else len(session2)

    return sessions, nSubjects, filepaths, savepath

#-------------------------------------------------------------------------------

def find_train_test_deltatime(sessions):
  """Find time between the training and scanning sessions for each participant."""
  dt = [[None for i in range(len(sessions[cs.SESSION_TRAINING]))] for j in range(2)]
  for i in range(len(sessions[cs.SESSION_TRAINING])):

    # Find the date and time for the training session.
    file = sessions[cs.SESSION_TRAINING][i]
    subject = file[-14:-11]
    trainingdate = file[9:23]
    traindatetime = datetime.strptime(trainingdate, '%d-%m-%y_%H-%M')
    dt[cs.SESSION_TRAINING][i] = traindatetime

    # Find the associated date and time for the scanning session for that subjects.
    for j in range(len(sessions[cs.SESSION_SCANNER])):
      scannerfile = sessions[cs.SESSION_SCANNER][j]
      scannersubject = scannerfile[-14:-11]
      if scannersubject == subject:
        testingdate = scannerfile[9:23]
        testdatetime = datetime.strptime(testingdate, '%d-%m-%y_%H-%M')
        dt[cs.SESSION_SCANNER][i] = testdatetime

  timediff = []
  for i in range(len(sessions[cs.SESSION_TRAINING])):
    diff = dt[cs.SESSION_SCANNER][i] - dt[cs.SESSION_TRAINING][i]
    hoursdiff = (diff.days * 24) + (diff.seconds / (60 * 60))
    timediff.append(hoursdiff)
  timediff = [diff for diff in timediff if diff > 0.01] # Remove the spurious 'zero' diffs.
  mean = np.mean(timediff)
  err = np.std(timediff) / np.sqrt(len(timediff))
  print('On average train and test (scanner) sessions were: {:.1f} +- {:.1f} hours apart (mean +- SEM).'.format(mean, err))

  return timediff

#-------------------------------------------------------------------------------

def compute_duration_per_agent(raw_data, num_subjects: int, allowed_states: Tuple[int, ...]) -> Tuple[List[float], List[float]]:
  """Calculate how much time is spent under human vs agent control per trial."""
  durations = []
  players_human_time = []
  players_computer_time = []
  for player_id in range(num_subjects):
    num_trials = len(raw_data[cs.SESSION_SCANNER][player_id]['allTrialData'])
    human_durations = []
    computer_durations = []
    for trial in range(num_trials):
      human_durations_per_trial = []
      computer_durations_per_trial = []
      previous_game_state = cs.STATE_GO
      control_idx = 0
      control_order = raw_data[cs.SESSION_SCANNER][player_id]['allTrialData'][trial]['controlStateOrder']
      if control_order:
        for i in range(2):  # Ensure there are enough switches in control order to handle the data.
          control_order.append(control_order[0])
          control_order.append(control_order[1])
      times = raw_data[cs.SESSION_SCANNER][player_id]['allTrialData'][trial]['timeStepTrackingData']
      first_timestep, time = 0., 0. # Default
      first_step = True
      for step_ind, step in enumerate(times):
        game_state = raw_data[cs.SESSION_SCANNER][player_id]['allTrialData'][trial]['stateTransitions'][step_ind]
        if step[0] != 'T':
          if (int(game_state) in allowed_states):
            time = step[:6]
            time = time.replace(',', '.')
            time = float(time)
            if first_step:
              first_timestep = time
              first_step = False
          duration = time - first_timestep
          if int(game_state) == cs.STATE_SHOWREWARD and int(previous_game_state) != cs.STATE_SHOWREWARD:
            # A control transition has occurred.
            if control_idx < len(control_order):
              if control_order[control_idx] == 'Human':
                human_durations_per_trial.append(duration)
              else:
                computer_durations_per_trial.append(duration)
              control_idx += 1
              first_timestep = copy.copy(time)
        previous_game_state = copy.copy(game_state)
      human_durations.append(np.sum(human_durations_per_trial))
      computer_durations.append(np.sum(computer_durations_per_trial))
    players_human_time.append(np.nanmean([i for i in human_durations if i]))
    players_computer_time.append(np.nanmean([i for i in computer_durations if i]))

  return players_human_time, players_computer_time

#-------------------------------------------------------------------------------

def player_in_which_square(xpos: float, zpos: float) -> int:
    """Takes a momentary player x,z position and outputs a gridsquare index.

    The player can only be in one square at a time.
    """
    squarelabel = 1
    squarepositions = [None]*(9*9)
    count = 0
    deltaSquare = 1

    for i in np.linspace(-4.5,3.5,9): # Note that these are the 'lower-left' corner positions for each of the squares
        for j in np.linspace(-4.5,3.5,9):
            # this is a list carrying coordinates of each square in the environment. Each index is the label of the square.
            squarepositions[count] = [i,j]
            envx = squarepositions[count][0]
            envz = squarepositions[count][1]

            if ( xpos >= envx ) and (xpos < (envx + deltaSquare)):
                if (zpos >= envz ) and (zpos < (envz + deltaSquare)):
                    squarelabel = count

            count += 1   # ready for evaluating the next square

    return squarelabel

#-------------------------------------------------------------------------------

def compute_occupancy_and_transitions(
    raw_data,
    num_subjects: int,
    allowed_states: Tuple[int, ...],
    *,
    vertical_maps: Tuple[str, ...],
    horizontal_maps: Tuple[str, ...],
) -> Tuple[np.ndarray, np.ndarray, Mapping[str, List[int]], Mapping[str, List[int]]]:
  """Calculates the duration and transitions per gridsquare in each context."""
  gridsize = 9
  dataFrequency = 0.04
  horizontal_transitions = {
      'right':[0]*(gridsize*gridsize),
      'left':[0]*(gridsize*gridsize),
      'up':[0]*(gridsize*gridsize),
      'down':[0]*(gridsize*gridsize),
      'no_move':[0]*(gridsize*gridsize),
  }
  vertical_transitions = {
      'right':[0]*(gridsize*gridsize),
      'left':[0]*(gridsize*gridsize),
      'up':[0]*(gridsize*gridsize),
      'down':[0]*(gridsize*gridsize),
      'no_move':[0]*(gridsize*gridsize),
  }
  vertical_heatmap_counts = [0]*(gridsize*gridsize)
  horizontal_heatmap_counts = [0]*(gridsize*gridsize)

  timestep_positions = []
  for player_id in range(num_subjects):
    player_positions = []
    num_trials = len(raw_data[cs.SESSION_SCANNER][player_id]['allTrialData'])
    for trial in range(num_trials):
      trial_positions = []
      positions = raw_data[cs.SESSION_SCANNER][player_id]['allTrialData'][trial]['timeStepTrackingData']
      for step_ind, step in enumerate(positions):
        game_state = raw_data[cs.SESSION_SCANNER][player_id]['allTrialData'][trial]['stateTransitions'][step_ind]
        if step[0] != 'T':
          if (int(game_state) in allowed_states):
            position = step.replace(',', '.')
            position = position.split(" ")
            position = [float(i) for i in position]
            position = position[1:] # ignore the time info and just retain position.
            square_label = player_in_which_square(position[0], position[1])
            if trial_positions:
              transition = moved_which_way(position, trial_positions[-1])
            else:
              prev_square_label = copy.copy(square_label)
              transition = 'no_move'
            trial_positions.append(position)

            if raw_data[cs.SESSION_SCANNER][player_id]['allTrialData'][trial]['mapName'] in vertical_maps:
              vertical_heatmap_counts[square_label] +=1
              vertical_transitions[transition][prev_square_label] += 1
            elif raw_data[cs.SESSION_SCANNER][player_id]['allTrialData'][trial]['mapName'] in horizontal_maps:
              horizontal_heatmap_counts[square_label] +=1
              horizontal_transitions[transition][prev_square_label] += 1
            prev_square_label = copy.copy(square_label)

  # Convert to units of time (s).
  horizontal_heatmap_counts = [float(horizontal_heatmap_counts[i])*dataFrequency for i in range(len(horizontal_heatmap_counts))]
  horizontal_counts = np.array(horizontal_heatmap_counts).reshape(gridsize,gridsize)
  vertical_heatmap_counts = [float(vertical_heatmap_counts[i])*dataFrequency for i in range(len(vertical_heatmap_counts))]
  vertical_counts = np.array(vertical_heatmap_counts).reshape(gridsize,gridsize)

  return horizontal_counts, vertical_counts, horizontal_transitions, vertical_transitions

#-------------------------------------------------------------------------------

def normalise_transitions(horizontal_transitions: Mapping[str, List[int]], vertical_transitions: Mapping[str, List[int]]) -> Mapping[str, Mapping[str, List[float]]]:
  """Calculate transition probabilities for each grid-square and context."""
  gridsize = 9
  transition_probs = {'horizontal':[], 'vertical':[]}
  for context in ['horizontal', 'vertical']:
    normalised_transitions = {
      'right':[0]*(gridsize*gridsize),
      'left':[0]*(gridsize*gridsize),
      'up':[0]*(gridsize*gridsize),
      'down':[0]*(gridsize*gridsize),
      'no_move':[0]*(gridsize*gridsize),
    }
    if context == 'horizontal':
      transitions = copy.copy(horizontal_transitions)
    elif context == 'vertical':
      transitions = copy.copy(vertical_transitions)
    else:
      raise ValueError('Unrecognised context.')

    for square_label in range(gridsize*gridsize):
      total_transition_from_square = 0
      for key in ['right', 'left', 'down', 'up']:
        total_transition_from_square += transitions[key][square_label]

      for key in ['right', 'left', 'down', 'up']:
        if total_transition_from_square == 0:
          normalised_transitions[key][square_label] = 0
        else:
          normalised_transitions[key][square_label] = float(transitions[key][square_label]) / float(total_transition_from_square)
    transition_probs[context] = normalised_transitions

  return transition_probs

#-------------------------------------------------------------------------------

def xy_direction(label):
  """Map the string label to a direction for plotting."""
  mapping = {'right':(1,0), 'left':(-1,0), 'up':(0,-1), 'down':(0,1), 'none':(0,0)}
  return mapping[label]

#-------------------------------------------------------------------------------

def vectorise_transitions(transition_probs: Mapping[str, Mapping[str, List[float]]]) -> Mapping[str, List[float]]:
  """Compute the vector of the summed transitions per gridsquare."""
  gridsize = 9
  direction_label = {0:'right', 1:'left', 2:'up', 3:'down', 4:'none'}
  transition_vectors = {'horizontal':[], 'vertical':[]}

  for context in ['horizontal', 'vertical']:
    direction_vector = [0] * (gridsize*gridsize)
    for square_label in range(gridsize*gridsize):
      normed_transitions = transition_probs[context]
      all_directions = np.array([normed_transitions[key][square_label] for key in ['right', 'left', 'up', 'down']])
      if np.all(np.array([i==all_directions[0] for i in all_directions])):
        direction = 4
      else:
        direction = np.argmax(all_directions)
      vector = all_directions[0]*np.array([1., 0.]) + all_directions[1]*np.array([-1., 0.]) + all_directions[2]*np.array([0.,-1.]) + all_directions[3]*np.array([0.,1.])
      direction_vector[square_label] = vector
    transition_vectors[context] = direction_vector
  return transition_vectors

#-------------------------------------------------------------------------------

def plot_transition_heatmap(horiz_counts, vert_counts, transition_vectors: Mapping[str, Any]) -> None:
  """Plot the occupancy heatmap and vector transitions."""
  plt.figure(figsize=(10,4))
  plt.subplot(1,2,1)
  axes = sns.heatmap(np.rot90(horiz_counts/(27*48), k=1), linewidths=.5, square=True, yticklabels=False,
                    xticklabels=False, cmap=cm.gist_heat_r, cbar_kws={"label":"Time spent per trial (sec)"})
  plt.title('Horizontal contexts')

  count = 0
  x,y = np.meshgrid(np.arange(0.5,9.5),np.arange(0.5,9.5))
  test = np.asarray([transition_vectors['horizontal']]).reshape((9,9,2), order='F')
  u, v = test[:,:,0], test[:,:,1]
  u[u==0] = np.nan
  axes.quiver(x,y,u,v, scale=7.)

  plt.subplot(1,2,2)
  axes = sns.heatmap(np.rot90(vert_counts/(27*48), k=1), linewidths=.5, square=True, yticklabels=False, xticklabels=False, cmap=cm.gist_heat_r,
                    cbar_kws={"label":"Time spent per trial (sec)"})
  plt.title('Vertical contexts')

  x,y = np.meshgrid(np.arange(0.5,9.5),np.arange(0.5,9.5))
  test = np.asarray(transition_vectors['vertical']).reshape((9,9,2), order='F')
  u, v = test[:,:,0], test[:,:,1]
  u[u==0] = np.nan
  axes.quiver(x,y,u,v, scale=7.)
  axes.set_aspect('equal')

#-------------------------------------------------------------------------------

def visualise_training_metadata(allPlayerLearningData, sessions, savepath, plotcolours, saveFig):
  """Plot across-subject metadata for the training session."""

  # Histogram of age.
  plt.figure(figsize=(12,3))
  subjectAges = [float(allPlayerLearningData[cs.SESSION_SCANNER][i]["participantAge"]) for i in range(len(allPlayerLearningData[cs.SESSION_SCANNER])) if allPlayerLearningData[cs.SESSION_SCANNER][i] is not None]
  plt.subplot(1,4,1)
  plt.hist(subjectAges, color=plotcolours[cs.SESSION_TRAINING])
  plt.xlabel('Age')
  plt.ylabel('Number of subjects')
  print('Mean age: {}, +- {} std'.format(np.mean(subjectAges), np.std(subjectAges)))

  # Total practice scores.
  trainingScore = [(allPlayerLearningData[cs.SESSION_TRAINING][i]["totalScore"]) for i in range(len(allPlayerLearningData[cs.SESSION_TRAINING])) if allPlayerLearningData[cs.SESSION_TRAINING][i] is not None]
  plt.subplot(1,4,2)
  plt.hist(trainingScore, color=plotcolours[cs.SESSION_TRAINING])
  plt.xlabel('Training score')

  # Determine how much time was between the two experiment sessions for pairs of datafiles by the same subject
  timeBetweenSessions = find_train_test_deltatime(sessions)
  plt.subplot(1,4,3)
  plt.hist(timeBetweenSessions, bins=15, color=plotcolours[cs.SESSION_TRAINING])
  plt.xlabel('Time between training and test sessions')

  # Practice duration.
  practiceDuration = [(allPlayerLearningData[cs.SESSION_TRAINING][i]["experimentDuration"]) for i in range(len(allPlayerLearningData[cs.SESSION_TRAINING])) if allPlayerLearningData[cs.SESSION_TRAINING][i] is not None]
  plt.subplot(1,4,4)
  plt.hist(practiceDuration, color=plotcolours[cs.SESSION_TRAINING])
  plt.xlabel('Practice duration (s)')

  plt.suptitle('Histograms of Training Session Metadata')
  if saveFig:
    plt.savefig('Training_metadata.pdf', bbox_inches='tight')
    files.download("Training_metadata.pdf")

#-------------------------------------------------------------------------------

def visualise_testing_metadata(allPlayerLearningData, sessions, savepath, plotcolours, saveFig):
  """Plot the across-subject metadata for the scanning session."""
  # how many subjects on each ordering of the stimuli
  experimentVersions = [(allPlayerLearningData[cs.SESSION_SCANNER][i]["experimentVersion"][16:])
   for i in range(len(allPlayerLearningData[cs.SESSION_SCANNER])) if allPlayerLearningData[cs.SESSION_SCANNER][i] is not None]
  for i in range(len(experimentVersions)):
    version = experimentVersions[i]
    version = version.replace("cheese", "c")
    version = version.replace("mushroom", "m")
    version = version.replace("pineapple","p")
    experimentVersions[i] = version

  plt.subplots_adjust(wspace=0.5)
  fig, axes = plt.subplots(1,3, figsize=(14,4))
  axes[0].hist(experimentVersions, color=plotcolours[cs.SESSION_SCANNER])
  axes[0].set_xlabel('Block test ordering')
  axes[0].set_ylabel('Number of subjects')
  axes[0].set_title('Different reward type orderings')

  # Plot an example trial sequence for a single sub to see the switching
  numericalMapSequence = []
  randsub = random.randint(0,len(allPlayerLearningData[cs.SESSION_SCANNER])-1)
  mapSequence = allPlayerLearningData[cs.SESSION_SCANNER][randsub]["mapName"][7:]
  for map in mapSequence:
    if "banana" in map:
      numericalMapSequence.append(1)
    elif "mushroom" in map:
      numericalMapSequence.append(2)
    elif "pineapple" in map:
      numericalMapSequence.append(3)
    elif "avocado" in map:
      numericalMapSequence.append(4)
    elif "watermelon" in map:
      numericalMapSequence.append(5)
    elif "cheese" in map:
      numericalMapSequence.append(6)
    else:
      numericalMapSequence.append(None)

  axes[1].plot(numericalMapSequence, color=plotcolours[cs.SESSION_SCANNER])
  for i in range(len(numericalMapSequence)):
    if numericalMapSequence[i] is None:
        axes[1].axvline(x=i, color='grey', linestyle=':')
  axes[1].set_yticks([i for i in range(1,7)])
  axes[1].set_yticklabels(['banana','mushroom','pineapple','avocado','watermelon','cheese'], rotation=60)
  axes[1].set_xlabel('Trial')
  axes[1].set_title("Context sequence for example subject #{0}".format(randsub))

  # dist of test duration
  testDuration = [(allPlayerLearningData[cs.SESSION_SCANNER][i]["experimentDuration"]) for i in range(len(allPlayerLearningData[cs.SESSION_SCANNER])) if allPlayerLearningData[cs.SESSION_SCANNER][i] is not None]
  axes[2].hist(testDuration, color=plotcolours[cs.SESSION_SCANNER])
  axes[2].set_xlabel('Scanner test duration (s)')
  axes[2].set_ylabel('Number of subjects')

  fig.suptitle('Scanning session metadata')
  if saveFig:
    plt.savefig('Testing_metadata.pdf', bbox_inches='tight')
    files.download("Testing_metadata.pdf")

#-------------------------------------------------------------------------------

def plot_shaded_curve(ax, trials, means, sems, colourname='dodgerblue', title='', linelabel=''):
    """Plot a pretty shaded learning curve."""
    # Offset the x-values by the number of trials at the start that are just for setup
    offset = 0
    for i in range(len(means)):
        if not math.isnan(means[i]):
            break
        offset += 1
    means = means[offset:]
    sems = sems[offset:]
    trials = trials[offset:]-trials[offset]

    # plot the rest breaks
    for i in range(len(means)):
        if math.isnan(means[i]):
            ax.axvline(x=trials[i], color='lightgray', linestyle=':')

    # plot the actual data
    ax.plot(trials, means, color=colourname, label=linelabel)
    ax.fill_between(trials, means-sems, means+sems, color=colourname, alpha=0.25)
    ax.set_xlim([0, np.max(trials)])
    ax.set_xlabel('Trial')
    ax.set_title(title)

#-------------------------------------------------------------------------------

def plot_scores_learning(
    ax,
    allPlayerLearningData,
    whichSession,
    savepath,
    colours,
    bunch_factor=1,
    plot_individuals=False,
    saveFig=False,
    title='',
    ylim=(0,32),
    ylabel='Trial score',
):
  """Plots trial score across the session."""
  nsubjects = len(allPlayerLearningData[whichSession])
  if plot_individuals:
    for i in range(nsubjects):
      scores = allPlayerLearningData[whichSession][i]["allScores"]
      ax.plot(scores, alpha=0.1)

  subScores = [allPlayerLearningData[whichSession][i]["allScores"]
    for i in range(len(allPlayerLearningData[whichSession]))]
  subScores = np.asarray(subScores, dtype=np.float)
  subScores = subScores[~np.isnan(subScores)]  # remove nans.

  # Average across adjacent trials using a bunch_factor
  subScores = np.reshape(subScores, (bunch_factor, -1, nsubjects), order='F')
  subScores = np.mean(subScores, axis=0)
  # Mean and sem
  meanScores =  np.nanmean(subScores, axis=1)
  seScores = np.nanstd(subScores, axis=1) / np.sqrt(nsubjects)
  subtrials = np.linspace(1,bunch_factor*meanScores.size, meanScores.size)

  # Plot the learning.
  ax.scatter(subtrials, meanScores, color=colours[whichSession])
  ax.errorbar(subtrials, meanScores, seScores, color=colours[whichSession])
  ax.set_title(title)
  ax.set_ylabel(ylabel)
  ax.set_ylim(ylim)
  ax.set_xlabel('Trials')
  return subScores

#-------------------------------------------------------------------------------

def plot_duration_learning(
    ax,
    allPlayerLearningData,
    whichSession,
    savepath,
    colours,
    bunch_factor=1,
    plot_individuals=False,
    saveFig=False,
    title='',
    ylim=(0,32),
    ylabel='Trial duration (s)',
):
  """Plot every subject's trial duration across the session."""
  nsubjects = len(allPlayerLearningData[whichSession])
  if plot_individuals:
    for i in range(nsubjects):
      scores = allPlayerLearningData[whichSession][i]["timesToGoal"]
      if whichSession != cs.SESSION_SCANNER:
        scores = [i-10 for i in scores]
      ax.plot(scores, alpha=0.1)

  subScores = [allPlayerLearningData[whichSession][i]["timesToGoal"]
    for i in range(len(allPlayerLearningData[whichSession]))]
  subScores = np.asarray(subScores, dtype=np.float)
  subScores = subScores[~np.isnan(subScores)]  # remove nans.

  # Average across adjacent trials using a bunch_factor
  subScores = np.reshape(subScores, (bunch_factor, -1, nsubjects), order='F')
  subScores = np.mean(subScores, axis=0)
  # Mean and sem
  meanScores =  np.nanmean(subScores, axis=1)
  seScores = np.nanstd(subScores, axis=1) / np.sqrt(nsubjects)
  subtrials = np.linspace(1,bunch_factor*meanScores.size, meanScores.size)

  if whichSession != cs.SESSION_SCANNER:
    meanScores -= 10  # Correct for a data preprocessing error. Training and test sessions have different durations.

  # Plot the learning.
  ax.scatter(subtrials, meanScores, color=colours[whichSession])
  ax.errorbar(subtrials, meanScores, seScores, color=colours[whichSession])
  ax.set_title(title)
  ax.set_ylabel(ylabel)
  ax.set_ylim(ylim)
  ax.set_xlabel('Trials')
  return subScores

#-------------------------------------------------------------------------------


from __future__ import print_function
import os
from os.path import dirname, join, abspath, isdir
import sys
import json
import requests
try:
    from urlparse import urljoin
except:
    from urllib.parse import urljoin        # python 3.x

# http://python-redmine.readthedocs.org/
from redmine import Redmine

if __name__=='__main__':
    SRC_ROOT = dirname(dirname(abspath(__file__)))
    sys.path.append(SRC_ROOT)

from datetime import datetime
from utils.msg_util import *

class RedmineIssueDownloader:
    """
    For a given Redmine project, download the issues in JSON format
    """

    #TIME_FORMAT_STRING = '%Y-%m%d-%H%M'
    TIME_FORMAT_STRING = '%Y-%m%d'

    # Redmine tickets are written to JSON files with the naming convention  "(issue id).json"
    # For file sorting, preceding zeros are tacked on.
    #   Example: If  ZERO_PADDING_LEVEL=5:
    #               issue #375 is written to file "00375.json"
    #               issue #1789 is written to file "01789.json"
    #               issue #2 is written to file "00002.json"
    #
    # If your issue numbers go beyond 99,999 then increase the ZERO_PADDING_LEVEL
    #
    ZERO_PADDING_LEVEL = 5

    def __init__(self, redmine_server, redmine_api_key, project_name_or_identifier, issues_base_directory, **kwargs):
        """
        Constructor

        :param redmine_server: str giving the url of the redmine server.  e.g. https://redmine.myorg.edu/
        :param redmine_api_key: str with a redmine api key
        :param project_name_or_identifier: str or int with either the redmine project id or project identifier
        :param issues_base_directory: str, directory to download the redmine issues in JSON format.  Directory will be crated
        :param specific_tickets_to_download: optional, list of specific ticket numbers to download. e.g. [2215, 2216, etc]
        """
        self.redmine_server = redmine_server
        self.redmine_api_key = redmine_api_key
        self.project_name_or_identifier = project_name_or_identifier
        self.issues_base_directory = issues_base_directory
        self.issue_status = kwargs.get('issue_status', '*') # values 'open', 'closed', '*'

        self.specific_tickets_to_download = kwargs.get('specific_tickets_to_download', None)

        self.redmine_conn = None
        self.redmine_project = None

        self.issue_dirname = join(self.issues_base_directory\
                                ,  datetime.today().strftime(RedmineIssueDownloader.TIME_FORMAT_STRING)\
                                )

        self.setup()

    def setup(self):
        self.connect_to_redmine()
        if not isdir(self.issue_dirname):
            os.makedirs(self.issue_dirname)
            msgt('Directory created: %s' % self.issue_dirname)

    def connect_to_redmine(self):
        self.redmine_conn = Redmine(self.redmine_server, key=self.redmine_api_key)
        self.redmine_project = self.redmine_conn.project.get(self.project_name_or_identifier)
        msg('Connected to server [%s] project [%s]' % (self.redmine_server, self.project_name_or_identifier))

    def get_issue_count(self):
        msgt('get_issue_count')

        issue_query_str = 'issues.json?project_id=%s&limit=1&status_id=%s' \
                            % (self.project_name_or_identifier, self.issue_status)

        url = urljoin(self.redmine_server, issue_query_str)

        msg('Issue count url: %s' % url)

        # Note: Auth purposely uses the API KEY "as a username with a random password via HTTP Basic authentication"
        #   from: http://www.redmine.org/projects/redmine/wiki/Rest_api
        #
        auth = (self.redmine_api_key, 'random-pw')
        r = requests.get(url, auth=auth)
        if not r.status_code == 200:
            msgt('Error!')
            msg(r.text)
            raise Exception("Request for issue count failed! Status code: %s\nUrl: %s\nAuth:%s" % (r.status_code, url, auth))

        msg('Convert result to JSON')
        try:
            data = r.json()     # Let it blow up
        except:
            msgt('Error!')
            msg('Data from request (as text): %s' % r.text)
            raise Exception('Failed to convert issue count data to JSON.\nUrl: %s\nAuth:%s" % (url, auth)')

        if not data.has_key('total_count'):
            msgx('Total count not found in data: \n[%s]' % data)

        return data['total_count']

        """
from __future__ import print_function
import requests

project_id = 'dvn'
redmine_api_key = 'some-key'
url = 'https://redmine.hmdc.harvard.edu/issues.json?project_id=%s&limit=1' % project_id

#---------------------
# Alternative 1
#---------------------
auth = (redmine_api_key, 'random-pw')
r = requests.get(url, auth=auth)
print (r.text)
print (r.status_code)
data = r.json()
print (data['total_count'])

#---------------------
# Alternative 2
#---------------------
url2 = '%s&key=%s' % (url, redmine_api_key)
r = requests.get(url2)
print (r.text)
print (r.status_code)
data = r.json()
print (data['total_count'])

"""

    def write_issue_list(self, issue_fname, issue_dict):
        if issue_fname is None or not type(issue_dict) == dict:
            msgx('ERROR: write_issue_list, issue_fname is None or issue_dict not dict')
            return
        fh = open(issue_fname, 'w')
        fh.write(json.dumps(issue_dict))
        fh.close()
        msg('file updated: %s' % issue_fname)

    def show_project_info(self):
        msg(self.redmine_project._attributes)

    def download_tickets2(self):
        """
        fyi: Retrieving total count via regular api, not python redmine package
        """
        issue_dict = {}
        issue_fname = join(self.issue_dirname, 'issue_list.json')
        msg('Gathering issue information.... (may take a minute)')

        ticket_cnt = self.get_issue_count()

        RECORD_RETRIEVAL_SIZE = 100

        num_loops = ticket_cnt / RECORD_RETRIEVAL_SIZE
        extra_recs = ticket_cnt % RECORD_RETRIEVAL_SIZE
        if extra_recs > 0:
            num_loops+=1
        #num_loops=3
        msg('num_loops: %d' % num_loops)
        msg('extra_recs: %d' % extra_recs)

        cnt = 0
        for loop_num in range(0, num_loops):
            start_record = loop_num * RECORD_RETRIEVAL_SIZE
            end_record = (loop_num+1) * RECORD_RETRIEVAL_SIZE

            msgt('Retrieve records via idx (skip last): %s - %s' % (start_record, end_record))

            # limit of 100 is returning 125
            rec_cnt = 0
            for item in self.redmine_conn.issue.filter(project_id=self.project_name_or_identifier, status_id=self.issue_status, sort='id', offset=start_record)[:RECORD_RETRIEVAL_SIZE]: #, limit=RECORD_RETRIEVAL_SIZE):   #[start_record:end_record]
                rec_cnt +=1
                cnt +=1
                msg('(%s) %s - %s' % (rec_cnt, item.id, item.subject))

                if self.specific_tickets_to_download is not None:
                    # only download specific tickets
                    #
                    if item.id in self.specific_tickets_to_download:
                        self.save_single_issue(item)
                        issue_dict[self.pad_issue_id(item.id)] = item.subject
                    continue    # go to next item
                else:
                    # Get all tickets
                    #
                    self.save_single_issue(item)
                    issue_dict[self.pad_issue_id(item.id)] = item.subject
                if rec_cnt == RECORD_RETRIEVAL_SIZE:
                    break
                #continue
                #self.save_single_issue(item)
            self.write_issue_list(issue_fname, issue_dict)

    def pad_issue_id(self, issue_id):
        if issue_id is None:
            msgx('ERROR. pad_issue_id. The "issue_id" is None')

        return ('%s' % issue_id).zfill(self.ZERO_PADDING_LEVEL)

    def save_single_issue(self, single_issue):
        """
        Write a single issue object to a file using JSON format

        :param single_issue: Issue object
        """
        if single_issue is None:
            msgx('ERROR. download_single_issue. The "single_issue" is None')

        ## FIX: Expensive adjustment -- to pull out full relation and journal info
        json_str = self.get_single_issue(single_issue.id)       # another call to redmine

        #json_str = json.dumps(single_issue._attributes, indent=4)

        fullpath = join(self.issue_dirname, self.pad_issue_id(single_issue.id) + '.json')
        open(fullpath, 'w').write(json_str)
        msg('Ticket retrieved: %s' % fullpath)

    def process_files(self, issues_dirname=None):
        if issues_dirname is None:
            issues_dirname = self.issue_dirname

        tracker_info = []
        status_info = []
        priority_info = []

        fnames = [x for x in os.listdir(issues_dirname) if x.endswith('.json')]
        for fname in fnames:
            content = open(join(issues_dirname, fname), 'rU').read()
            d = json.loads(content)

            # Tracker Info
            tracker = d.get('tracker', None)
            if tracker:
                tracker_str = '%s|%s' % (tracker['id'], tracker['name'])
                if not tracker_str in tracker_info:
                    tracker_info.append(tracker_str)
            # Status Info
            status = d.get('status', None)
            if status:
                status_str = '%s|%s' % (status['id'], status['name'])
                if not status_str in status_info:
                    status_info.append(status_str)

            # Priority Info
            priority = d.get('priority', None)
            if priority:
                priority_str = '%s|%s' % (priority['id'], priority['name'])
                if not priority_str in priority_info:
                    priority_info.append(priority_str)
        #print d.keys()
        msg(tracker_info)
        msg(status_info)
        msg(priority_info)

    def get_single_issue(self, issue_id):
        """
        Download a single issue

        :param ticket_id: int of issue id in redmine
        :returns: json string with issue information
        """
        # test using .issue.get
        issue = self.redmine_conn.issue.get(issue_id, include='children,journals,watchers,relations')
        json_str = json.dumps(issue._attributes, indent=4)
        msg('Issue retrieved: %s' % issue_id)
        return json_str

if __name__=='__main__':
    from settings.base import REDMINE_SERVER, REDMINE_PROJECT_ID, REDMINE_API_KEY, REDMINE_ISSUES_DIRECTORY
    #rn = RedmineIssueDownloader(REDMINE_SERVER, REDMINE_API_KEY, 'dvn', REDMINE_ISSUES_DIRECTORY)
    #Only import some specific tickets
    #kwargs = dict(specific_tickets_to_download=[1371, 1399, 1843, 2214, 2215, 2216, 3362, 3387, 3397, 3400, 3232, 3271, 3305, 3426, 3425, 3313, 3208])
    rn = RedmineIssueDownloader(REDMINE_SERVER, REDMINE_API_KEY, REDMINE_PROJECT_ID, REDMINE_ISSUES_DIRECTORY)
    rn.download_tickets2()

    msg(rn.get_issue_count())
    #rn.show_project_info()
    #rn.process_files()
    #msg(rn.get_single_issue(3232))

"""
import json
c = open('issue_list2.txt', 'rU').read()
d = json.loads(c)
print(len(d))
"""


# -*- coding: utf-8 -*-
#! /usr/bin/env python
"""
Created on  05 10 10:39:56  2023
@Author : Lulu Shi
@Mails : crazzy_rabbit@163.com
"""
import click
import pandas as pd

def load_pop(list):
    pop = pd.read_csv(list, sep='\t', header=None)
    return pop

def load_data(infile):
    df = pd.read_csv(infile, sep='\t')
    return df

def target(poplist, infile):
    target_df = infile.filter(items=['chr', 'start', 'end'] + poplist[0].tolist())
    return target_df

def cal_sd2(pop):
    sd2 = pop.iloc[:, 3:].var(numeric_only=True, axis=1)
    return sd2

@click.command()
@click.option('--file', help='GetCleanCNV.py的结果文件')
@click.option('--p1', help='属于群体1的个体ID，一行一个ID')
@click.option('--p2', help='属于群体2的个体ID，一行一个ID')
@click.option('--out', help='输出文件前缀')
def main(file, p1, p2, out):
    data = load_data(file)

    # 计算1群体方差
    pop1list = load_pop(p1)
    pop1 = target(pop1list, data)
    pop1_sd_weight = cal_sd2(pop1) * len(pop1)

    # 计算2群体方差
    pop2list = load_pop(p2)
    pop2 = target(pop2list, data)
    pop2_sd_weight = cal_sd2(pop2) * len(pop2)

    # 两群体方差加权群体大小
    Nt = len(pop1) + len(pop2)
    Vs = (pop1_sd_weight + pop2_sd_weight) / Nt

    # 两比较群体样本量之和的方差
    total = pd.concat([pop1.iloc[:, 3:], pop2.iloc[:, 3:]], axis=1)
    Vt = total.var(numeric_only=True, axis=1)

    # Vst
    Vst = (Vt - Vs) / Vt

    chat = pop1.iloc[:, :3]
    chat['Vst'] = Vst
    chat.to_csv(f'{out}.Vst.txt', sep='\t', index=False)

if __name__ == '__main__':
    main()


#encoding=utf-8

from esAPI.ESsetting import *
import requests
import json

LOCALURL=LOCALHOST

header={'Content-Type':'application/json'}

def createDOC(index_type,index_name,data_json,unique_id="-1"):
    tmp_url=LOCALURL+"/"+index_type+"/"+index_name
    if(unique_id!="-1"):
        tmp_url=tmp_url+"/"+str(unique_id)
    response=requests.post(tmp_url,data=json.dumps(data_json),headers=header)
    return response.status_code

def createIndex(index_type,data_json,number_of_shards=3,number_of_replicas=1):
    tmp_url=LOCALURL+"/"+index_type+"/"
    data={
        "settings": {
            "index": {
                "number_of_shards": number_of_shards,
                "number_of_replicas": number_of_replicas
            }
        },
        "mappings": {
            "properties":data_json
        }
    }
    response=requests.put(tmp_url,data=json.dumps(data),headers=header)
    return response.status_code

def insertDoc(index_type,index_name,data_json):
    tmp_url = LOCALURL + "/" + index_type + "/"+ index_name
    response = requests.post(tmp_url,data=json.dumps(data_json),headers=header)
    return response


import abc
from typing import Any, ClassVar, Dict, Optional, Set, Tuple

import colt
import flax
import jax

Array = Any

class Model(abc.ABC, flax.linen.Module, colt.Registrable):
    rngkeys: ClassVar[Set[str]] = set()
    mutables: ClassVar[Set[str]] = set()
    default_training_module: ClassVar[str] = "default"

    @abc.abstractmethod
    def __call__(
        self,
        *args: Any,
        train: bool = False,
        **kwargs: Any,
    ) -> Dict[str, Any]:
        raise NotImplementedError

    def split_rngs(
        self,
        rngs: Any,
        train: bool = False,
        additional_keys: Optional[Set[str]] = None,
    ) -> Tuple[Any, Dict[str, Any]]:
        additional_keys = additional_keys or set()
        required_rngkeys = list(self.rngkeys | additional_keys)

        if not train:
            if "dropout" in required_rngkeys:
                required_rngkeys.remove("dropout")

        rngs, *new_rngs = jax.random.split(rngs, len(required_rngkeys) + 1)
        rngs_for_call = {name: rng for name, rng in zip(required_rngkeys, new_rngs)}
        return rngs, rngs_for_call


import os
import pipes
import subprocess
import logging

def run(command):

    logger = logging.getLogger(__name__)

    logger.debug(command)

    os.environ['PATH'] = '/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/usr/local/share/npm/bin:'

    process = subprocess.Popen([command],
        shell=True,
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE)

    stdout = process.stdout.readline()
    stderr = process.stderr.readline()

    if stdout: logger.info(stdout)
    if stderr: logger.warning(stderr)

def preBuild(site):
    run('coffee -c %s/js/*.coffee' % pipes.quote(site.static_path))


#!/usr/bin/env python3

import bpaTools
from shapely.geometry import LineString, Point
from rdp import rdp

cstGeoType:str              = "type"
cstGeojsonGeometry:str      = "geometry"
cstGeoFeature:str           = "Feature"
cstGeoProperties:str        = "properties"
cstGeoCoordinates:str       = "coordinates"
cstGeoPoint:str             = "Point"
cstGeoLine:str              = "LineString"
cstGeoPolygon:str           = "Polygon"
errLocalisationPoint:list   = [-5,45]

#geojson Colors attributes
def addColorProperties(prop:dict, dstProp:dict, oLog:bpaTools.Logger):
    sClass = prop["class"]

    #Red            - #ff0000
    #medium red     - #ff6480
    #ligth Red      - #ff8080
    #Purple         - #800080
    #medium purple  - #ffb9dc
    #light purple   - #ff96dc
    #bordeau        - #aa0014
    #Gres           - #444444
    #ligth Gres     - #a0868b
    #Orange         - #f07800
    #Blue           - #0000ff
    #ligth blue     - #ceeffe
    #bright blue    - #80ffff

    #Red and fill
    if sClass in ["P","ZIT"]:
        sStroke = "#ff0000"
        sFill = "#ff8080"
        nFillOpacity = 0.6
    #Red and fill
    elif sClass in ["A","B","C","P","ZIT","CTR","CTR-P","TMA","TMA-P","TMZ","RMZ/TMZ","TMZ/RMZ"]:
        sStroke = "#ff0000"
        sFill = "#ff8080"
        if prop.get("lower", None)=="SFC":
            nFillOpacity = 0.6
        else:
            nFillOpacity = 0.3
    #Red and No-Fill
    elif sClass in ["CTA","CTA-P","FIR","FIR-P","NO-FIR","PART","CLASS","SECTOR","SECTOR-C","OCA","OCA-P","OTA","OTA-P","UTA","UTA-P","UIR","UIR-P","TSA","CBA","RCA","RAS","TRA","AMA","ASR","ADIZ","POLITICAL","OTHER","AWY"]:
        sStroke = "#ff0000"
        sFill = "#ff8080"
        nFillOpacity = 0
    #Purple and No-fill
    elif sClass=="D" and prop.get("type", None)=="LTA":
        sStroke = "#800080"
        sFill = "#ff96dc"
        nFillOpacity = 0
    #Red and fill
    elif sClass in ["D","D-AMC"] and not bool(prop.get("declassifiable", False)):
        sStroke = "#ff0000"
        sFill = "#ff8080"
        if prop.get("lower", None)=="SFC":
            nFillOpacity = 0.6
        else:
            nFillOpacity = 0.3
    #Light-Red and Purple fill
    elif sClass in ["D","D-AMC"] and bool(prop.get("declassifiable", False)):
        sStroke = "#ff0000"
        sFill = "#ffb9dc"
        if prop.get("lower", None)=="SFC":
            nFillOpacity = 0.6
        else:
            nFillOpacity = 0.3
    #Purple and fill
    elif sClass in ["R","R-AMC"] and prop.get("type", None)!="RTBA":
        sStroke = "#800080"
        sFill = "#ffb9dc"
        if prop.get("lower", None)=="SFC":
            nFillOpacity = 0.6
        else:
            nFillOpacity = 0.3
    #Gres and fill
    elif sClass in ["R","R-AMC"] and prop.get("type", None)=="RTBA":
        sStroke = "#444444"
        sFill = "#a0868b"
        if prop.get("lower", None)=="SFC":
            nFillOpacity = 0.6
        else:
            nFillOpacity = 0.3
    #Orange and ligth-fill
    elif sClass in ["GP","Q","VV","VL","BA","PA"]:
        sStroke = "#f07800"
        sFill = "#f07800"
        if prop.get("lower", None)=="SFC":
            nFillOpacity = 0.3
        else:
            nFillOpacity = 0.1
    #Orange and No-Fill
    elif sClass in ["RMZ","W"]:
        sStroke = "#f07800"
        sFill = "#f07800"
        nFillOpacity = 0
    #Blue
    elif sClass in ["ZSM","BIRD","PROTECT","D-OTHER","SUR","AER","TRPLA","TRVL","VOL","REFUEL"]:
        sStroke = "#0000ff"
        sFill = "#ceeffe"
        if prop.get("lower", None)=="SFC":
            nFillOpacity = 0.4
        else:
            nFillOpacity = 0.2
    #Green
    elif sClass in ["E","F","G","SIV","FIS","FFVL","FFVP"]:
        sStroke = "#008040"
        sFill = "#80ff80"
        if prop.get("lower", None)=="SFC":
            nFillOpacity = 0.4
        else:
            nFillOpacity = 0.2
    else:
        oLog.warning("GeoJSON Color not found for Class={0}".format(sClass), outConsole=False)
        return

    #Quasi transparence pour toutes les zones au dessus de FL115
    if nFillOpacity!=0 and prop.get("lowerM", None)>=3505:
            nFillOpacity = 0.1

    dstProp.update({"stroke": sStroke})
    dstProp.update({"stroke-width": 2})
    dstProp.update({"stroke-opacity": 0.8})
    dstProp.update({"fill": sFill})
    dstProp.update({"fill-opacity": nFillOpacity})
    return

class Aixm2json4_5:

    def __init__(self, oCtrl) -> None:
        bpaTools.initEvent(__file__, oCtrl.oLog)
        self.oCtrl = oCtrl
        self.oAirspacesCatalog = None
        self.geoBorders = None                  #Geographic borders dictionary
        self.geoAirspaces = None                #Geographic airspaces dictionary
        self.epsilonReduce:float = -1           #Default value=0 for no-doubling with no-compression, =0.0001 for good compression
        return

    def parseControlTowers(self) -> None:
        sTitle = "Aerodrome control towers"
        sXmlTag = "Uni"

        sMsg = "Parsing {0} to GeoJSON - {1}".format(sXmlTag, sTitle)
        self.oCtrl.oLog.info(sMsg)

        oList = self.oCtrl.oAixm.root.find_all(sXmlTag, recursive=False)
        barre = bpaTools.ProgressBar(len(oList), 20, title=sMsg, isSilent=self.oCtrl.oLog.isSilent)
        idx = 0
        geojson = []
        for uni in oList:
            idx+=1
            twr = self.tower2json(uni)
            if twr:
                geojson.append(twr)
            barre.update(idx)
        barre.reset
        self.oCtrl.oAixmTools.writeGeojsonFile("towers", geojson)
        return

    def tower2json(self, uni) -> dict:
        if uni.codeType.string in ("TWR", "OTHER"):
            if uni.find("geoLat", recursive=False):
                prop = self.oCtrl.oAixmTools.initProperty("Aerodrome Control Tower")
                prop = self.oCtrl.oAixmTools.addProperty(prop, uni.OrgUid, "txtName", "organisationAuthority")
                prop = self.oCtrl.oAixmTools.addProperty(prop, uni, "codeType", "codeType")
                prop = self.oCtrl.oAixmTools.addProperty(prop, uni.UniUid, "txtName", "name")
                geom = {cstGeoType:cstGeoPoint, cstGeoCoordinates:self.oCtrl.oAixmTools.geo2coordinates("dd", uni)[::-1]}
                return {cstGeoType:cstGeoFeature, cstGeoProperties:prop, cstGeojsonGeometry:geom}
            else:
                self.oCtrl.oLog.warning("Missing TWR coordinates {0}".format(uni.UniUid), outConsole=False)
        return

    def parseAerodromes(self) -> None:
        sTitle = "Aerodromes / Heliports"
        sXmlTag = "Ahp"

        sMsg = "Parsing {0} to GeoJSON - {1}".format(sXmlTag, sTitle)
        self.oCtrl.oLog.info(sMsg)

        oList = self.oCtrl.oAixm.root.find_all(sXmlTag, recursive=False)
        barre = bpaTools.ProgressBar(len(oList), 20, title=sMsg, isSilent=self.oCtrl.oLog.isSilent)
        idx = 0
        geojson = []
        for ahp in oList:
            idx+=1
            geojson.append(self.ahp2json(ahp))
            barre.update(idx)
        barre.reset()
        self.oCtrl.oAixmTools.writeGeojsonFile("aerodromes", geojson)
        return

    def ahp2json(self, ahp) -> dict:
        prop = self.oCtrl.oAixmTools.initProperty("Aerodrome / Heliport")
        prop = self.oCtrl.oAixmTools.addProperty(prop, ahp.OrgUid, "txtName", "organisationAuthority")
        prop = self.oCtrl.oAixmTools.addProperty(prop, ahp, "codeType")
        prop = self.oCtrl.oAixmTools.addProperty(prop, ahp, "txtName", "name")
        prop = self.oCtrl.oAixmTools.addProperty(prop, ahp.AhpUid, "codeId", "codeId")
        prop = self.oCtrl.oAixmTools.addProperty(prop, ahp, "codeIcao", optional=True)
        prop = self.oCtrl.oAixmTools.addProperty(prop, ahp, "codeIata", optional=True)
        prop = self.oCtrl.oAixmTools.addProperty(prop, ahp, "valElev", "elevation", optional=True)
        prop = self.oCtrl.oAixmTools.addProperty(prop, ahp, "uomDistVer", "verticalUnit", optional=True)
        prop = self.oCtrl.oAixmTools.addProperty(prop, ahp, "txtDescrRefPt", "description", optional=True)
        prop = self.oCtrl.oAixmTools.addProperty(prop, ahp, "txtDescrSite", "descriptionSite", optional=True)
        prop = self.oCtrl.oAixmTools.addProperty(prop, ahp, "txtNameCitySer", "cityServing", optional=True)
        prop = self.oCtrl.oAixmTools.addProperty(prop, ahp, "txtNameAdmin", "nameAdmin", optional=True)
        prop = self.oCtrl.oAixmTools.addProperty(prop, ahp, "txtRmk", "remark", optional=True)
        geom = {cstGeoType:cstGeoPoint, cstGeoCoordinates:self.oCtrl.oAixmTools.geo2coordinates("dd", ahp)[::-1]}
        return {cstGeoType:cstGeoFeature, cstGeoProperties:prop, cstGeojsonGeometry:geom}

    def parseObstacles(self) -> None:
        sTitle = "Obstacles"
        sXmlTag = "Obs"

        sMsg = "Parsing {0} to GeoJSON - {1}".format(sXmlTag, sTitle)
        self.oCtrl.oLog.info(sMsg)

        oList = self.oCtrl.oAixm.root.find_all(sXmlTag, recursive=False)
        barre = bpaTools.ProgressBar(len(oList), 20, title=sMsg, isSilent=self.oCtrl.oLog.isSilent)
        idx = 0
        geojson = []
        for obs in oList:
            idx+=1
            geojson.append(self.obs2json(obs))
            barre.update(idx)
        barre.reset()
        self.oCtrl.oAixmTools.writeGeojsonFile("obstacles", geojson)
        return

    def obs2json(self, obs) -> dict:
        prop = self.oCtrl.oAixmTools.initProperty("Obstacle")
        prop = self.oCtrl.oAixmTools.addProperty(prop, obs, "txtDescrType", "type")
        prop = self.oCtrl.oAixmTools.addProperty(prop, obs, "txtName", "name")
        prop = self.oCtrl.oAixmTools.addProperty(prop, obs, "txtDescrMarking", "marked", optional=True)
        prop = self.oCtrl.oAixmTools.addProperty(prop, obs, "codeLgt", "codeLight")
        prop = self.oCtrl.oAixmTools.addProperty(prop, obs, "txtDescrLgt", "descrLight", optional=True)
        prop = self.oCtrl.oAixmTools.addProperty(prop, obs, "valElev", "elevation")
        prop = self.oCtrl.oAixmTools.addProperty(prop, obs, "valHgt", "height", optional=True)
        prop = self.oCtrl.oAixmTools.addProperty(prop, obs, "uomDistVer", "verticalUnit")
        prop = self.oCtrl.oAixmTools.addProperty(prop, obs, "txtRmk", "remark", optional=True)
        geom = {cstGeoType:cstGeoPoint, cstGeoCoordinates:self.oCtrl.oAixmTools.geo2coordinates("dd", obs.ObsUid)[::-1]}
        return {cstGeoType:cstGeoFeature, cstGeoProperties:prop, cstGeojsonGeometry:geom}

    def parseRunwayCenterLinePosition(self) -> None:
        sTitle = "Runway Center Line Position"
        sXmlTag = "Rcp"

        sMsg = "Parsing {0} to GeoJSON - {1}".format(sXmlTag, sTitle)
        self.oCtrl.oLog.info(sMsg)

        oList = self.oCtrl.oAixm.root.find_all(sXmlTag, recursive=False)
        barre = bpaTools.ProgressBar(len(oList), 20, title=sMsg, isSilent=self.oCtrl.oLog.isSilent)
        idx = 0
        geojson = []
        for rcp in oList:
            idx+=1
            geojson.append(self.rcp2json(rcp))
            barre.update(idx)
        barre.reset()
        self.oCtrl.oAixmTools.writeGeojsonFile("runwaysCenter", geojson)
        return

    def rcp2json(self, rcp) -> dict:
        prop = self.oCtrl.oAixmTools.initProperty("Runway Center")
        prop = self.oCtrl.oAixmTools.addProperty(prop, rcp.RcpUid.RwyUid.AhpUid, "codeId")
        prop = self.oCtrl.oAixmTools.addProperty(prop, rcp.RcpUid.RwyUid, "txtDesig", "designator")
        prop = self.oCtrl.oAixmTools.addProperty(prop, rcp, "valElev", "elevation", optional=True)
        prop = self.oCtrl.oAixmTools.addProperty(prop, rcp, "uomDistVer", "verticalUnit", optional=True)
        geom = {cstGeoType:cstGeoPoint, cstGeoCoordinates:self.oCtrl.oAixmTools.geo2coordinates("dd", rcp.RcpUid)[::-1]}
        return {cstGeoType:cstGeoFeature, cstGeoProperties:prop, cstGeojsonGeometry:geom}

    def parseGateStands(self) -> None:
        sTitle = "Gates and Stands"
        sXmlTag = "Gsd"

        sMsg = "Parsing {0} to GeoJSON - {1}".format(sXmlTag, sTitle)
        self.oCtrl.oLog.info(sMsg)

        oList = self.oCtrl.oAixm.root.find_all(sXmlTag, recursive=False)
        barre = bpaTools.ProgressBar(len(oList), 20, title=sMsg, isSilent=self.oCtrl.oLog.isSilent)
        idx = 0
        geojson = []
        for gsd in oList:
            idx+=1
            geojson.append(self.gsd2json(gsd))
            barre.update(idx)
        barre.reset()
        self.oCtrl.oAixmTools.writeGeojsonFile("gates-stands", geojson)
        return

    def gsd2json(self, gsd) -> dict:
        prop = self.oCtrl.oAixmTools.initProperty("Gate or Stand")
        prop = self.oCtrl.oAixmTools.addProperty(prop, gsd.GsdUid, "txtDesig", "designator")
        prop = self.oCtrl.oAixmTools.addProperty(prop, gsd.ApnUid, "txtName", "parkingPosition")
        prop = self.oCtrl.oAixmTools.addProperty(prop, gsd.ApnUid.AhpUid, "codeId")
        prop = self.oCtrl.oAixmTools.addProperty(prop, gsd, "codeType")
        prop = self.oCtrl.oAixmTools.addProperty(prop, gsd, "txtDescrRestrUse", optional=True)
        prop = self.oCtrl.oAixmTools.addProperty(prop, gsd, "txtRmk", optional=True)
        geom = {cstGeoType:cstGeoPoint, cstGeoCoordinates:self.oCtrl.oAixmTools.geo2coordinates("dd", gsd)[::-1]}
        return {cstGeoType:cstGeoFeature, cstGeoProperties:prop, cstGeojsonGeometry:geom}

    def parseGeographicBorders(self) -> None:
        sTitle = "Geographic borders"
        sXmlTag = "Gbr"

        sMsg = "Parsing {0} to GeoJSON - {1}".format(sXmlTag, sTitle)
        self.oCtrl.oLog.info(sMsg)

        if self.geoBorders == None:
            self.geoBorders = dict()
            oList = self.oCtrl.oAixm.root.find_all(sXmlTag, recursive=False)
            barre = bpaTools.ProgressBar(len(oList), 20, title=sMsg, isSilent=self.oCtrl.oLog.isSilent)
            idx = 0
            geojson = []
            for gbr in oList:
                idx+=1
                j,l = self.gbr2json(gbr)
                geojson.append(j)
                self.geoBorders[gbr.GbrUid["mid"]] = LineString(l)
                barre.update(idx)
        barre.reset()
        self.oCtrl.oAixmTools.writeGeojsonFile("borders", geojson)
        return

    def gbr2json(self, gbr):
        prop = self.oCtrl.oAixmTools.initProperty("Geographic border")
        prop = self.oCtrl.oAixmTools.addProperty(prop, gbr, "codeType", "type")
        prop = self.oCtrl.oAixmTools.addProperty(prop, gbr.GbrUid, "txtName", "name")
        prop = self.oCtrl.oAixmTools.addProperty(prop, gbr, "txtRmk", "desc", optional=True)
        # geometry
        g = []
        l = []
        ddPrev = None
        for gbv in gbr.find_all("Gbv", recursive=False):
            if gbv.codeType.string not in ("GRC", "END"):
                self.oCtrl.oLog.critical("Not recognized codetype\n{0}".format(gbv), outConsole=True)
            dd = self.oCtrl.oAixmTools.geo2coordinates("dd", gbv)[::-1]
            #Suppression des doublons (après prise en compte de l'arrondi 'GeojsonDigitOptimize')
            if dd != ddPrev:
                g.append(dd)
                l.append((dd[0], dd[1]))
                ddPrev = dd
        geom = {cstGeoType:cstGeoLine, cstGeoCoordinates:g}
        return ({cstGeoType:cstGeoFeature, cstGeoProperties:prop, cstGeojsonGeometry:geom}, l)

    def findJsonObjectAirspacesBorders(self, sAseUid) -> dict:
        for o in self.geoAirspaces:
            if o[cstGeoProperties]["UId"]==sAseUid:
                return o[cstGeojsonGeometry]
        return None

    def parseAirspacesBorders(self, airspacesCatalog) -> None:
        self.oAirspacesCatalog = airspacesCatalog

        #Controle de prerequis
        if self.geoBorders == None:
            self.parseGeographicBorders()

        sTitle = "Airspaces Borders"
        sXmlTag = "Abd"

        if not self.oCtrl.oAixm.root.find(sXmlTag, recursive=False):
            sMsg = "Missing tags {0} - {1}".format(sXmlTag, sTitle)
            self.oCtrl.oLog.warning(sMsg, outConsole=True)
            return

        sMsg = "Parsing {0} to GeoJSON - {1}".format(sXmlTag, sTitle)
        self.oCtrl.oLog.info(sMsg)

        barre = bpaTools.ProgressBar(len(self.oAirspacesCatalog.oAirspaces), 20, title=sMsg, isSilent=self.oCtrl.oLog.isSilent)
        idx = 0
        self.geoAirspaces = []                #Réinitialisation avant traitement global
        for k,oZone in self.oAirspacesCatalog.oAirspaces.items():
            idx+=1
            if not oZone["groupZone"]:          #Ne pas traiter les zones de type 'Regroupement'
                sAseUid = oZone["UId"]

                #Map 20200801
                #if oZone["id"] in ["EBS27"]:  #[W] LOW FLYING AREA GOLF ONE
                #    self.oCtrl.oLog.critical("just for bug {0}".format(oZone["id"]), outConsole=False)

                oBorder = self.oAirspacesCatalog.findAixmObjectAirspacesBorders(sAseUid)
                if oBorder:
                    self.parseAirspaceBorder(oZone, oBorder)
                else:
                    sAseUidBase = self.oAirspacesCatalog.findZoneUIdBase(sAseUid)       #Identifier la zone de base (de référence)
                    if sAseUidBase==None:
                        self.oCtrl.oLog.warning("Missing Airspaces Borders AseUid={0} of {1}".format(sAseUid, oZone["nameV"]), outConsole=False)
                        self.geoAirspaces.append({cstGeoType:cstGeoFeature, cstGeoProperties:oZone, cstGeojsonGeometry:{cstGeoType: cstGeoPoint, cstGeoCoordinates: errLocalisationPoint}})
                    else:
                        geom = self.findJsonObjectAirspacesBorders(sAseUidBase)         #Recherche si la zone de base a déjà été parsé
                        if geom:
                            if geom[cstGeoCoordinates]==errLocalisationPoint:
                                self.oCtrl.oLog.warning("Missing Airspaces Borders sAseUidBase={0} used by AseUid={1} of {2}".format(sAseUidBase, sAseUid, oZone["nameV"]), outConsole=False)
                            self.geoAirspaces.append({cstGeoType:cstGeoFeature, cstGeoProperties:oZone, cstGeojsonGeometry:geom})
                        else:
                            oBorder = self.oAirspacesCatalog.findAixmObjectAirspacesBorders(sAseUidBase)
                            if oBorder==None:
                                self.oCtrl.oLog.warning("Missing Airspaces Borders AseUid={0} AseUidBase={1} of {2}".format(sAseUid, sAseUidBase, oZone["nameV"]), outConsole=False)
                                self.geoAirspaces.append({cstGeoType:cstGeoFeature, cstGeoProperties:oZone, cstGeojsonGeometry:{cstGeoType: cstGeoPoint, cstGeoCoordinates: errLocalisationPoint}})
                            else:
                                self.parseAirspaceBorder(oZone, oBorder)
            barre.update(idx)

        barre.reset()
        return

    def parseAirspaceBorder(self, oZone, oBorder) -> None:
        g = []              #geometry GeoJSON
        points4map = []

        #Init
        bIsSpecCircle:bool = False
        bIsCircle:bool = bool(oBorder.Circle)   #Zone uniqumement contituée d'un cercle
        if not bIsCircle:
            #Extraction des vecteurs
            avx_list = oBorder.find_all("Avx", recursive=False)

            #Si la zone est 'Danger' ou 'Vol libre et Vol a voile' ; et uniquement déclarée sous forme d'un 'Point' ; alors reconstituer un cercle !
            if len(avx_list)==1 and oZone.get("class",None) in ["Q","W"]:   bIsSpecCircle = True

        #Construction d'un cercle standard
        if bIsCircle:
            lat_c, lon_c = self.oCtrl.oAixmTools.geo2coordinates("dd",
                                                                 oBorder.Circle,
                                                                 latitude=oBorder.Circle.geoLatCen.string,
                                                                 longitude=oBorder.Circle.geoLongCen.string,
                                                                 oZone=oZone)

            radius = float(oBorder.Circle.valRadius.string)
            radius = self.oCtrl.oAixmTools.convertLength(radius, oBorder.uomRadius.string, "M")   #Convert radius in Meter for GeoJSON format

            Pcenter = Point(lon_c, lat_c)
            if self.oCtrl.MakePoints4map:
                points4map.append(self.oCtrl.oAixmTools.make_point(Pcenter, "Circle Center of {0}".format(oZone["nameV"])))
            g = self.oCtrl.oAixmTools.make_arc(Pcenter, radius)
            #oZone.update({"name":str(len(g)) + " Segments - " + oZone.get("name")})
            geom = {cstGeoType:cstGeoPolygon, cstGeoCoordinates:[g]}

        #Construction spécifique d'un cercle sur la base d'un Point unique
        elif bIsSpecCircle:
            lat_c, lon_c = self.oCtrl.oAixmTools.geo2coordinates("dd", avx_list[0],oZone=oZone)

            #Radius in Meter for GeoJSON format / Depend of area type
            radius:float = float(1000)              #Fixe un rayon de 1000m par défaut
            if   oZone["type"] in ["TRVL"]:         #TRVL Treuil-Vol-Libre
                radius = float(2000)                #Fixe un rayon de 2000m
            elif oZone["type"] in ["TRPLA"]:        #TRPLA Treuil Planeurs
                radius = float(5000)                #Fixe un rayon de 5000m
            elif oZone["type"] in ["PJE"]:          #PJE=Parachute Jumping Exercise
                radius = float(1000)                #Fixe un rayon de 1000m

            Pcenter = Point(lon_c, lat_c)
            if self.oCtrl.MakePoints4map:
                points4map.append(self.oCtrl.oAixmTools.make_point(Pcenter, "Circle Center of {0}".format(oZone["nameV"])))
            g = self.oCtrl.oAixmTools.make_arc(Pcenter, radius)
            geom = {cstGeoType:cstGeoPolygon, cstGeoCoordinates:[g]}

        #Construction d'un tracé sur la base d'une suite de Points et/ou Arcs
        else:
            for avx_cur in range(0,len(avx_list)):
                avx = avx_list[avx_cur]
                codeType = avx.codeType.string

                # 'Great Circle' or 'Rhumb Line' segment
                if codeType in ["GRC", "RHL"]:
                    lat, lon = self.oCtrl.oAixmTools.geo2coordinates("dd", avx, oZone=oZone)
                    if self.oCtrl.MakePoints4map:
                        pt = Point(lon, lat)
                        points4map.append(self.oCtrl.oAixmTools.make_point(pt, "Point {0} of {1}; type={2}".format(avx_cur, oZone["nameV"], codeType)))
                    g.append([lon, lat])

                # 'Counter Clockwise Arc' or 'ClockWise Arc'
                #Nota: 'ABE' = 'Arc By Edge' ne semble pas utilisé dans les fichiers SIA-France et Eurocontrol-Europe
                elif codeType in ["CCA", "CWA"]:
                    start = self.oCtrl.oAixmTools.geo2coordinates("dd", avx, oZone=oZone)[::-1]
                    if avx_cur+1 == len(avx_list):
                        stop = g[0]
                    else:
                        stop = self.oCtrl.oAixmTools.geo2coordinates("dd", avx_list[avx_cur+1], oZone=oZone)[::-1]

                    center = self.oCtrl.oAixmTools.geo2coordinates("dd",
                                                                   avx,
                                                                   latitude=avx.geoLatArc.string,
                                                                   longitude=avx.geoLongArc.string,
                                                                   oZone=oZone)[::-1]

                    Pcenter = Point(center[0], center[1])
                    Pstart = Point(start[0], start[1])
                    Pstop = Point(stop[0], stop[1])

                    if self.oCtrl.MakePoints4map:
                        points4map.append(self.oCtrl.oAixmTools.make_point(Pstart, "Arc Start {0} of {1}".format(avx_cur, oZone["nameV"])))
                        points4map.append(self.oCtrl.oAixmTools.make_point(Pcenter, "Arc Center {0} of {1}".format(avx_cur, oZone["nameV"])))
                        points4map.append(self.oCtrl.oAixmTools.make_point(Pstop, "Arc Stop {0} of {1}".format(avx_cur, oZone["nameV"])))

                    #Alignement pas toujours idéal sur les extremités d'arcs
                    radius = float(avx.valRadiusArc.string)
                    radius = self.oCtrl.oAixmTools.convertLength(radius, oBorder.uomRadiusArc.string, "M")   #Convert radius in Meter for GeoJSON format

                    #Amélioration de l'alignement des arcs par recalcul systématique du rayon sur la base des coordonnées d'entrée et de sortie d'arc
                    arc = self.oCtrl.oAixmTools.make_arc2(Pcenter, Pstart, Pstop, 0.0, (codeType=="CWA"))
                    #arc = self.oCtrl.oAixmTools.make_arc2(Pcenter, Pstart, Pstop, radius, (codeType=="CWA"))   #Sans recalcul - Des erreurs d'alignement !
                    for o in arc:
                        g.append(o)

                # 'Sequence of geographical (political) border vertexes'
                elif codeType == "FNT":
                    # geographic borders
                    start = self.oCtrl.oAixmTools.geo2coordinates("dd", avx, oZone=oZone)[::-1]
                    if avx_cur+1 == len(avx_list):
                        stop = g[0]
                    else:
                        stop = self.oCtrl.oAixmTools.geo2coordinates("dd", avx_list[avx_cur+1], oZone=oZone)[::-1]

                    if avx.GbrUid["mid"] in self.geoBorders:
                        fnt = self.geoBorders[avx.GbrUid["mid"]]
                        start_d = fnt.project(Point(start[0], start[1]), normalized=True)
                        stop_d = fnt.project(Point(stop[0], stop[1]), normalized=True)
                        geom = self.oCtrl.oAixmTools.substring(fnt, start_d, stop_d, normalized=True)
                        for c in geom.coords:
                            lon, lat = c
                            g.append([lon, lat])
                    else:
                        self.oCtrl.oLog.warning("Missing geoBorder GbrUid='{0}' Name={1} of {2}".format(avx.GbrUid["mid"], avx.GbrUid.txtName.string, oZone["nameV"]), outConsole=False)
                        g.append(start)
                else:
                    self.oCtrl.oLog.warning("Default case - GbrUid='{0}' Name={1} of {2}".format(avx.GbrUid["mid"], avx.GbrUid.txtName.string, oZone["nameV"]), outConsole=False)
                    g.append(self.oCtrl.oAixmTools.geo2coordinates("dd", avx, oZone=oZone)[::-1])

            if len(g) == 0:
                self.oCtrl.oLog.error("Empty geometry of {0}\n{1}".format(oZone["nameV"], oBorder.prettify()), outConsole=True)
                geom = None
            elif len(g) == 1:
                geom = {cstGeoType:cstGeoPoint, cstGeoCoordinates:g[0]}
            elif len(g) == 2:
                geom = {cstGeoType:cstGeoLine, cstGeoCoordinates:g}
            else:
                #Contrôle de fermeture du Polygone
                if g[0] != g[-1]:
                    g.append(g[0])
                geom = {cstGeoType:cstGeoPolygon, cstGeoCoordinates:[g]}

        #Ajout spécifique des points complémentaires pour map des cartographies
        for g0 in points4map:
            for g1 in g0:
                self.geoAirspaces.append(g1)
        self.geoAirspaces.append({cstGeoType:cstGeoFeature, cstGeoProperties:oZone, cstGeojsonGeometry:geom})
        return

    #Use epsilonReduce for compress shape in file (Default value=0 for no-doubling with no-compression, =0.0001 for good compression)
    def saveAirspacesFilter(self, aContext, epsilonReduce:float=None) -> None:
        if epsilonReduce and epsilonReduce>0:
            self.epsilonReduce = epsilonReduce
        else:
            self.epsilonReduce = self.oCtrl.epsilonReduce
        if not self.geoAirspaces:                                   #Contrôle si le fichier est vide
            return
        context = aContext[0]
        sMsg = "Prepare GeoJSON file - {0}".format(aContext[1])
        self.oCtrl.oLog.info(sMsg)
        barre = bpaTools.ProgressBar(len(self.oAirspacesCatalog.oAirspaces), 20, title=sMsg, isSilent=self.oCtrl.oLog.isSilent)
        idx = 0
        geojson = []       #Initialisation avant filtrage spécifique
        for o in self.geoAirspaces:
            oZone = o[cstGeoProperties]
            idx+=1
            include = not oZone["groupZone"]                         				#Ne pas traiter les zones de type 'Regroupement'
            #if include and ("excludeAirspaceNotCoord" in oZone):     				#Ne pas traiter les zones qui n'ont pas de coordonnées geometriques
            #    include = not oZone["excludeAirspaceNotCoord"]
            if include:
                include = False
                if context=="all":
                    include = True
                elif context=="ifr":
                    include = (not oZone["vfrZone"]) and (not oZone["groupZone"])
                elif context=="vfr":
                    include = oZone["vfrZone"]
                    include = include or oZone.get("vfrZoneExt", False)     		#Exporter l'extension de vol possible en VFR de 0m jusqu'au FL195/5944m
                elif context=="ff":
                    include = oZone["freeFlightZone"]
                    include = include or oZone.get("freeFlightZoneExt", False) 	    #Exporter l'extension de vol possible en VFR de 0m jusqu'au FL195/5944m
                    include = include and (o[cstGeojsonGeometry][cstGeoCoordinates]!=errLocalisationPoint)
            if include:
                #Extract single parts of properties
                oSingleCat:dict = {}
                aSinglePorperties:list = ["name", "nameV", "class", "type", "codeActivity",
                                          "lower", "lowerMin", "ordinalLowerM", "lowerM",
                                          "upper", "upperMax", "ordinalUpperM", "upperM", "groundEstimatedHeight",
                                          "desc", "activationCode", "activationDesc", "declassifiable", "timeScheduling", "Mhz",
                                          "GUId", "UId", "id"]  #Exclude: zoneType, groupZone, srcClass, srcType, vfrZone, vfrZoneExt, freeFlightZone, freeFlightZoneExt, srcName; etc...
                for sProp in aSinglePorperties:
                    value = o[cstGeoProperties].get(sProp, None)
                    if value!=None:
                        oSingleCat.update({sProp:value})
                addColorProperties(o[cstGeoProperties], oSingleCat, self.oCtrl.oLog)    #Ajout des propriétés pour colorisation de la zone

                #Extraction des coordonnées
                oAsGeo = o[cstGeojsonGeometry]
                if oAsGeo[cstGeoType].lower()==(cstGeoPoint).lower():
                    oCoords:list = oAsGeo[cstGeoCoordinates]                        #get coordinates of geometry
                elif oAsGeo[cstGeoType].lower()==(cstGeoLine).lower():
                    oCoords:list = oAsGeo[cstGeoCoordinates]                        #get coordinates of geometry
                elif oAsGeo[cstGeoType].lower()==(cstGeoPolygon).lower():
                    oCoords:list = oAsGeo[cstGeoCoordinates][0]                     #get coordinates of geometry

                #Optimisation du tracé GeoJSON
                oCoordsDst:list = []
                #if self.epsilonReduce<0: --> do not change !
                if self.epsilonReduce==0 or (self.epsilonReduce>0 and len(oCoords)>40):     #Ne pas optimiser le tracé des zones ayant moins de 40 segments (préservation des tracés de cercle minimaliste)
                    oCoordsDst = rdp(oCoords, epsilon=self.epsilonReduce)                   #Optimisation du tracé des coordonnées

                #Remplacement du tracé s'il a été optimisé
                iOrgSize:int = len(oCoords)
                iNewSize:int = len(oCoordsDst)
                if iNewSize>0 and iNewSize!=iOrgSize:
                    percent:float = (1-(iNewSize/iOrgSize))*100
                    percent = int(percent) if percent>1 else round(percent,1)
                    sOpti:str = "Segments optimisés à {0}% ({1}->{2}) [rdp={3}] ***".format(percent, iOrgSize, iNewSize, self.epsilonReduce)
                    self.oCtrl.oLog.debug("GeoJSON RDP Optimisation: {0} - {1}".format(oZone["nameV"], sOpti), level=2, outConsole=False)
                    #self.oCtrl.oLog.debug("RDP Src: {0}".format(oCoords), level=8)
                    #self.oCtrl.oLog.debug("RDP Dst: {0}".format(oCoordsDst), level=8)
                    if oAsGeo[cstGeoType].lower()==(cstGeoPoint).lower():
                        oNewGeo:dict = {cstGeoType:oAsGeo[cstGeoType], cstGeoCoordinates:oCoordsDst}
                    elif oAsGeo[cstGeoType].lower()==(cstGeoLine).lower():
                       oNewGeo:dict = {cstGeoType:oAsGeo[cstGeoType], cstGeoCoordinates:oCoordsDst}
                    elif oAsGeo[cstGeoType].lower()==(cstGeoPolygon).lower():
                        oNewGeo:dict = {cstGeoType:oAsGeo[cstGeoType], cstGeoCoordinates:[oCoordsDst]}
                    oArea:dict = {cstGeoType:cstGeoFeature, cstGeoProperties:oSingleCat, cstGeojsonGeometry:oNewGeo}
                else:
                    oArea:dict = {cstGeoType:cstGeoFeature, cstGeoProperties:oSingleCat, cstGeojsonGeometry:oAsGeo}

                geojson.append(oArea)
            barre.update(idx)
        barre.reset()
        if geojson:
            self.oCtrl.oAixmTools.writeGeojsonFile("airspaces", geojson, context)
        return

    #Nétoyage du catalogue de zones pour desactivation des éléments qui ne sont pas valides ; ou constitués que d'un ou deux 'Point' a exclure uniquement pour le freefligth
    #Ces simples 'Point remarquable' sont supprimés de la cartographie freefligth (ex: un VOR, un émmzteur radio, un centre de piste)
    #Idem, suppression des 'lignes' (ex: Axe d'approche d'un aérodrome ou autres...)
    def cleanAirspacesCalalog(self, airspacesCatalog) -> None:
        if not self.geoAirspaces:                                   #Contrôle si le fichier est vide
            return
        self.oAirspacesCatalog = airspacesCatalog
        if self.oAirspacesCatalog.cleanAirspacesCalalog:            #Contrôle si l'optimisation est déjà réalisée
            return
        sMsg = "Clean catalog"
        self.oCtrl.oLog.info(sMsg)
        barre = bpaTools.ProgressBar(len(self.geoAirspaces), 20, title=sMsg, isSilent=self.oCtrl.oLog.isSilent)
        idx = 0
        lNbChange = 0
        for o in self.geoAirspaces:
            oZone = o[cstGeoProperties]
            idx+=1

            #Flag all not valid area
            oGeom:dict = o[cstGeojsonGeometry]                              #Sample - "geometry": {"type": "Polygon", "coordinates": [[[3.069444, 45.943611], [3.539167, 45.990556], ../..
            if oGeom[cstGeoType] in [cstGeoPoint] and oGeom[cstGeoCoordinates]==errLocalisationPoint:
                oZone.update({"excludeAirspaceNotCoord":True})       #Flag this change in catalog
                lNbChange+=1

            #if oZone["freeFlightZone"]:
            if oGeom[cstGeoType] in [cstGeoPoint,cstGeoLine]:
                exclude=True
            elif len(oGeom[cstGeoCoordinates][0])<3:
                exclude=True
            else:
                exclude=False
            if exclude:
                oZone.update({"freeFlightZone":False})              #Change value in catalog
                if oZone.get("freeFlightZoneExt", False)==True:
                    oZone.update({"freeFlightZoneExt":False})       #Change value in catalog
                oZone.update({"excludeAirspaceNotFfArea":True})     #Flag this change in catalog
                oZone.update({"geometryType":oGeom[cstGeoType]})    #info for catalog 'Point' or 'LineString'
                lNbChange+=1
            barre.update(idx)
        barre.reset()

        if lNbChange>0:
            self.oAirspacesCatalog.saveAirspacesCalalog()               #Save the new catalogs

        self.oAirspacesCatalog.cleanAirspacesCalalog = True  #Marqueur de traitement réalisé
        return