from __future__ import (print_function, division, unicode_literals)
import os
CG_PATH = os.path.dirname(os.path.abspath(__file__))
import sys
sys.path.insert(0, os.path.join(CG_PATH, '..'))
import glimpse
from glimpse.backports import *
from glimpse.imports import (np, pandas, re, datetime, sharedmem, cv2, shapely)
import glob
import requests
try:
    from functools import lru_cache
except ImportError:
    # Python 2
    from backports.functools_lru_cache import lru_cache
try:
    FileNotFoundError
except NameError:
    # Python 2
    FileNotFoundError = IOError

# ---- Environment variables ---

print('cg: Remember to set IMAGE_PATH, KEYPOINT_PATH, and MATCH_PATH')
IMAGE_PATH = None
KEYPOINT_PATH = None
MATCH_PATH = None
FLAT_IMAGE_PATH = False

# ---- Images ----

@lru_cache(maxsize=1)
def Sequences():
    """
    Return sequences metadata.
    """
    df = pandas.read_csv(
        os.path.join(CG_PATH, 'sequences.csv'),
        parse_dates=['first_time_utc', 'last_time_utc'])
    # Floor start time subseconds for comparisons to filename times
    df.first_time_utc = df.first_time_utc.apply(
        datetime.datetime.replace, microsecond=0)
    return df.sort_values('first_time_utc').reset_index(drop=True)

@lru_cache(maxsize=1)
def Stations():
    """
    Return stations metadata.
    """
    path = os.path.join(CG_PATH, 'geojson', 'stations.geojson')
    return glimpse.helpers.read_geojson(path, crs=32606, key='id')['features']

def _station_break_index(path):
    """
    Return index of image in motion break sequence.

    Arguments:
        path (str): Image path

    Returns:
        int: Either 0 (original viewdir) or i (viewdir of break i + 1)
    """
    stations = Stations()
    ids = parse_image_path(path)
    station = stations[ids['station']]
    if 'breaks' not in station['properties']:
        return 0
    breaks = station['properties']['breaks']
    if not breaks:
        return 0
    break_images = np.array([x['start'] for x in breaks])
    idx = np.argsort(break_images)
    i = np.where(break_images[idx] <= ids['basename'])[0]
    if i.size > 0:
        return idx[i[-1]] + 1
    else:
        return 0

def paths_to_datetimes(paths):
    """
    Return datetime objects parsed from image paths.

    Arguments:
        paths (iterable): Image paths
    """
    pattern = re.compile(r'_([0-9]{8}_[0-9]{6})[^\/]*$')
    datetimes_str = [pattern.findall(path)[0] for path in paths]
    return pandas.to_datetime(datetimes_str, format='%Y%m%d_%H%M%S').to_pydatetime()

def parse_image_path(path, sequence=False):
    """
    Return metadata parsed from image path.

    Arguments:
        path (str): Image path or basename
        sequence (bool): Whether to include sequence metadata (camera, service, ...)
    """
    basename = glimpse.helpers.strip_path(path)
    station, date_str, time_str = re.findall('^([^_]+)_([0-9]{8})_([0-9]{6})', basename)[0]
    capture_time = datetime.datetime.strptime(date_str + time_str, '%Y%m%d%H%M%S')
    results = dict(basename=basename, station=station,
        date_str=date_str, time_str=time_str, datetime=capture_time)
    if sequence:
        sequences = Sequences()
        is_row = ((sequences.station == station) &
            (sequences.first_time_utc <= capture_time) &
            (sequences.last_time_utc >= capture_time))
        rows = np.where(is_row)[0]
        if len(rows) != 1:
            raise ValueError(
                'Image path has zero or multiple sequence matches: ' + path)
        results = glimpse.helpers.merge_dicts(
            sequences.loc[rows[0]].to_dict(), results)
    return results

def find_image(path):
    """
    Return path to image file.

    Arguments:
        path (str): Image path or basename
    """
    ids = parse_image_path(path, sequence=True)
    filename = ids['basename'] + '.JPG'
    if FLAT_IMAGE_PATH:
        img_path = os.path.join(IMAGE_PATH, filename)
        if os.path.isfile(img_path):
            return img_path
        else:
            raise ValueError('Image not found: ' + path)
    else:
        service_dir = os.path.join(IMAGE_PATH, ids['station'],
            ids['station'] + '_' + ids['service'])
        found_img = None
        if os.path.isdir(service_dir):
            subdirs = [''] + next(os.walk(service_dir))[1]
            for subdir in subdirs:
                img_path = os.path.join(service_dir, subdir, filename)
                if os.path.isfile(img_path):
                    found_img = img_path
                    break
        if found_img:
            return found_img
        else:
            raise ValueError('Image not found: ' + path)

def load_images(station, services, use_exif=False, service_exif=False, anchors=False,
    viewdir=True, viewdir_as_anchor=False, file_errors=True, **kwargs):
    """
    Return list of calibrated Image objects.

    Any available station, camera, image, and viewdir calibrations are loaded
    and images with image calibrations are marked as anchors.

    Arguments:
        station (str): Station identifier
        services (iterable): Service identifiers
        use_exif (bool): Whether to parse image datetimes from EXIF (slower)
            rather than parsed from paths (faster)
        service_exif (bool): Whether to extract EXIF from first image (faster)
            or all images (slower) in service.
            If `True`, `Image.datetime` is parsed from path.
            Always `False` if `use_exif=True`.
        anchors (bool): Whether to include anchor images even if
            filtered out by `kwargs['snap']`
        **kwargs: Arguments to `glimpse.helpers.select_datetimes()`
    """
    if use_exif:
        service_exif = False
    # Sort services in time
    if isinstance(services, str):
        services = services,
    services = np.sort(services)
    # Parse datetimes of all candidate images
    paths_service = [glob.glob(os.path.join(IMAGE_PATH, station, station + '_' + service, '*.JPG'))
        for service in services]
    paths = np.hstack(paths_service)
    basenames = [glimpse.helpers.strip_path(path) for path in paths]
    if use_exif:
        exifs = [glimpse.Exif(path) for path in paths]
        datetimes = np.array([exif.datetime for exif in exifs])
    else:
        datetimes = paths_to_datetimes(basenames)
    # Select images based on datetimes
    indices = glimpse.helpers.select_datetimes(datetimes, **kwargs)
    if anchors:
        # Add anchors
        # HACK: Ignore any <image>-<suffix>.json files
        anchor_paths = glob.glob(os.path.join(CG_PATH, 'images', station + '_*[0-9].json'))
        anchor_basenames = [glimpse.helpers.strip_path(path) for path in anchor_paths]
        if 'start' in kwargs or 'end' in kwargs:
            # Filter by start, end
            anchor_datetimes = np.asarray(paths_to_datetimes(anchor_basenames))
            inrange = glimpse.helpers.select_datetimes(
                anchor_datetimes, **glimpse.helpers.merge_dicts(kwargs, dict(snap=None)))
            anchor_basenames = np.asarray(anchor_basenames)[inrange]
        anchor_indices = np.where(np.isin(basenames, anchor_basenames))[0]
        indices = np.unique(np.hstack((indices, anchor_indices)))
    service_breaks = np.hstack((0, np.cumsum([len(x) for x in paths_service])))
    station_calibration = load_calibrations(
        station_estimate=station, station=station, merge=True, file_errors=False)
    images = []
    for i, service in enumerate(services):
        index = indices[(indices >= service_breaks[i]) & (indices < service_breaks[i + 1])]
        if not index.size:
            continue
        service_calibration = glimpse.helpers.merge_dicts(
            station_calibration,
            load_calibrations(path=paths[index[0]], camera=True, merge=True,
            file_errors=file_errors))
        if service_exif:
            exif = glimpse.Exif(paths[index[0]])
        for j in index:
            basename = basenames[j]
            calibrations = load_calibrations(image=basename,
                viewdir=basename if viewdir else False,
                station_estimate=station, merge=False, file_errors=False)
            if calibrations['image']:
                calibration = glimpse.helpers.merge_dicts(
                    service_calibration, calibrations['image'])
                anchor = True
            else:
                calibration = glimpse.helpers.merge_dicts(
                    service_calibration,
                    dict(viewdir=calibrations['station_estimate']['viewdir']))
                anchor = False
            if viewdir and calibrations['viewdir']:
                calibration = glimpse.helpers.merge_dicts(
                    calibration, calibrations['viewdir'])
                if viewdir_as_anchor:
                    anchor = True
            if use_exif:
                exif = exifs[j]
            elif not service_exif:
                exif = None
            if KEYPOINT_PATH:
                keypoint_path = os.path.join(KEYPOINT_PATH, basename + '.pkl')
            else:
                keypoint_path = None
            image = glimpse.Image(
                path=paths[j], cam=calibration, anchor=anchor, exif=exif,
                datetime=None if use_exif else datetimes[j],
                keypoints_path=keypoint_path)
            images.append(image)
    return images

def load_masks(images):
    """
    Return a list of boolean land masks.

    Images must all be from the same station.

    Arguments:
        images (iterable): Image objects
    """
    # All images must be from the same station (for now)
    station = parse_image_path(images[0].path)['station']
    pattern = re.compile(station + r'_[0-9]{8}_[0-9]{6}[^\/]*$')
    is_station = [pattern.search(img.path) is not None for img in images[1:]]
    assert all(is_station)
    # Find all station svg with 'land' markup
    imgsz = images[0].cam.imgsz
    svg_paths = glob.glob(os.path.join(CG_PATH, 'svg', station + '_*.svg'))
    markups = [glimpse.svg.parse_svg(path, imgsz=imgsz) for path in svg_paths]
    land_index = np.where(['land' in markup for markup in markups])[0]
    if len(land_index) == 0:
        raise ValueError('No land masks found for station')
    svg_paths = np.array(svg_paths)[land_index]
    land_markups = np.array(markups)[land_index]
    # Select svg files nearest to images, with preference within breaks
    svg_datetimes = paths_to_datetimes(svg_paths)
    svg_break_indices = np.array([_station_break_index(path)
        for path in svg_paths])
    img_datetimes = [img.datetime for img in images]
    distances = glimpse.helpers.pairwise_distance_datetimes(
        img_datetimes, svg_datetimes)
    nearest_index = []
    for i, img in enumerate(images):
        break_index = _station_break_index(img.path)
        same_break = np.where(break_index == svg_break_indices)[0]
        if same_break.size > 0:
            i = same_break[np.argmin(distances[i][same_break])]
        else:
            raise ValueError('No mask found within motion breaks for image', i)
            i = np.argmin(distances[i])
        nearest_index.append(i)
    nearest = np.unique(nearest_index)
    # Make masks and expand per image without copying
    masks = [None] * len(images)
    image_sizes = np.array([img.cam.imgsz for img in images])
    sizes = np.unique(image_sizes, axis=0)
    for i in nearest:
        polygons = land_markups[i]['land'].values()
        is_nearest = nearest_index == i
        for size in sizes:
            scale = size / imgsz
            rpolygons = [polygon * scale for polygon in polygons]
            mask = glimpse.helpers.polygons_to_mask(rpolygons, size=size).astype(np.uint8)
            mask = sharedmem.copy(mask)
            for j in np.where(is_nearest & np.all(image_sizes == size, axis=1))[0]:
                masks[j] = mask
    return masks

# ---- Calibration controls ----

def svg_controls(img, svg=None, keys=None, correction=True, step=None):
    """
    Return control objects for an Image.

    Arguments:
        img (Image): Image object
        svg: Path to SVG file (str) or parsed result (dict).
            If `None`, looks for SVG file 'svg/<image>.svg'.
        keys (iterable): SVG layers to include, or all if `None`
        correction: Whether control objects should use elevation correction (bool)
            or arguments to `glimpse.helpers.elevation_corrections()`
    """
    if svg is None:
        basename = parse_image_path(img.path)['basename']
        svg = os.path.join(CG_PATH, 'svg', basename + '.svg')
    controls = []
    if isinstance(svg, (bytes, str)):
        if not os.path.isfile(svg):
            return controls
        svg = glimpse.svg.parse_svg(svg, imgsz=img.cam.imgsz)
    if keys is None:
        keys = svg.keys()
    for key in keys:
        if key in svg:
            if key == 'gcp':
                controls.append(gcp_points(img, svg[key], correction=correction))
            elif key == 'coast':
                controls.append(coast_lines(img, svg[key], correction=correction, step=step))
            elif key == 'terminus':
                controls.append(terminus_lines(img, svg[key], correction=correction, step=step))
            elif key == 'moraines':
                controls.extend(moraines_mlines(img, svg[key], correction=correction, step=step))
            elif key == 'horizon':
                controls.append(horizon_lines(img, svg[key], correction=correction, step=step))
    return controls

def gcp_points(img, markup, correction=True):
    """
    Return ground control Points object for an Image.

    Arguments:
        img (Image): Image object
        markup (dict): Parsed SVG layer
        correction: Whether Points should use elevation correction (bool)
            or arguments to `glimpse.helpers.elevation_corrections()`
    """
    uv = np.vstack(markup.values())
    geo = glimpse.helpers.read_geojson(
        os.path.join(CG_PATH, 'geojson', 'gcp.geojson'), key='id', crs=32606)
    xyz = np.vstack((geo['features'][key]['geometry']['coordinates']
        for key in markup))
    return glimpse.optimize.Points(img.cam, uv, xyz, correction=correction)

def coast_lines(img, markup, correction=True, step=None):
    """
    Return coast Lines object for an Image.

    Arguments:
        img (Image): Image object
        markup (dict): Parsed SVG layer
        correction (bool): Whether to set Lines to use elevation correction
    """
    luv = tuple(markup.values())
    geo = glimpse.helpers.read_geojson(
        os.path.join(CG_PATH, 'geojson', 'coast.geojson'), crs=32606)
    lxy = [feature['geometry']['coordinates'] for feature in geo['features']]
    lxyz = [np.hstack((xy, sea_height(xy, t=img.datetime))) for xy in lxy]
    return glimpse.optimize.Lines(img.cam, luv, lxyz, correction=correction, step=step)

def terminus_lines(img, markup, correction=True, step=None):
    """
    Return terminus Lines object for an Image.

    Arguments:
        img (Image): Image object
        markup (dict): Parsed SVG layer
        correction: Whether Lines should use elevation correction (bool)
            or arguments to `glimpse.helpers.elevation_corrections()`
    """
    luv = tuple(markup.values())
    # HACK: Select terminus with matching date and preferred type
    termini = Termini()
    date_str = img.datetime.strftime('%Y-%m-%d')
    features = [(feature, feature['properties']['type'])
        for feature in termini
        if feature['properties']['date'] == date_str]
    type_order = ('aerometric', 'worldview', 'landsat-8', 'landsat-7', 'terrasar', 'tandem', 'arcticdem', 'landsat-5')
    order = [type_order.index(f[1]) for f in features]
    xy = features[np.argmin(order)[0]]['geometry']['coordinates']
    xyz = np.hstack((xy, sea_height(xy, t=img.datetime)))
    return glimpse.optimize.Lines(img.cam, luv, [xyz], correction=correction, step=step)

def horizon_lines(img, markup, correction=True, step=None):
    """
    Return horizon Lines object for an Image.

    Arguments:
        img (Image): Image object
        markup (dict): Parsed SVG layer
        correction: Whether Lines should use elevation correction (bool)
            or arguments to `glimpse.helpers.elevation_corrections()`
    """
    luv = tuple(markup.values())
    station = parse_image_path(img.path)['station']
    geo = glimpse.helpers.read_geojson(
        os.path.join(CG_PATH, 'geojson', 'horizons', station + '.geojson'), crs=32606)
    lxyz = [coords for coords in glimpse.helpers.geojson_itercoords(geo)]
    return glimpse.optimize.Lines(img.cam, luv, lxyz, correction=correction, step=step)

def moraines_mlines(img, markup, correction=True, step=None):
    """
    Return list of moraine Lines objects for an Image.

    Arguments:
        img (Image): Image object
        markup (dict): Parsed SVG layer
        correction: Whether Lines should use elevation correction (bool)
            or arguments to `glimpse.helpers.elevation_corrections()`
    """
    date_str = img.datetime.strftime('%Y%m%d')
    geo = glimpse.helpers.read_geojson(
        os.path.join(CG_PATH, 'geojson', 'moraines', date_str + '.geojson'), key='id', crs=32606)
    mlines = []
    for key, moraine in markup.items():
        luv = tuple(moraine.values())
        xyz = geo['features'][key]['geometry']['coordinates']
        mlines.append(glimpse.optimize.Lines(img.cam, luv, [xyz], correction=correction, step=step))
    return mlines

def tide_height(t):
    if isinstance(t, datetime.datetime):
        t = [t]
    t = np.asarray(t)
    dt = datetime.timedelta(hours=1.5)
    t_begin = np.nanmin(t).replace(minute=0, second=0, microsecond=0)
    t_end = np.nanmax(t) + dt
    # https://tidesandcurrents.noaa.gov/api/
    params = dict(
        format='json',
        units='metric',
        time_zone='gmt',
        datum='MSL',
        product='hourly_height',
        station=9454240, # Valdez
        begin_date=t_begin.strftime('%Y%m%d %H:%M'),
        end_date=t_end.strftime('%Y%m%d %H:%M'))
    r = requests.get('https://tidesandcurrents.noaa.gov/api/datagetter', params=params)
    v = [float(item['v']) for item in r.json()['data']]
    return np.interp(
        [dti.total_seconds() for dti in t - t_begin],
        np.linspace(0, 3600 * len(v[1:]), len(v)), v)

def sea_height(xy, t=None):
    """
    Return the height of sea level relative to the WGS 84 ellipsoid.

    Uses the EGM 2008 geoid height and the NOAA tide gauge in Valdez, Alaska.

    Arguments:
        xy (array): World coordinates (n, 2)
        t (datetime): Datetime at which to estimate tidal height.
            If `None`, tide is ignored in result.
    """
    egm2008 = glimpse.Raster.read(os.path.join(CG_PATH, 'egm2008.tif'))
    geoid = egm2008.sample(xy).reshape(-1, 1)
    if t is not None:
        if not isinstance(t, datetime.datetime) and len(tide) > 1:
            raise ValueError('t must specify a single datetime')
        tide = tide_height(t)[0]
    else:
        tide = 0
    return geoid + tide

def synth_controls(img, step=None, directions=False):
    image = glimpse.helpers.strip_path(img.path)
    basename = os.path.join(CG_PATH, 'svg-synth', image)
    controls = []
    # Load svg
    path = basename + '.svg'
    if os.path.isfile(path):
        svg = glimpse.svg.parse_svg(path, imgsz=img.cam.imgsz)
        if 'points' in svg or 'lines' in svg or 'points-auto' in svg:
            scam = glimpse.helpers.read_json(basename + '-synth.json')
            simg = glimpse.Image(basename + '-synth.JPG', cam=scam)
            if not directions:
                depth = glimpse.Raster.read(basename + '-depth.tif')
                scale = depth.n / img.cam.imgsz
            # NOTE: Ignoring parallax potential
        if 'points-auto' in svg:
            # Length-2 paths traced from image to synthetic image
            uv = np.vstack([x[0] for x in svg['points-auto'].values()])
            suv = np.vstack([x[1] for x in svg['points-auto'].values()])
            d = 1 if directions else depth.sample(suv * scale)
            xyz = simg.cam.invproject(suv, directions=directions, depth=d)
            points = glimpse.optimize.Points(cam=img.cam, uv=uv, xyz=xyz,
                directions=directions, correction=False)
            controls.append(points)
        if 'points' in svg:
            # Length-2 paths traced from image to synthetic image
            uv = np.vstack([x[0] for x in svg['points'].values()])
            suv = np.vstack([x[1] for x in svg['points'].values()])
            d = 1 if directions else depth.sample(suv * scale)
            xyz = simg.cam.invproject(suv, directions=directions, depth=d)
            points = glimpse.optimize.Points(cam=img.cam, uv=uv, xyz=xyz,
                directions=directions, correction=False)
            controls.append(points)
        if 'lines' in svg:
            for layer in svg['lines'].values():
                # Group with paths named 'image*' and 'synth*'
                uvs = [layer[key] for key in layer if key.find('image') == 0]
                suvs = [layer[key] for key in layer if key.find('synth') == 0]
                depths = [1 if directions else depth.sample(suv * scale)
                    for suv in suvs]
                xyzs = [simg.cam.invproject(suv, directions=directions, depth=d)
                    for suv, d in zip(suvs, depths)]
                lines = glimpse.optimize.Lines(cam=img.cam, uvs=uvs, xyzs=xyzs,
                    step=step, directions=directions, correction=False)
                controls.append(lines)
    return controls

# ---- Control bundles ----

def station_svg_controls(station, size=1, force_size=False, keys=None,
    svgs=None, correction=True, step=None, station_calib=False, camera_calib=True,
    synth=True):
    """
    Return all SVG control objects for a station.

    Arguments:
        station (str): Station identifier
        size: Image scale factor (number) or image size in pixels (nx, ny)
        force_size (bool): Whether to force `size` even if different aspect ratio
            than original size.
        keys (iterable): SVG layers to include
        correction: Whether control objects should use elevation correction (bool)
            or arguments to `glimpse.helpers.elevation_corrections()`
        station_calib (bool): Whether to load station calibration. If `False`,
            falls back to the station estimate.
        camera_calib (bool): Whether to load camera calibrations. If `False`,
            falls back to the EXIF estimate.

    Returns:
        list: Image objects
        list: Control objects (Points, Lines)
        list: Per-camera calibration parameters [{'viewdir': True}, ...]
    """
    paths = glob.glob(os.path.join(CG_PATH, 'svg', station + '*.svg'))
    if synth:
        paths += glob.glob(os.path.join(CG_PATH, 'svg-synth', station + '*.pkl'))
        paths += glob.glob(os.path.join(CG_PATH, 'svg-synth', station + '*.svg'))
    basenames = np.unique([glimpse.helpers.strip_path(path) for path in paths])
    images, controls, cam_params = [], [], []
    for basename in basenames:
        calibration = load_calibrations(basename, camera=camera_calib,
            station=station_calib, station_estimate=not station_calib, merge=True)
        img_path = find_image(basename)
        img = glimpse.Image(img_path, cam=calibration)
        control = []
        if svgs is None or basename in svgs:
            control += svg_controls(img, keys=keys, correction=correction, step=step)
        if synth:
            control += synth_controls(img, step=None, directions=False)
        if control:
            for x in control:
                x.resize(size, force=force_size)
            images.append(img)
            controls.extend(control)
            cam_params.append(dict(viewdir=True))
    return images, controls, cam_params

def camera_svg_controls(camera, size=1, force_size=False, keys=None,
    svgs=None, correction=True, step=None, station_calib=False,
    camera_calib=False, synth=True):
    """
    Return all SVG control objects available for a camera.

    Arguments:
        camera (str): Camera identifer
        size: Image scale factor (number) or image size in pixels (nx, ny)
        force_size (bool): Whether to force `size` even if different aspect ratio
            than original size.
        keys (iterable): SVG layers to include
        svgs (iterable): SVG basenames to include
        correction: Whether control objects should use elevation correction (bool)
            or arguments to `glimpse.helpers.elevation_corrections()`
        station_calib (bool): Whether to load station calibration. If `False`,
            falls back to the station estimate.
        camera_calib (bool): Whether to load camera calibrations. If `False`,
            falls back to the EXIF estimate.

    Returns:
        list: Image objects
        list: Control objects (Points, Lines)
        list: Per-camera calibration parameters [{'viewdir': True}, ...]
    """
    paths = glob.glob(os.path.join(CG_PATH, 'svg', '*.svg'))
    if synth:
        paths += glob.glob(os.path.join(CG_PATH, 'svg-synth', '*.pkl'))
        paths += glob.glob(os.path.join(CG_PATH, 'svg-synth', '*.svg'))
    basenames = np.unique([glimpse.helpers.strip_path(path) for path in paths])
    images, controls, cam_params = [], [], []
    for basename in basenames:
        ids = parse_image_path(basename, sequence=True)
        if ids['camera'] == camera:
            calibration = load_calibrations(basename, camera=camera_calib,
                station=station_calib, station_estimate=not station_calib, merge=True)
            img_path = find_image(basename)
            img = glimpse.Image(img_path, cam=calibration)
            control = []
            if svgs is None or basename in svgs:
                control += svg_controls(img, keys=keys, correction=correction, step=step)
            if synth:
                control += synth_controls(img, step=None, directions=False)
            if control:
                for x in control:
                    x.resize(size, force=force_size)
                images.append(img)
                controls.extend(control)
                cam_params.append(dict(viewdir=True))
    return images, controls, cam_params

def camera_motion_matches(camera, size=None, force_size=False,
    station_calib=False, camera_calib=False):
    """
    Returns all motion Matches objects available for a camera.

    Arguments:
        camera (str): Camera identifier
        size: Image scale factor (number) or image size in pixels (nx, ny)
        force_size (bool): Whether to force `size` even if different aspect ratio
            than original size.
        station_calib (bool): Whether to load station calibration. If `False`,
            falls back to the station estimate.
        camera_calib (bool): Whether to load camera calibrations. If `False`,
            falls back to the EXIF estimate.

    Returns:
        list: Image objects
        list: Matches objects
        list: Per-camera calibration parameters [{}, {'viewdir': True}, ...]
    """
    motion = glimpse.helpers.read_json(os.path.join(CG_PATH, 'motion.json'))
    sequences = [item['paths'] for item in motion
        if parse_image_path(item['paths'][0], sequence=True)['camera'] == camera]
    all_images, all_matches, cam_params = [], [], []
    for sequence in sequences:
        paths = [find_image(path) for path in sequence]
        cams = [load_calibrations(path,  camera=camera_calib,
            station=station_calib, station_estimate=not station_calib, merge=True)
            for path in paths]
        images = [glimpse.Image(path, cam=cam)
            for path, cam in zip(paths, cams)]
        matches = [load_motion_match(images[i], images[i + 1])
            for i in range(len(sequence) - 1)]
        if size is not None:
            for match in matches:
                match.resize(size, force=force_size)
        all_images.extend(images)
        all_matches.extend(matches)
        cam_params.extend([dict()] + [dict(viewdir=True)] * (len(sequence) - 1))
    return all_images, all_matches, cam_params

def build_sequential_matches(images, detect=dict(), match=dict()):
    """
    Returns Matches objects for sequential Image pairs.

    Arguments:
        images (iterable): Image objects
        detect (dict): Arguments passed to `glimpse.optimize.detect_keypoints()`
        match (dict): Arguments passed to `glimpse.optimize.match_keypoints()`
    """
    keypoints = [glimpse.optimize.detect_keypoints(img.read(), **detect) for img in images]
    matches = []
    for i in range(len(images) - 1):
        uvA, uvB = glimpse.optimize.match_keypoints(keypoints[i], keypoints[i + 1], **match)
        matches.append(glimpse.optimize.Matches(
            cams=(images[i].cam, images[i + 1].cam), uvs=(uvA, uvB)))
    return matches

def load_motion_match(imgA, imgB):
    """
    Returns motion Matches object for an Image pair.

    Arguments:
        imgA (Image): Image object
        imgB (Image): Image object
    """
    basename = glimpse.helpers.strip_path(imgA.path) + '-' + glimpse.helpers.strip_path(imgB.path)
    path = os.path.join(CG_PATH, 'motion', basename + '.pkl')
    match = glimpse.helpers.read_pickle(path)
    match.cams = (imgA.cam, imgB.cam)
    return match

# ---- Calibrations ----

def load_calibrations(path=None, station_estimate=False, station=False,
    camera=False, image=False, viewdir=False, merge=False, file_errors=True):
    """
    Return camera calibrations.

    Arguments:
        path (str): Image basename or path
        station_estimate: Whether to load station estimate (bool) or
            station identifier to load (str).
            If `True`, the station identifier is parsed from `path`.
            If `path` or `image` specified, `viewdir` is based on the position
            of the image in the motion break sequence.
        station: Whether to load station (bool) or
            station identifier to load (str).
            If `True`, the station identifier is parsed from `path`.
            viewdir is loaded from station_estimate.
        camera: Whether to load camera (bool) or
            camera identifier to load (str).
            If `True`, the camera identifier is parsed from `path`.
        image: Whether to load image (bool) or
            image to load (str).
            If `True`, the image basename is parsed from `path`.
        viewdir: Whether to load view direction (bool) or
            view direction to load (str).
            If `True`, the image basename is parsed from `path`.
        merge (bool): Whether to merge calibrations, in the order
            station_estimate, station, camera, image, viewdir
        file_errors (bool): Whether to raise an error if a requested calibration
            file is not found
    """
    def _try_except(fun, arg, **kwargs):
        try:
            return fun(arg, **kwargs)
        except FileNotFoundError as e:
            if file_errors:
                raise e
            else:
                return None
    if path:
        ids = parse_image_path(path, sequence=(camera is True))
        if station_estimate is True:
            station_estimate = ids['station']
        if station is True:
            station = ids['station']
        if camera is True:
            camera = ids['camera']
        if image is True:
            image = ids['basename']
        if viewdir is True:
            viewdir = ids['basename']
    calibrations = dict()
    if station_estimate:
        img_path = image if isinstance(image, str) else path if path else None
        calibrations['station_estimate'] = _try_except(_load_station_estimate, station_estimate, path=img_path)
    if station:
        calibrations['station'] = _try_except(_load_station_estimate, station, path=path)
        station_calib = _try_except(_load_station, station)
        if station_calib and 'xyz' in station_calib:
            calibrations['station']['xyz'] = station_calib['xyz']
    if camera:
        calibrations['camera'] = _try_except(_load_camera, camera)
    if image:
        calibrations['image'] = _try_except(_load_image, image)
    if viewdir:
        calibrations['viewdir'] = _try_except(_load_viewdir, viewdir)
    if merge:
        return merge_calibrations(calibrations)
    else:
        return calibrations

def merge_calibrations(calibrations, keys=('station_estimate', 'station', 'camera', 'image', 'viewdir')):
    """
    Merge camera calibrations.

    Arguments:
        calibrations (iterable): Dictionaries of calibration parameters
        keys (iterable): Calibration types, in order from lowest to highest
            overwrite priority
    """
    calibration = dict()
    for key in keys:
        if key in calibrations and calibrations[key]:
            calibration = glimpse.helpers.merge_dicts(calibration, calibrations[key])
    return calibration

def _load_station_estimate(station, path=None):
    stations = Stations()
    feature = stations[station]
    viewdir = feature['properties']['viewdir']
    i = _station_break_index(path) if path else 0
    if i and 'viewdir' in feature['properties']['breaks'][i - 1]:
        viewdir = feature['properties']['breaks'][i - 1]['viewdir']
    return dict(
        xyz=np.reshape(feature['geometry']['coordinates'], -1),
        viewdir=viewdir)

def _load_station(station):
    station_path = os.path.join(CG_PATH, 'stations', station + '.json')
    return glimpse.helpers.read_json(station_path)

def _load_camera(camera):
    camera_path = os.path.join(CG_PATH, 'cameras', camera + '.json')
    return glimpse.helpers.read_json(camera_path)

def _load_image(path):
    basename = glimpse.helpers.strip_path(path)
    image_path = os.path.join(CG_PATH, 'images', basename + '.json')
    return glimpse.helpers.read_json(image_path)

def _load_viewdir(path):
    basename = glimpse.helpers.strip_path(path)
    viewdir_path = os.path.join(CG_PATH, 'viewdirs', basename + '.json')
    return glimpse.helpers.read_json(viewdir_path)

def write_image_viewdirs(images, viewdirs=None):
    """
    Write Image view directions to file.

    Arguments:
        images (iterable): Image objects
        viewdirs (iterable): Camera view directions to write.
            If `None`, these are read from `images[i].cam.viewdir`.
    """
    for i, img in enumerate(images):
        basename = glimpse.helpers.strip_path(img.path)
        path = os.path.join(CG_PATH, 'viewdirs', basename + '.json')
        if viewdirs is None:
            d = dict(viewdir=tuple(img.cam.viewdir))
        else:
            d = dict(viewdir=tuple(viewdirs[i]))
        glimpse.helpers.write_json(d, path=path)

# ---- Tracking ----

@lru_cache(maxsize=1)
def Termini():
    """
    Return terminus traces.
    """
    path = os.path.join(CG_PATH, 'geojson', 'termini.geojson')
    geo = glimpse.helpers.read_geojson(path, crs=32606)['features']
    # Check that all termini run west to east
    bad = []
    for f in geo:
        x_start = f['geometry']['coordinates'][0, 0]
        x_end = f['geometry']['coordinates'][-1, 0]
        if x_start > x_end:
            bad.append((f['properties']['date'], f['properties']['type']))
    if bad:
        raise ValueError('Some termini traced east to west:' + str(bad))
    # Sort by date, then type
    geo.sort(key=lambda x: (x['properties']['date'], x['properties']['type']))
    return geo

@lru_cache(maxsize=1)
def Glacier():
    """
    Return the maximal glacier extent.
    """
    path = os.path.join(CG_PATH, 'geojson', 'glacier.geojson')
    geo = glimpse.helpers.read_geojson(path, crs=32606)
    return geo['features'][0]['geometry']['coordinates'][0]

@lru_cache(maxsize=1)
def Coast():
    """
    Return the forebay coastlines.
    """
    path = os.path.join(CG_PATH, 'geojson', 'coast.geojson')
    geo = glimpse.helpers.read_geojson(path, crs=32606, key='id')
    return {key: geo['features'][key]['geometry']['coordinates'] for key in geo['features']}

@lru_cache(maxsize=1)
def Forebay():
    """
    Return the forebay polygon.
    """
    path = os.path.join(CG_PATH, 'geojson', 'forebay.geojson')
    geo = glimpse.helpers.read_geojson(path, crs=32606)
    return geo['features'][0]['geometry']['coordinates'][0]

def parse_dem_path(path):
    """
    Return datetime and type from DEM path.
    """
    datestr = re.findall(r'([0-9]{8})', glimpse.helpers.strip_path(path))[0]
    # HACK: Aerial and satellite imagery taken around local noon (~ 22:00 UTC)
    t = datetime.datetime.strptime(datestr + str(22), '%Y%m%d%H')
    typestr = re.findall(r'dem-([^\/]+)', path)[0]
    return dict(datetime=t, type=typestr)

def get_dem_terminus(t, demtype):
    """
    Return the terminus corresponding to a DEM path.
    """
    termini = Termini()
    termini_keys = [(f['properties']['date'], f['properties']['type'])
        for f in termini]
    i = termini_keys.index((t.strftime('%Y-%m-%d'), demtype))
    return termini[i]['geometry']['coordinates']

def get_nearest_terminus(t):
    """
    Return the terminus nearest a datetime.
    """
    types = ('aerometric', 'arcticdem', 'ifsar', 'tandem',
        'landsat-8', 'landsat-7', 'terrasar')
    termini = [f for f in Termini() if
        len(f['properties']['date']) == 10 and
        f['properties']['type'] in types]
    termini.sort(key=lambda x: (x['properties']['date'],
        types.index(x['properties']['type'])))
    datetimes = [datetime.datetime.strptime(
        f['properties']['date'] + '22', '%Y-%m-%d%H') for f in termini]
    dt = np.abs(np.array(datetimes) - t)
    i = np.where(np.min(dt) == dt)[0][0]
    return termini[i]['geometry']['coordinates']

def clip_terminus_with_coast(line):
    """
    Clip a terminus with the west and east coastlines.
    """
    # Convert to shapely format
    coast = Coast()
    cline_west = shapely.geometry.LineString(coast['west'])
    cline_east = shapely.geometry.LineString(coast['east'])
    tline = shapely.geometry.LineString(line)
    # Cut terminus at west coastline
    tline = shapely.ops.split(tline, cline_west.buffer(distance=100))[-1]
    # Cut terminus at east coastline
    tline = shapely.ops.split(tline, cline_east.buffer(distance=100))[0]
    return np.asarray(tline.coords)

def clip_glacier_with_terminus(line):
    """
    Clip glacier extent with a terminus.
    """
    gpoly = shapely.geometry.Polygon(shell=Glacier())
    # Extend western edge past polygon boundary
    wpoint = shapely.geometry.Point(line[0])
    west_snaps = shapely.ops.nearest_points(wpoint, gpoly.exterior)
    if west_snaps[0] != west_snaps[1]:
        new_west = west_snaps[1].coords
        d = new_west - line[0]
        d /= np.linalg.norm(d)
        line = np.row_stack((new_west + d, line))
    # Extend eastern edge past polygon boundary
    epoint = shapely.geometry.Point(line[-1])
    east_snaps = shapely.ops.nearest_points(epoint, gpoly.exterior)
    if east_snaps[0] != east_snaps[1]:
        new_east = east_snaps[1].coords
        d = new_east - line[-1]
        d /= np.linalg.norm(d)
        line = np.row_stack((line, new_east + d))
    tline = shapely.geometry.LineString(line)
    # Split glacier at terminus
    splits = shapely.ops.split(gpoly, tline)
    if len(splits) < 2:
        raise ValueError('Glacier polygon not split by terminus')
    else:
        areas = [split.area for split in splits]
        return np.asarray(splits[np.argmax(areas)].exterior.coords)

def load_glacier_polygon(t, demtype=None):
    """
    Return the glacier extent at a datetime.
    """
    if demtype is not None:
        line = get_dem_terminus(t, demtype=demtype)
    else:
        line = get_nearest_terminus(t)
    cline = clip_terminus_with_coast(line)
    return clip_glacier_with_terminus(cline)

def load_forebay_polygon(glacier):
    """
    Return the forebay extent for a given glacier extent.
    """
    gpoly = shapely.geometry.Polygon(shell=glacier)
    fpoly = shapely.geometry.Polygon(shell=Forebay())
    diff = fpoly.difference(gpoly)
    return np.asarray(diff.exterior.coords)

def intersect_polygons(polygons):
    """
    Return intersection of polygons.
    """
    shapes = [shapely.geometry.Polygon(shell=xy) for xy in polygons]
    shape = shapes[0]
    for i in range(1, len(shapes)):
        shape = shape.intersection(shapes[i])
    if np.iterable(shape):
        i = np.argmax([poly.area for poly in shape])
        shape = shape[i]
    return np.asarray(shape.exterior.coords)

def select_track_points(xy, images, polygon, dem, max_distance):
    """
    Return track points mask for a set of starting images.

    Returns:
        array: Coordinates of points to track (n, 2)
        array: Visibility mask for each image (n, m)
    """
    # In DEM, in polygon, and DEM not NaN
    z = dem.sample(xy, bounds_error=False, fill_value=np.nan)
    mask = ~np.isnan(z) & glimpse.helpers.points_in_polygon(xy, polygon)
    # Visible in one or more images
    xyz = np.column_stack((xy, z))[mask]
    visible = np.tile(mask.reshape(-1, 1), reps=(1, len(images)))
    for i, img in enumerate(images):
        uv = img.cam.project(xyz, correction=True)
        # In image frame
        visible[mask, i] &= img.cam.inframe(uv)
        # In range
        distance = np.linalg.norm(xyz[:, 0:2] - img.cam.xyz[0:2], axis=1)
        visible[mask, i] &= distance < max_distance
        # In DEM viewshed
        viewshed = glimpse.Raster(
            Z=dem.viewshed(img.cam.xyz), x=dem.xlim, y=dem.ylim)
        visible[mask, i] &= viewshed.sample(xyz[:, 0:2], order=1) > 0.99
        # Not in land mask
        land_mask = glimpse.Raster(load_masks([img])[0])
        visible[mask, i] &= land_mask.sample(uv, order=1, bounds_error=False,
            fill_value=1.0) == 0
    return visible


import torch._subclasses
from torch._subclasses.fake_tensor import DynamicOutputShapeException

def is_builtin(op):
    return op.namespace in ('aten', 'prims', 'prim')

def fake_check(op, args, kwargs, dynamic_only):
    with torch._subclasses.CrossRefFakeMode(ignore_op_fn=is_builtin):
        try:
            op(*args, **kwargs)
        except DynamicOutputShapeException:
            if not dynamic_only:
                raise
            return
        if dynamic_only:
            raise AssertionError(
                f"fake_check({op}, ..., dynamic_only={dynamic_only}): "
                f"dynamic_only means that the operator is expected to have "
                f"data-dependent output shape. We have not detected that this is "
                f"the case. Please check that your operator's FakeTensor "
                f"implementation is actually data dependent")


import csv
from collections import defaultdict

def sum_and_count_values(csv_file):
    # Read the CSV file into a dictionary of lists
    with open(csv_file, 'r') as file:
        reader = csv.reader(file)
        rows = list(reader)

    # Create a dictionary to store the sums and counts
    sum_counts = defaultdict(lambda: [0, 0])  # [Sum, Count]

    # Calculate the HRI
    for row in rows[1:]:
        try:
            key = row[1] # owner is the key value
            value = int(row[2]) # the rank in the NFT collection
            sum_counts[key][0] += value  # Sum = running total of all rank values
            sum_counts[key][1] += 1      # Count aka Total Comboeys: is assigned to each owner bc that's our primary key
            # Do the Comverse special maths to calculate HRI
            # 1. Divide the Sum by Count, get Average
            sum_counts[key][0] = sum_counts[key][0]/sum_counts[key][1]
            # 2. Divide Average by 2x count
            sum_counts[key][0] = sum_counts[key][0]/(sum_counts[key][1]*2)
            # 3. if NFT count < 10, incur HRI balance penalty
            if sum_counts[key][1] < 10:
                sum_counts[key][0] = sum_counts[key][0] + (10 - sum_counts[key][1])
        except ValueError:
            continue
    # Create the new CSV file with three columns
    new_csv_file = 'HRI_combeys.csv'
    with open(new_csv_file, 'w', newline='') as file:
        writer = csv.writer(file)
        writer.writerow(['Holder', 'HRI', 'Combeys'])
        for key, (sum_val, count) in sum_counts.items():
            writer.writerow([key, sum_val, count])

    return new_csv_file

# Usage example
csv_file = './combeys-holders-ranks.csv'
new_csv_file_path = sum_and_count_values(csv_file)
print(f"New CSV file created: {new_csv_file_path}")


import requests
import json
from geopy.geocoders import Nominatim
from dotenv import load_dotenv
import os
import pandas as pd
from utils import time_difference_correction

load_dotenv()

# Load Meteostat API credentials from .env file
api_key = os.environ['Meteostats_API_KEY']
api_host = os.environ['Meteostats_API_HOST']

# Create a pandas Timedelta object of 30 days
ONE_MONTH = pd.Timedelta('30 days')

def max_30_days_timedeltas(df: pd.DataFrame) -> pd.DataFrame:
    """This function takes a pandas dataframe and return another one where the time difference between the start and
    end dates is limited to 30 days or less, as the API doesn't allow requests for longer time periods"""
    # List to store new rows
    new_rows = []
    for _, row in df.iterrows():
        delta = row['end_date'] - row['start_date']
        # If delta is greater than 30 days, then we split the delta in parts no larger than 30 days
        if delta > ONE_MONTH:
            curr_date = row['start_date']
            while curr_date < row['end_date']:
                new_end = min(curr_date + ONE_MONTH, row['end_date'])
                new_rows.append([curr_date, new_end, row['location']])
                curr_date = new_end + pd.Timedelta('1 day')
        else:
            new_rows.append([row['start_date'], row['end_date'], row['location']])
    new_df = pd.DataFrame(new_rows, columns=df.columns)
    return new_df

def lat_long_location(location_name: str):
    """This function takes a location name and returns its latitude and longitude coordinates using Geopy"""
    # Creating a geolocator object using Nominatim
    geolocator = Nominatim(user_agent='my_application')
    # Getting the location object
    location = geolocator.geocode(location_name)
    # Extracting the latitude and longitude coordinates from the location object
    latitude = location.latitude
    longitude = location.longitude
    return latitude, longitude

def api_call_nearest_station(latitude: float, longitude: float):
    """This function takes a latitude and longitude coordinates of a location and makes an API call to Meteostat API to get the ID of the nearest weather station"""
    # Define the API endpoint and set the query parameters
    url = "https://meteostat.p.rapidapi.com/stations/nearby"
    querystring = {"lat":latitude,"lon":longitude, "radius": 1000000}
    # Set the headers for the API request
    headers = {
        "X-RapidAPI-Key": api_key,
        "X-RapidAPI-Host": api_host
    }
    # Create a requests session and make the API call
    with requests.Session() as session:
        response = session.get(url, headers=headers, params=querystring)
    # Extract the station ID and name for the nearest weather station from the API response
    stations = json.loads(response.text)['data']
    stations_dict = {station['id']: station['name']['en'] for station in stations}
    return stations_dict

def api_call_weather_data(stations_dict: dict, start_date: str, end_date: str):
    """Calls the Meteostat API to get hourly weather data for a list of stations within a specified date range"""
    url = "https://meteostat.p.rapidapi.com/stations/hourly"
    with requests.Session() as session:
        for station in stations_dict.keys():
            querystring = {"station":station,"start":start_date,"end":end_date}
            headers = { "X-RapidAPI-Key": api_key,
                        "X-RapidAPI-Host": api_host }
            print(querystring)
            response = session.get(url, headers=headers, params=querystring)
            weather_data = json.loads(response.text)['data']
            if weather_data:
                station_id = station
                station_name = stations_dict[station]
                return weather_data, station_id, station_name

def dict_weather_code():
    dict_weather_code = {1:'Clear', 2:'Fair', 3:'Cloudy', 4:'Overcast', 5:'Fog', 6:'Freezing Fog', 7:'Light Rain',\
                         8:'Rain', 9:'Heavy Rain', 10:'Freezing Rain', 11:'Heavy Freezing Rain', 12:'Sleet',\
                         13:'Heavy Sleet', 14:'Light Snowfall', 15:'Snowfall', 16:'Heavy Snowfall', 17:'Rain Shower',\
                         18:'Heavy Rain Shower', 19:'Sleet Shower', 20:'Heavy Sleet Shower', 21:'Snow Shower', 22:'Heavy Snow Shower',\
                         23:'Lightning', 24:'Hail', 25:'Thunderstorm', 26:'Heavy Thunderstorm', 27:'Storm'}
    return dict_weather_code

def get_weather_data():
    # Read the location log file to obtain the start and end dates for each location
    df = pd.read_excel('files/work_files/weather_work_files/location_log.xlsx')

    # Filter out any rows with end dates greater than 30 days ago
    df = max_30_days_timedeltas(df)

    # Define the column names for the weather data DataFrame
    col_names = ['location', 'closest_station_id', 'closest_station_name', 'utc_timestamp', 'temperature',
                 'dew_point', 'relative_humidity_%', 'precipitation', 'snow_depth', 'wind_direction', 'wind_speed',
                 'peak_wind_gust', 'sea_level_air_pressure', 'sunshine_total_time','weather_condition_code']

    # Create an empty DataFrame to store the weather data
    df_weather = pd.DataFrame(columns=col_names, index=range(0,0))

    # Read in the alreadu processed weather data
    df_weather_processed = pd.read_csv('files/processed_files/weather_processed.csv', sep = '|')

    # Iterate over each row in the location log file
    for _, row in df.iterrows():

        # Check if the end date for this location has already been processed
        if (pd.to_datetime(df_weather_processed['utc_timestamp']) == row['end_date']).any():
            continue
        # Retrieve the necessary information for this location
        location_name = row['location']
        start_date = row['start_date'].strftime("%Y-%m-%d")
        end_date = row['end_date'].strftime("%Y-%m-%d")
        latitude, longitude = lat_long_location(location_name)
        stations_dict = api_call_nearest_station(latitude, longitude)
        weather_data, station_id, station_name = api_call_weather_data(stations_dict, start_date, end_date)

        # Convert the weather data for this location into a DataFrame and append it to the main DataFrame
        hourly_data = []
        for data_point in weather_data:
            hourly_data.append([location_name, station_id,station_name,data_point['time'],data_point['temp'],
                                data_point['dwpt'],data_point['rhum'],data_point['prcp'],data_point['snow'],
                                data_point['wdir'],data_point['wspd'],data_point['wpgt'],data_point['pres'],
                                data_point['tsun'],data_point['coco']])
        hourly_df = pd.DataFrame(hourly_data, columns=col_names)
        df_weather = pd.concat([df_weather, hourly_df], ignore_index=True)

    # Add additional columns to the weather data DataFrame
    df_weather['weather_assessment'] = df_weather['weather_condition_code'].map(dict_weather_code())
    df_weather['utc_timestamp'] = pd.to_datetime(df_weather['utc_timestamp'])
    df_weather['tz_timestamp'] = df_weather['utc_timestamp'].apply(lambda x: time_difference_correction(x))

    # Combine the newly processed data with the existing data and remove any duplicate rows
    df_weather = pd.concat([df_weather_processed, df_weather], ignore_index=True)
    df_weather.drop_duplicates(inplace=True)

    # Save to a CSV file
    df_weather.to_csv('files/processed_files/weather_processed.csv', sep = '|', index=False)


"""This module contains functions about retrieving of patients names,
 doctors names from Enrollment object, and lists all Enrollments connected
 to certain doctor(its owner). """

from django.shortcuts import render
from enrollment_system.models import Enrollment
from doctors_auth.models import DoctorProfile
from .forms import PatientFilterForm, DoctorFilterForm

def extract_filter_values(values):
    """It takes request param 'text' and returns its value"""
    text = values['text'] if 'text' in values else ''
    return text

def all_patients(request):
    """It retrieves patients names form Enrollments records. """

    text = extract_filter_values(request.GET)
    patient = Enrollment.objects.filter(patient_name__contains=text)
    context = {
        'patients': patient,
        'patient_filter_form': PatientFilterForm(initial={'text': text})
    }
    return render(request, 'hospital_units/patients_list.html', context)

def all_doctors(request):
    """It retrieves doctors names from DoctorProfile records. """

    text = extract_filter_values(request.GET)
    doctor = DoctorProfile.objects.filter(name__contains=text)
    context = {
        'doctors': doctor,
        'doctor_filter_form': DoctorFilterForm(initial={'text': text})
    }
    return render(request, 'hospital_units/doctors_list.html', context)

def doctors_records(request):
    """It filters Enrollment records by their owners. """

    context = {
        'enrollments': Enrollment.objects.filter(doctor_name_id=request.user.doctorprofile.id)
    }
    return render(request, 'hospital_units/doctors_enrollments.html', context)


#!/usr/bin/env python
"""
vmclone.py
----------
Re-Identify a cloned Linux Virtual Machine (Hostname & Networking)

See README.md for usage and more information
"""
__author__ = 'Blayne Campbell'
__date__ = '2014-01-29'
__version__ = '1.2.0'

from time import sleep
import subprocess
import datetime
import shutil
import json
import glob
import sys
import os
import re

date = str(datetime.datetime.now().strftime('%Y-%m-%d'))

# Set Working Directory
abspath = os.path.abspath(__file__)
script_path = os.path.dirname(abspath)
os.chdir(script_path)
try:
    from subprocess import DEVNULL  # py3k
except ImportError:
    DEVNULL = open(os.devnull, 'wb')

# Import Settings #
try:
    from settings import *
except ImportError:
    sys.exit("Unable to import settings..\n"
             "Try re-naming example-settings.py to settings.py")

# Validations #
valip = '\\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.)' \
        '{3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\b'
valmac = '\\b([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})\\b'
valproto = '\\b((?i)dhcp|(?i)none)\\b'
valtabs = '\\b(\\t+)\\b'
valyesno = '\\b((?i)yes|(?i)no)\\b'
# Validations End #

# Minimal Mode - Skip all dependency checks
minimal_mode = 0

def dependency_check():
    """
    Dependency check for required utilities
    """
    # List of applications required by script #
    all_dependencies = ['nc', 'ntp', 'ntpdate']
    missing_dependencies = []
    for dependency in all_dependencies:
        d = subprocess.Popen(['which', '%s' % dependency], stderr=DEVNULL)
        if d.wait() != 0:
            missing_dependencies.append(dependency)
    if len(missing_dependencies) >= 1:
        print('The following dependencies are not installed: %s'
              % " ".join(str(x) for x in missing_dependencies))
        dep_prompt = raw_input("Would you like to install these now?[y/N]")
        if dep_prompt.lower() == 'y':
            for dependency in missing_dependencies:
                d = subprocess.Popen(['yum', 'install', '%s' % dependency])
                if d.wait() != 0:
                    sys.exit("Problem while installing dependency: %s"
                             % dependency)
        else:
            sys.exit('You chose not to install dependencies.. Exiting.')
    return True

def get_release():
    release = subprocess.Popen(['cat', '/etc/redhat-release'],
                               stdout=subprocess.PIPE)
    release = release.stdout.read()
    release_number = re.match(r'^.*release\s(\d{1,2}).*$', release)
    if release_number:
        return release_number.group(1)
    else:
        return False

def get_interfaces():
    """ Return interface(s) reporting a permanent address via ethtool utility.
    :return: dict = {'interface': {'perm_address': '00:00:00:00:00:00'}
    """
    procnetdev = subprocess.Popen(['cat', '/proc/net/dev'],
                                  stdout=subprocess.PIPE).stdout.readlines()
    interface_list = []
    physical_interfaces = {}
    for i in procnetdev:
        iface = re.match(r'^(.*):.*$', i)
        if iface:
            interface_found = iface.group(1).strip()
            if interface_found != 'lo':
                interface_list.append(interface_found)
    for interface in interface_list:
        perm_addr = subprocess.Popen(['ethtool', '-P', interface],
                                     stdout=subprocess.PIPE,
                                     stderr=DEVNULL).stdout.readlines()
        if not perm_addr:
            continue
        mac = re.match(r'^Permanent\saddress:\s(.*)$', perm_addr[0])
        if mac:
            if mac.group(1) != '00:00:00:00:00:00':
                physical_interfaces[interface] = {'perm_address': mac.group(1)}
    return physical_interfaces

def show_usage():
    """
    Show script usage
    """
    print("\nUsage:\nclone: Re-identify this server (write networking files)")
    print("check: Show available Name servers and NTP servers\n")
    sys.exit('Try: %s <check|clone>\n' % sys.argv[0])

def current_hostname(lookup):
    """
    Find current hostname of system via /etc/sysconfig/network
    """
    cur = open(network)
    cur = cur.readlines()
    for i in cur:
        if lookup in i:
            host = i.split('=')[1]
            return host

def current_mac(cfgfile):
    """
    Find current MAC address in given interface configuration
    :param cfgfile: interface configuration
    :return: Current MAC address
    """
    cur = open(cfgfile)
    cur = cur.readlines()
    for i in cur:
        if 'HWADDR' in i:
            return i.strip()[7:]

def backup_file(cfgfile):
    """
    Creates Backup of configuration file
    :param cfgfile: configuration file
    :return: creates backup if backup not already exists
    """
    backup_dir = script_path + '/cfg_backups/%s' % date
    if not os.path.exists(backup_dir):
        os.makedirs(backup_dir)
    p, f = os.path.split(cfgfile)
    if os.path.isfile("%s%s" % (backup_dir, f)):
        pass
    else:
        if os.path.isfile(cfgfile):
            shutil.copy(cfgfile, backup_dir)
        else:
            print("Unable to backup %s. File does not exist." % cfgfile)

def findmac(iface):
    """
    Return permanent MAC address for interface
    """
    interfaces = get_interfaces()
    return interfaces[iface]['perm_address']

def valid_ip(ip):
    """
    Validate IP Addresses
    """
    valid = re.match(valip, ip)
    if valid:
        return True
    else:
        print("Invalid IP address...")

def replace(cfgfile, pattern, subst):
    """
    Matches a pattern in a file and replaces with provided substitution.
    """
    with open(cfgfile, 'r') as filein:
        filecont = filein.read()
    if re.search(pattern, filecont):
        filecont = (re.sub(pattern, subst, filecont))
        with open(cfgfile, 'w') as fileout:
            fileout.write(filecont)
    else:
        with open(cfgfile, 'a') as fileout:
            fileout.write('\n' + subst)

def gen_interface(cfgfile, iface):
    """
    Re-Generate ifcfg-eth<x> file with temporary network information
    :param cfgfile: full path to interface configuration
    :param iface: interface name (ie: eth0)
    :return:
    """
    newuuid = subprocess.Popen('uuidgen', stdout=subprocess.PIPE)
    newuuid = newuuid.stdout.readlines()[0].strip()
    print("Generating interface %s.." % iface)
    with open(cfgfile, 'w') as f:
        f.write('# This file was generated by vmclone.py on %s\n'
                % datetime.datetime.now())
        f.write('DEVICE=%s\n' % iface)
        f.write('HWADDR=%s\n' % findmac(iface))
        f.write('IPADDR=123.123.123.123\n')
        f.write('NETMASK=255.255.255.0\n')
        f.write('GATEWAY=123.123.123.123\n')
        f.write('BOOTPROTO=none\n')
        f.write('ONBOOT=yes\n')
        f.write('UUID=%s\n' % newuuid)
        f.write('NM_CONTROLLED=no\n')

def get_nameservers(write=None):
    """
    Display responsive nameservers outlined in the settings file
    :param write: Writes nameservers to resolv.conf
    :return: Nameservers found as reachable
    """
    print("\nChecking for available Name Servers..\n"
          "The following servers are reachable:\n")
    if write:
        with open(resolvconf, 'r') as f:
            lines = f.readlines()
        with open(resolvconf, 'w') as f:
            for line in lines:
                if "nameserver" in line:
                    pass
                else:
                    f.write(line)
    for ns in nameservers:
        if valid_ip(ns):
            r = subprocess.Popen(['nc', '-v', '-z', '%s' % ns, '53'],
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.STDOUT)
            if r.wait() == 0:
                print('nameserver %s' % ns)
                if write:
                    with open(resolvconf, 'a') as f:
                        f.write('\nnameserver %s' % ns)
    if write:
        print("The above servers have been written to %s" % resolvconf)

def get_ntpservers():
    """
    Display responsive NTP servers outlined in the settings file
    :return: NTP servers found as reachable
    """
    print("\nChecking for available NTP Servers..\n"
          "The following servers are reachable:\n")
    for ntp in ntpservers:
        r = subprocess.Popen(['ntpdate', '-u', '%s' % ntp],
                             stdout=subprocess.PIPE,
                             stderr=subprocess.STDOUT)
        if r.wait() == 0:
            print('server %s' % ntp)

def clean_shutdown(option):
    """
    Removes udev net rules and ssh host files that are automatically generated
    on boot.
    This ensures that any new servers cloned from this 'template' will
    have unique MAC addresses and ssh host keys.
    """
    if int(get_release()) <= 6:
        if os.path.isfile(persistent):
            print("deleting %s" % persistent)
            os.remove(persistent)
    for sshfile in glob.glob('/etc/ssh/ssh_host_*'):
        if os.path.isfile(sshfile):
            print("deleting %s" % sshfile)
            os.remove(sshfile)
    for i_cfg in glob.glob('%s/ifcfg-*' % ifcfg_path):
        if re.match(r'^.*ifcfg-lo$', i_cfg):
            continue
        else:
            backup_file(i_cfg)
            os.remove(i_cfg)
    sleep(2)
    if option == 'halt':
        command = "/sbin/shutdown -h now"
    else:
        command = "/sbin/shutdown -r now"
    subprocess.Popen(command.split())

class ServerClone:
    """
    Main Configuration Object
    """
    def __init__(self, mode=0):
        self.runmode = mode
        self.old_serv = current_hostname('HOSTNAME')
        self.new_serv = None
        self.interfaces = dict()
        self.oldip = 'IPADDR=%s' % valip
        self.oldnm = 'NETMASK=%s' % valip
        self.oldgw = 'GATEWAY=%s' % valip
        self.oldmac = 'HWADDR=%s' % valmac
        self.proto_dhcp = 'BOOTPROTO=%s' % valproto
        self.proto_stat = 'BOOTPROTO=none'
        self.ip = None
        self.nm = None
        self.gw = None
        self.new_ip = None
        self.new_nm = None
        self.new_gw = None
        self.ntppos = None

    def set_hostname(self):
        """
        Prompt for new hostname
        :return: Set object hostname
        """
        while not self.new_serv:
            self.new_serv = raw_input('Enter NEW Server Name: ')

    def set_ntpservers(self):
        """
        Tests NTP servers outlined in settings and writes accessible servers
        to /etc/ntp.conf
        :return:
        """
        if not os.path.exists(ntpconf):
            sys.exit("Unable to open %s" % ntpconf)
        print("\nChecking for available NTP Servers..\n"
              "The following servers are reachable:\n")
        with open(ntpconf, 'r') as f:
            lines = f.readlines()
            self.ntppos = [i for i, item in enumerate(lines)
                           if re.search(r'\bserver\b', item)]
        with open(ntpconf, 'w') as f:
            for line in lines:
                if "server" in line:
                    pass
                else:
                    f.write(line)
        accessible = []
        for ntp in ntpservers:
            r = subprocess.Popen(['ntpdate', '-u', '%s' % ntp],
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.STDOUT)
            if r.wait() == 0:
                accessible.append(ntp)
                print('server %s' % ntp)
        if accessible:
            if self.ntppos:
                with open(ntpconf, 'r') as f:
                    lines = f.readlines()
                for i, a in enumerate(accessible):
                    if i == 0:
                        newline = '\nserver %s\n' % a
                        lines.insert(self.ntppos[0], newline)
                    else:
                        newline = '\nserver %s' % a
                        lines.insert(self.ntppos[0], newline)
                with open(ntpconf, 'w') as out:
                    out.writelines(lines)
            else:
                with open(ntpconf, 'a') as out:
                    for i, a in enumerate(accessible):
                        newline = '\nserver %s' % a
                        out.write(newline)
            print("The above servers have been written to %s" % ntpconf)
        else:
            print("Warning:\n"
                  "All servers specified by settings are inaccessible.")

    def show_settings(self):
        """
        Show proposed object configuration
        :return:
        """
        os.system('clear')
        print("Proposed Network Configuration:")
        print("Hostname: %s\n" % clone.new_serv)
        for i in sorted(self.interfaces):
            print('Interface %s\n' % i)
            print("IP: %s\nNetmask: %s\nGateway: %s\n"
                  % (self.interfaces[i]['ip'],
                     self.interfaces[i]['nm'],
                     self.interfaces[i]['gw']))

    def confirm_settings(self):
        """
        Prompts to confirm settings before writing configuration files
        :return:
        """
        applyconf = raw_input("Would you like apply the above "
                              "configuration?[y/N]")
        if 'y' in applyconf:
            self.commit_settings()
        else:
            print("\nOK.. No changes have been made to this system.\n"
                  "The configuration you have entered will be saved.")
            sys.exit()

    def commit_settings(self):
        """
        Write configuration files
        """
        try:
            for i in sorted(self.interfaces):
                cfg_file = ifcfg_path + '/ifcfg-%s' % i
                backup_file(cfg_file)
                gen_interface(cfg_file, i)
                replace(cfg_file, self.oldip, 'IPADDR=%s'
                        % self.interfaces[i]['ip'])
                replace(cfg_file, self.oldnm, 'NETMASK=%s'
                        % self.interfaces[i]['nm'])
                replace(cfg_file, self.oldgw, 'GATEWAY=%s'
                        % self.interfaces[i]['gw'])
                replace(cfg_file, self.oldmac, 'HWADDR=%s' % findmac(i))
                replace(cfg_file, self.proto_dhcp, self.proto_stat)
                print('Interface %s configured..' % i)
                sleep(1)
            if self.old_serv:
                backup_file(network)
                replace(network, self.old_serv, self.new_serv)
            # Hosts strings
            host_pattern = '%s%s%s %s.%s\n' % (valip, valtabs, self.old_serv,
                                               self.old_serv, domain)
            # hostfile replacements
            backup_file(hosts)
            for i in sorted(self.interfaces):
                ip = self.interfaces[i]['ip']
                host_record = '%s\t\t%s %s.%s\n' % (ip, self.new_serv,
                                                    self.new_serv, domain)
                replace(hosts, host_pattern, host_record)
            print("Restarting the network service...")
            if get_release() >= '7':
                subprocess.call(['systemctl', 'restart', 'network.service'])
            else:
                subprocess.call('start_udev')
                subprocess.call(['service', 'network', 'restart'])
        except Exception as e:
            sys.exit(e)

def config_interface(preconf, interface):
    """
    Function for configuring interface IP addresses.
    :param preconf:
    :param interface:
    :return:
    """
    print('\nConfiguring Interface: %s' % interface)
    while True:
        if preconf == 1:
            ip = vmconf[interface]['ip']
            clone.ip = raw_input('Primary IP Address[%s]: ' % ip) or ip
            if valid_ip(clone.ip):
                clone.new_ip = 'IPADDR=%s' % clone.ip
                break
        else:
            clone.ip = raw_input('Primary IP Address: ')
            if valid_ip(clone.ip):
                clone.new_ip = 'IPADDR=%s' % clone.ip
                break
    while True:
        if preconf == 1:
            nm = vmconf[interface]['nm']
            clone.nm = raw_input('Primary Netmask[%s]: ' % nm) or nm
            if valid_ip(clone.nm):
                clone.new_nm = 'NETMASK=%s' % clone.nm
                break
        else:
            clone.nm = raw_input('Primary Netmask: ')
            if valid_ip(clone.nm):
                clone.new_nm = 'NETMASK=%s' % clone.nm
                break
    # Calculate Gateway based on IP & Netmask
    prcidr = sum([bin(int(x)).count('1') for x in clone.nm.split('.')])
    if clone.runmode == 0:
        clone.gw = IPNetwork('%s/%s' % (clone.ip, prcidr))[1].format()
        while True:
            clone.gw = raw_input('Primary Gateway IP[%s]: '
                                 % clone.gw) or clone.gw
            if valid_ip(clone.gw):
                clone.new_gw = 'GATEWAY=%s' % clone.gw
                break
    else:
        while True:
            if preconf == 1:
                gw = vmconf[interface]['gw']
                clone.gw = raw_input('Primary Gateway IP[%s]: '
                                       % gw) or gw
                if valid_ip(clone.gw):
                    clone.new_gw = 'GATEWAY=%s' % clone.gw
                    break
            else:
                clone.gw = raw_input('Primary Gateway IP: ')
                if valid_ip(clone.gw):
                    clone.new_gw = 'GATEWAY=%s' % clone.gw
                    break
    clone.interfaces[interface] = {'ip': clone.ip,
                                   'nm': clone.nm,
                                   'gw': clone.gw}

def main(preconf):
    clone.set_hostname()
    # while True:
    for i in sorted(get_interfaces()):
        config_interface(preconf, i)
    print("\n" * 2 + "Saving configuration..")
    sleep(1)
    with open(script_path + '/vmconf.json', 'w') as j:
        json.dump(clone.interfaces, j, sort_keys=True,
                  indent=4, separators=(',', ': '))
    clone.show_settings()
    clone.confirm_settings()

if __name__ == "__main__":
    try:
        from netaddr import IPNetwork
    except ImportError:
        print('Installing python-netaddr library..')
        netmod = subprocess.Popen(['yum', 'install', 'python-netaddr', '-y'],
                                  stdout=DEVNULL, stderr=subprocess.STDOUT)
        if netmod.wait() == 0:
            from netaddr import IPNetwork
        else:
            minimal_mode = 1
    os.system("clear")
    if not os.geteuid() == 0:
        sys.exit("\nOnly root can run this script\n")
    if len(sys.argv) == 2:
        if sys.argv[1] == 'check':
            if minimal_mode == 0:
                if dependency_check():
                    get_nameservers()
                    get_ntpservers()
            else:
                raw_input('Unable to perform \'check\'\nPress the \'any\' key'
                          'to exit.')
                sys.exit()
        elif sys.argv[1] == 'clone':
            clone = ServerClone(minimal_mode)
            try:
                with open('vmconf.json', 'r') as data:
                    print("Previous Configuration Detected. Loading...\n")
                    vmconf = json.load(data)
                    main(1)
            except IOError:
                main(2)
            print('\n\n' + ('=' * 45))
            print("Would you like to prepare the server for cloning?\n"
                  "Answering \'yes\' will do the following:\n"
                  "Remove:\n"
                  "/etc/ssh/ssh_host_*\n"
                  "/etc/sysconfig/network-scripts/ifcfg-eth*\n"
                  "udev 70-persistent-net.rules (CentOS 6 Only)\n"
                  ".. followed by a shutdown (halt)\n\n")
            clean = raw_input("Prepare to Clone? [y/N]")
            if clean.lower() == 'y':
                clean_shutdown('halt')
            else:
                pass
        else:
            show_usage()
    else:
        current_interfaces = get_interfaces()
        print("Current detected interfaces:\n")
        for c_interface in sorted(current_interfaces):
            interface_mac = current_interfaces[c_interface]['perm_address']
            print("%s - %s" % (c_interface, interface_mac))
        print('\n')
        interface_prompt = raw_input("Are the above interfaces correct?[Y/n]")
        if interface_prompt.lower() == 'y':
            show_usage()
        else:
            reset_prompt = raw_input("Remove udev rules/ssh_host/ifcfg files "
                                     "and reboot?[y/N]")
            if reset_prompt.lower() == 'y':
                clean_shutdown('reboot')


import sys
import numpy as np
from scipy.linalg import eigh
import molecule

# read coordinate
input_geom = np.loadtxt(sys.argv[-1], comments='#')
mol = molecule.Molecule(input_geom)
print('***Cartesian Coordinates (a.u.)***')
mol.print_coord()

# read hessian
hess_file = sys.argv[-1].split("_")[0] + "_hessian.txt"
input_hess = np.loadtxt(hess_file, comments='#')
mol.hessian = input_hess.reshape( (mol.natom*3, mol.natom*3) )
print(mol.hessian)

# build mass weighted hessian
mol.mass_mat = np.repeat(mol.atms_mass, 3)
mol.mw_hess = np.einsum('ij,i,j->ij',
                        mol.hessian, mol.mass_mat**-.5, mol.mass_mat**-.5)

print(mol.mw_hess)

la, v = eigh(mol.mw_hess)
print(la)


from datetime import datetime
import logging
import os
from pathlib import Path
import subprocess
import sys
from typing import Any, Dict, List, Match, Optional, Set, Tuple, Union

import click

from DDR import config
from DDR import docstore
from DDR import fileio
from DDR import identifier
from DDR import storage
from DDR import util

logging.basicConfig(
    level=logging.ERROR,
    format='%(asctime)s %(levelname)-8s %(message)s',
    stream=sys.stdout,
)

@click.command()
@click.argument('fileroles')
@click.argument('sourcedir')
@click.argument('destbase')
@click.option('--force','-f',  is_flag=True, help='Force')
@click.option('--b2sync','-b',  is_flag=True, help='Sync with Backblaze (requires environment vars)')
@click.option('--rsync','-r',  help='Rsync to specified target')
def ddrpubcopy(fileroles, sourcedir, destbase, force, b2sync, rsync):
    """ddrpubcopy - Copies binaries from source dir to dest dir for publication.

    \b
    This command copies specified types of binaries from a collection to a
    destination folder.  Destination files are in a very simple hierarchy (just
    files within a single directory per collection) that is suitable for use by
    ddr-public.

    \b
    ddr-pubcopy produces a very simple file layout:
        $BASE/$COLLECTION_ID/$FILENAME

    \b
    An individual run of ddrpubcopy may copy from a subdirectory of a collection
    but these files will go into the collection's destination folder.
    Example:
        $ ddrpubcopy mezzanine /var/www/media/ddr/ddr-test-123 /media/USBHARDDRIVE

    \b
    Use the --b2sync/-b flag to upload files to Backblaze after rsyncing them
    to the dest dir.  In order to use this flag you must define the following
    environment values:
        B2KEYID   Backblaze Key ID
        B2APPKEY  Backblaze Application Key
        B2BUCKET  Backblaze target bucket name
    Example:
        $ export  B2KEYID=REDACTED
        $ export B2APPKEY=REDACTED
        $ export B2BUCKET=REDACTED
        $ ddrpubcopy mezzanine,transcript /var/www/media/ddr/ddr-test-123 \\
            /media/USBHARDDRIVE --b2sync

    \b
    Use the --rsync/-r arg to rsync files to another server after rsyncing them
    to the dest dir.  You will be asked for your password when rsyncing starts.
    Example:
        $ ddrpubcopy mezzanine,transcript /var/www/media/ddr/ddr-test-123 \\
            /media/USBHARDDRIVE --rsync=USER@HOST:/var/www/media
    """
    # validate inputs
    if fileroles == 'all':
        roles = identifier.VALID_COMPONENTS['role']
    else:
        roles = fileroles.replace(' ','').split(',')
        for role in roles:
            if role not in identifier.VALID_COMPONENTS['role']:
                valid_roles = ','.join(identifier.VALID_COMPONENTS['role'])
                click.echo(f'ERROR: File role "{role}" is invalid.')
                click.echo(f'Valid roles: {valid_roles}, or all')
                sys.exit(1)
    sourcedir = Path(sourcedir)
    destbase = Path(destbase)
    if destbase == sourcedir:
        click.echo('ERROR: Source and destination are the same!')
        sys.exit(1)
    try:
        sourcedir_oi = identifier.Identifier(path=sourcedir)
        cidentifier = sourcedir_oi.collection()
    except:
        click.echo('ERROR: Source dir must be part of a DDR collection.')
        sys.exit(1)
    # b2
    B2KEYID = os.environ.get('B2KEYID')
    B2APPKEY = os.environ.get('B2APPKEY')
    B2BUCKET = os.environ.get('B2BUCKET')
    if b2sync:
        if not (B2KEYID and B2APPKEY and B2BUCKET):
            click.echo(
                'ERROR: --b2sync requires environment variables ' \
                'B2KEYID, B2APPKEY, B2BUCKET'
            )
            sys.exit(1)
        click.echo('Backblaze: authenticating')
        click.echo(f'  B2BUCKET {B2BUCKET}')
        click.echo(f'  B2APPKEY ...{B2APPKEY[-6:]}')
        click.echo(f'  B2KEYID  ...{B2KEYID[-6:]}')
        try:
            backblaze = storage.Backblaze(B2KEYID, B2APPKEY, B2BUCKET)
        except Exception as err:
            click.echo(f'ERROR: {err}')
            sys.exit(1)

    # prepare
    started = datetime.now()
    LOG = destbase / 'ddrpubcopy.log'
    # Note: Files from subdirectories of a collection will all go to a single
    # collection tmpdir
    destdir = destbase / cidentifier.id
    # if collection dir doesn't exist in destdir, mkdir
    if not destdir.exists():
        destdir.mkdir(parents=True)

    # do the work
    logprint(LOG, f'Finding files')
    files = find_files(sourcedir, LOG)
    logprint(LOG, f'found {len(files)}')
    logprint(LOG, 'Filtering: {}'.format(','.join(roles)))
    to_copy = filter_files(files, roles, force, LOG)
    if to_copy:
        num = len(to_copy)
        for n,path_status in enumerate(
                rsync_to_tmpdir(to_copy, cidentifier.path_abs(), destdir, LOG)
        ):
            path,status = path_status
            if status != 'exists':
                logprint(LOG, f'{n}/{num} {status} {path}')
        num_files = len([f for f in destdir.iterdir()])
        logprint(LOG, f'{num_files} files in {destdir}')
        if b2sync:
            logprint(LOG, f'Backblaze: syncing {destdir}')
            for line in backblaze.sync_dir(destdir, basedir=cidentifier.id):
                # b2sdk output is passed to STDOUT
                pass
        if rsync:
            for output in rsync_to_target(destdir, rsync, LOG):
                if output:
                    logprint(LOG, output)
    finished = datetime.now()
    elapsed = finished - started
    logprint(LOG, 'DONE!')
    logprint_nots(LOG, '%s elapsed' % elapsed)

def dtfmt(dt: datetime) -> str:
    """Consistent date format.
    """
    return dt.strftime('%Y-%m-%dT%H:%M:%S.%f')

def logprint(filename: Path, msg: str):
    """Print to log file and console, with timestamp.
    """
    msg = '%s - %s\n' % (dtfmt(datetime.now()), msg)
    fileio.append_text(msg, str(filename))
    click.echo(msg.strip('\n'))

def logprint_nots(filename: Path, msg: str):
    """Print to log file and console, no timestamp.
    """
    msg = '%s\n' % msg
    fileio.append_text(msg, str(filename))
    click.echo(msg.strip('\n'))

def _subproc(cmd):
    """Run a command and yield stdout as it appears
    """
    popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, universal_newlines=True)
    for stdout_line in iter(popen.stdout.readline, ""):
        yield stdout_line
    popen.stdout.close()
    return_code = popen.wait()
    if return_code:
        raise subprocess.CalledProcessError(return_code, cmd)

def find_files(sourcedir: Path, LOG) -> List[Path]:
    """List files using git-annex-find.

    Only includes files present in local filesystem.
    This avoids rsync errors for missing files.
    """
    os.chdir(sourcedir)
    cmd = 'git annex find'
    return [
        Path(path.strip())
        for path in [
            line for line in _subproc(cmd.split())
        ]
        if path
    ]

def filter_files(files: List[Path],
                 roles: List[str],
                 force: bool,
                 LOG: Path) -> List[Path]:
    """Binary and access files for each File in roles
    """
    num_files_total = len(files)
    logprint(LOG, f'{num_files_total} collection files')
    # load objects
    # extract file IDs from list of files, rm duplicates
    oids = sorted(list(set([
        os.path.splitext(os.path.basename(
            str(path).replace(config.ACCESS_FILE_SUFFIX, '')
        ))[0]
        for path in files
    ])))
    num_objects_total = len(oids)
    oidentifiers = [
        oi
        for oi in [
            identifier.Identifier(oid, config.MEDIA_BASE)
            for oid in oids
        ]
        if oi.idparts['role'] in roles
    ]
    parents = {
        oid: oi.object()
        for oid,oi in docstore._all_parents(oidentifiers).items()
    }
    publishable = [
        x['identifier'].object()
        for x in docstore.publishable(oidentifiers, parents)
        if x['action'] == 'POST'
    ]
    num_objects_publishable = len(publishable)
    logprint(LOG, f'{num_objects_publishable}/{num_objects_total} publishable objects')
    # list binaries and access files
    binaries = [
        Path(o.path_rel) for o in publishable if Path(o.path_abs).exists()
    ]
    accesses = [
        Path(o.access_rel) for o in publishable if Path(o.access_abs).exists()
    ]
    paths = sorted(list(set(binaries + accesses)))
    num_files_tocopy = len(paths)
    logprint(LOG, f'{num_files_tocopy}/{num_files_total} publishable files')
    return paths

def rsync_to_tmpdir(to_copy: List[Path],
                collection_path: Path,
                destdir: Path,
                LOG: Path) -> List[str]:
    """Rsync files from collection to S3-style bucket tmpdir

    Skip files already present in destdir
    """
    os.chdir(collection_path)
    errs = []
    for n,f in enumerate(to_copy):
        src = collection_path / f
        src = f
        dest = destdir / f.name
        if dest.exists():
            yield dest,'exists'
        else:
            cmd = 'rsync --copy-links %s %s/' % (src, destdir)
            for x in _subproc(cmd.split()):
                pass
            yield dest,'rsync'

def rsync_to_target(
        tmpdir: Path,
        target: str,
        LOG: Path) -> List[str]:
    """Rsync from tmpdir/COLLECTIONID to target
    """
    os.chdir(tmpdir)
    cmd = f'rsync -avz {tmpdir} {target}'
    logprint(LOG, cmd)
    for x in _subproc(cmd.split()):
        yield x


# -*- coding: utf-8 -*-

# (c) 2016 Marcos Dione <mdione@grulic.org.ar>

# This file is part of ayrton.
#
# ayrton is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# ayrton is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with ayrton.  If not, see <http://www.gnu.org/licenses/>.

from importlib.abc import MetaPathFinder, Loader
try:
    # py3.4
    from importlib.machinery import ModuleSpec
except ImportError:  # pragma: no cover
    # py3.3
    pass  # sorry, no support

import sys
import os
import os.path

import logging
logger= logging.getLogger ('ayrton.importer')

from ayrton.file_test import a, d
from ayrton import Ayrton
import ayrton.utils

class AyrtonLoader (Loader):

    @classmethod
    def exec_module (klass, module):
        # module is a freshly created, empty module
        # «the loader should execute the module’s code
        # in the module’s global name space (module.__dict__).»
        load_path= module.__spec__.origin
        logger.debug2 ('loading %s [%s]', module, load_path)
        # I *need* to polute the globals, so modules can use any of ayrton's builtins
        loader= Ayrton (g=module.__dict__)
        loader.run_file (load_path)

        # set the __path__
        # TODO: read PEP 420
        init_file_name= '__init__.ay'
        if load_path.endswith (init_file_name):
            # also remove the '/'
            module.__path__= [ load_path[:-len (init_file_name)-1] ]

        logger.debug3 ('module.__dict__: %s ', ayrton.utils.dump_dict (module.__dict__))

loader= AyrtonLoader ()

class AyrtonFinder (MetaPathFinder):

    @classmethod
    def find_spec (klass, full_name, paths=None, target=None):
        # full_name is the full python path (as in grandparent.parent.child)
        # and path is the path of the parent (in a list, see PEP 420);
        # if None, then we're loading a root module
        # let's start with a single file
        # TODO: read PEP 420 :)
        logger.debug2 ('searching for %s under %s for %s', full_name, paths, target)
        last_mile= full_name.split ('.')[-1]

        if paths is not None:
            python_path= paths  # search only there
        else:
            python_path= sys.path

        logger.debug2 (python_path)
        for path in python_path:
            full_path= os.path.join (path, last_mile)
            init_full_path= os.path.join (full_path, '__init__.ay')
            module_full_path= full_path+'.ay'

            logger.debug2 ('trying %s', init_full_path)
            if -d (full_path) and -a (init_full_path):
                logger.debug2 ('found package %s', full_path)
                return ModuleSpec (full_name, loader, origin=init_full_path)

            else:
                logger.debug2 ('trying %s', module_full_path)
                if -a (module_full_path):
                    logger.debug2 ('found module %s', module_full_path)
                    return ModuleSpec (full_name, loader, origin=module_full_path)

        logger.debug2 ('404 Not Found')
        return None

finder= AyrtonFinder ()

# I must insert it at the beginning so it goes before FileFinder
sys.meta_path.insert (0, finder)


# -*- coding: utf-8 -*-
"""
Created on Sat Jan 19 13:25:54 2019

*args

source = 'source name', startID = int(optional)

if no int is given, id's will start at 001
example: {'WebSource_001': ['feces', 'comes with the other hole']}

parse_html module searches for patterns of:
    <tag>word or words (2-3)</tag>
    <tag>paragraph</tag>

and returns a dictionary of:
    {'Source_UniqueID': ['frontface', 'backface']}

TODO:
    Separate gen function and parser
    Refactor and check is passing too much
    Add API interface specification

@author: vince
"""
import re
from bs4 import BeautifulSoup

def searchDataType(**kwargs):
    dataInfo = kwargs
    dataFile = dataInfo['data']

    # read a line of input to assess type

    #html
    pattern = '<*>'
    rep = re.compile(pattern)
    if rep.search(dataFile):
        return {'type': 'html', 'file': dataFile}

def validateData(terms, descriptions):
    return len(terms) == len(descriptions)

def packData(**kwargs):

    terms = kwargs['terms']
    descriptions = kwargs['descriptions']
    source = kwargs['source']

    packed_data = {}
    for i in range(len(terms)):
        uid = '{0}_{1:03d}'.format(source, i)
        packed_data[uid] = [terms[i],descriptions[i]]
    return packed_data

def cleanTags(instring):
    replace_set = ["b'", "
", "
", "", ""]

    for pattern in replace_set:
        instring = instring.replace(pattern, '')

    return instring

if __name__ == "__main__":

    raw = open("Lexicon_Raw_HTML.html", "r")
    firstLine = raw.readline()

    if searchDataType(data=firstLine)['type'] == 'html':
        soup = BeautifulSoup(firstLine, features='html.parser')

        ## Look for term and description tags
        if soup.dd:
            terms = soup.find_all('dt')
            descriptions = soup.find_all('dd')

            terms_cleaned = []
            for term in terms:
                if not term('a'):
                    terms_cleaned.append(term.string)

            descriptions_cleaned = []
            for description in descriptions:
                if not description.string:
                    for child in description.children:
                        descriptions_cleaned.append(cleanTags(str(child.encode('utf-8'))))
                else:
                    descriptions_cleaned.append(description.string)

    if validateData(terms_cleaned, descriptions_cleaned):
        print('Data validated')

    print(packData(terms=terms_cleaned, descriptions=descriptions_cleaned, source='PMP_Lexicon'))

    raw.close()


import os

from catalog.models import Entry, License, EntryType, Tag
from filestorage.models import FileSet
from django.core.files.base import ContentFile

def import_dir(dir_name):
    for f in os.listdir(dir_name):
        fn = os.path.join(dir_name, f)
        if os.path.isfile(fn):
            import_file(fn)

def import_file(file_name):
    f = open(file_name, 'rb')
    try:
        data = f.read()
    finally:
        f.close()

    info = {}
    files = {}
    target = None

    lines = data.splitlines(True)
    for line in lines:
        handled = False
        for q in ['title', 'description', 'license', 'author', 'created',
                  'modified', 'change_comments', 'entry_type', 'tags',
                  'maturity', 'url', 'pypi_name', 'pypi']:
            if line.upper().rstrip() == "***" + q.upper():
                target = (info, q)
                handled = True
        if handled:
            continue

        if line.startswith('***FILE:'):
            fn = line[8:].strip()
            if not fn:
                raise ValueError("Invalid file name %s (%s)" % (fn, file_name))
            target = (files, fn)
            continue
        elif line.startswith('***SNIPPET'):
            target = (files, 'snippet.py')
            continue
        else:
            target[0].setdefault(target[1], "")
            target[0][target[1]] += line

    if 'pypi' in info:
        info['pypi_name'] = info.pop('pypi')
    if 'snippet.py' in files:
        info['entry_type'] = 'snippet'

    info.setdefault('entry_type', 'module')
    info.setdefault('license', 'public-domain')
    info.setdefault('maturity', 0.0)

    if 'entry_type' in info:
        info['entry_type'] = EntryType.objects.get(name=info['entry_type'])
    if 'license' in info:
        info['license'] = License.objects.get(slug=info['license'])
    if 'tags' in info:
        info['tags'] = [Tag.objects.get(name=name) for name in
                        [x.strip() for x in info['tags'].split(',')]]

    entry = Entry.new_from_title(**info)
    fileset = FileSet.new_from_title(entry.slug)
    fileset.save()

    entry.files = fileset

    if 'snippet.py' in files:
        fileset.snippet = files['snippet.py']
    else:
        for name, data in files.items():
            fileset.write_file(name, ContentFile(data))

    fileset.save()
    entry.save()


from threading import Lock

import numpy as np
import sklearn
import tensorflow as tf
from tensorflow.keras.models import model_from_json
import tensorflow.keras.backend as K

class LocalModel(object):
    """
    Local Model
    Each Client has its own model. The Weights will be sent to the Server.
    The Server updates a Global Model and sends this back to the clients.
    """

    def __init__(self, model_config, data_collected, optimizer):
        """
        Create Local Modal.
        Retrieved Configuration from server and local client data are applied here.
        :param model_config:
        :param data_collected:
        :param optimizer:
        """
        assert optimizer is not None, "please prove a optimizer dict"
        assert optimizer['loss'] is not None, "a loss function must be set in the optimizer dict"
        assert optimizer['metrics'] is not None, "a metric must be defined in the optimizer dict"
        assert optimizer['optimizer'] is not None, "a optimizer must be defined in the optimizer dict"
        assert data_collected["x_train"] is not None, "X matrix for training must be provided"
        assert data_collected["y_train"] is not None, "y vector for training must be provided"
        assert data_collected["x_test"] is not None, "X matrix for testing must be provided"
        assert data_collected["y_test"] is not None, "y vector for testing must be provided"
        self.model_config = model_config
        self.update_lock = Lock()
        self.graph = tf.Graph()
        with self.graph.as_default():
            self.session = tf.Session()
            with self.session.as_default():
                self.model = model_from_json(model_config['model_json'])
                if len(self.model.layers) >= 32: # hardcoded if model is lfw
                    for l in self.model.layers[:-4]:
                        l.trainable = False
                self.optimizer = optimizer
                self.model.compile(loss=optimizer['loss'],
                                   optimizer=optimizer['optimizer'](),
                                   metrics=optimizer['metrics'])
                self.model._make_predict_function()

        self.x_train = np.array(data_collected["x_train"])
        self.y_train = np.array(data_collected["y_train"])
        self.x_test = np.array(data_collected["x_test"])
        self.y_test = np.array(data_collected["y_test"])

    def get_weights(self):
        """
        Get Keras Model Weights
        :return: weights
        """
        return self.model.get_weights()

    def set_weights(self, new_weights):
        """
        Sets the Keras model Weights
        :param new_weights:
        :return:
        """
        with self.update_lock:
            with self.graph.as_default():
                with self.session.as_default():
                    self.model.set_weights(new_weights)

    def get_batch(self, x, y):
        """
        Returns a random training batch
        :return: Training Batch
        """
        x, y = sklearn.utils.shuffle(x, y)
        residual = (len(x) % self.model_config['batch_size'])
        return x[:-residual], y[:-residual]

    def train_one_round(self):
        """
        Train one round
        :return: weights and score
        """

        x_train, y_train = self.get_batch(self.x_train, self.y_train)
        with self.update_lock:
            with self.graph.as_default():
                with self.session.as_default():
                    self.model.fit(x_train, y_train,
                                   epochs=self.model_config['epoch_per_round'],
                                   batch_size=self.model_config['batch_size'],
                                   verbose=1,
                                   validation_data=(x_train, y_train))
                    score = self.model.evaluate(x_train, y_train, batch_size=self.model_config['batch_size'], verbose=0)
                    score[0] = np.mean(score[0])
                    print('Train loss:', score[0])
                    print('Train accuracy:', score[1])
                    return self.model.get_weights(), score[0], score[1]

    def evaluate(self):
        """
        Evaluation fo Test set after global model converged
        :return:
        """
        with self.update_lock:
            with self.graph.as_default():
                with self.session.as_default():
                    x_test, y_test = self.get_batch(self.x_test, self.y_test)
                    score = self.model.evaluate(x_test, y_test, batch_size=self.model_config['batch_size'], verbose=0)
                    score[0] = np.mean(score[0])
                    print('Test loss:', score[0])
                    print('Test accuracy:', score[1])
                    return score

    def save_model(self, path, cid):
        print(f'saving local model to {path}/{cid}_local_model.h5')
        with self.update_lock:
                with self.graph.as_default():
                    with self.session.as_default():
                        self.model.save(f'{path}/{cid}_local_model.h5')

    def save(self, path, cid, train_indices, test_indices):
        """
        Save Global Model wrt. to client
        :param path:
        :param cid:
        :param train_indices:
        :param test_indices:
        :param validation_indices:
        :return:
        """
        save_indices = {"train": train_indices, "test": test_indices}
        print(f'saving indices to {path}/{cid}_indices.npy')
        np.save(f'{path}/{cid}_indices.npy', save_indices)
        self.save_model(path, cid)


# -*- coding: utf-8 -*-
"""Daily check for card and waiver expiry reminders.

The daily check is invoked via the cron API, well, once per day.

"""

# standard library imports
from datetime import datetime

# third-party imports
from flask import current_app

# application imports
from emol.models import CardReminder, WaiverReminder
from emol.utility.date import today

def daily_check():
    """Perform the daily check for card and waiver reminders.

    Check CardReminder and WaiverReminder for any records with a
    date of today or earlier (any earlier records most likely did not get
    processed for whatever reason on their date). Fire off the reminder email
    for each found record, then delete the record.

    """
    current_app.logger.info('Daily check initiated')

    this_day = today()
    reminders = CardReminder.query.filter(CardReminder.reminder_date <= this_day).all()
    for reminder in reminders:
        current_app.logger.debug('Mail {0}'.format(reminder))
        reminder.mail()
        current_app.db.session.delete(reminder)

    reminders = WaiverReminder.query.filter(WaiverReminder.reminder_date <= this_day).all()
    for reminder in reminders:
        current_app.logger.debug('Mail {0}'.format(reminder))
        reminder.mail()
        current_app.db.session.delete(reminder)

    current_app.db.session.commit()

    current_app.logger.info('Daily check complete')