text:
#
# Run this to produce matrices (lat, lon, elev)
#
#
import sys
import numpy as np
import scipy as sp
import scipy.ndimage
import scipy.interpolate
import os
import time
from struct import *
import glob
import png
from scipy.spatial import cKDTree
import multiprocessing
from joblib import Parallel, delayed

# read binary displacement data
def readBinary(fname):
    coords = []
    # data format is lon, lat, elevation
    # we will use lat, lon, elevation to be consistent with DEM
    nbytes = 4  # data is recorded as floats
    with open(fname, "rb") as f:
        byte = f.read(nbytes)
        while len(byte) > 0:
            # binary data is big-endian
            lat = unpack('>f', byte)[0]
            byte = f.read(nbytes)
            lon = unpack('>f', byte)[0]
            byte = f.read(nbytes)
            elev = unpack('>f', byte)[0]
            coords.append((lon, lat, elev))
            byte = f.read(nbytes)
    return coords

def interpolate(fname, dem_lat, dem_lon, output_dir, files):
    slice = readBinary(fname)

    # slice is sim results at each timestep
    # each slice has array of (lat, lon, elevation)
    # we want the simulation elevation at the dem lat, lon coords

    # size of data dimensions
    n_lat = len(dem_lat)
    n_lon = len(dem_lon)

    # record time taken to interpolate
    s = time.time()

    # the slice data is our function of (lat, lon)
    # define the lat and lon vectors and ndarrays
    lat, lon, elev = zip(*slice)
    lat = np.array(lat, dtype=np.float)
    lon = np.array(lon, dtype=np.float)
    elev = np.array(elev, dtype=np.float)

    # search for nearby points using k-nearest neighbour
    k = 9  # number of neighbours
    r = 1  # max distance of neighbourhood
    a = np.column_stack((lat, lon))  # format input array for tree
    tree = scipy.spatial.cKDTree(a.copy())  # init tree with copy of data
    disp = np.zeros((n_lat, n_lon))  # init output displacement matrix

    # find nearest neighbours for each DEM lat lon
    for i in range(n_lat)[:]:
        for j in range(n_lon)[:]:
            # print dem_lat[i], dem_lon[j]
            q = np.array([dem_lat[i], dem_lon[j]])  # query point
            n_d, n_idx = tree.query(q, k=k, eps=0, p=2,
                                    distance_upper_bound=r)  # query returns neighbour distances and indices
            disp[i][j] = 0
            # print d, idx
            wm = 0  # weighted mean
            count = 0
            # determine mean of weigthed contributions from neighbours
            for ni in range(len(n_idx)):
                if (np.isfinite(n_d[ni])):
                    count = count + 1
                    w = (r - n_d[ni]) / r
                    wm += w * elev[n_idx[ni]]  # weighted distace x elevation at neighbour
            if count > 0:
                disp[i][j] = wm / count  # average of weighted contributions

    disp = np.flipud(disp)

    e = time.time()
    print(os.path.basename(fname) + "    %.2f s" % (e - s))
    sys.stdout.flush()

    fname = "%s/disp_%d.csv" % (output_dir, files.index(fname))
    np.savetxt(fname, disp, delimiter=",")

def scale(d, n):
    return d * n

if __name__ == '__main__':
    # input and output files
    in_file = sys.argv[1]
    input_dir = os.path.abspath(sys.argv[2])
    output_dir = os.path.abspath(sys.argv[3])

    print("Reading DEM basemap..."),
    s = time.time()

    # Open and read .in file
    o_lat = []
    o_lon = []
    o_elev = []
    with open(in_file, "r") as file:
        # read lines
        lines = file.readlines()
        # determine number of latitude and longitude points
        n_lat, n_lon = lines[0].split()
        n_lat = int(n_lat)
        n_lon = int(n_lon)
        # read latitude values
        o_lat = [float(x) for x in lines[1].split()]
        # read longitude values
        o_lon = [float(x) for x in lines[2].split()]
        # read lat, long grid
        # Bottom row in image is top row in data
        for i in range(0, n_lat):
            # flat row format
            o_elev = [float(x) for x in lines[3 + i].split()] + o_elev

    e = time.time()
    print("done %.2f s" % (e - s))

    # reshape data
    o_elev = np.array(o_elev)
    elev = o_elev.reshape((n_lat, n_lon))
    # Write out base map:
    np.savetxt("dem.csv", elev, delimiter=",")

    # read displacement data
    files = glob.glob(input_dir + "/*")
    files.sort()

    if (len(files) == 0):
        print("No input files found");
        sys.exit(1)

    # Map displacement data to DEM grid
    print("Interpolating data ...")
    sys.stdout.flush()

    num_cores = multiprocessing.cpu_count()
    Parallel(n_jobs=num_cores)(delayed(interpolate)(f, o_lat, o_lon, output_dir, files) for f in files)

ext: py

max_stars_repo_name: UoA-eResearch/earthquake-viz

max_issues_repo_name: UoA-eResearch/earthquake-viz

max_forks_repo_name: UoA-eResearch/earthquake-viz

reason: import numpy,import scipy,from scipy


text:
from fenics import *
import fred.conf as conf

if conf.with_adjoint:
    from fenics_adjoint import *
import numpy as np
import zmq
from io import StringIO
import logging

logger = logging.getLogger("cm.controller")

class Controller:

    def __init__(self, wind_farm):
        self._wind_farm = wind_farm
        self._turbines = wind_farm.get_turbines()

        self._controls = {}
        for control in conf.par.wind_farm.controller.controls:
            self._controls[control] = Control(name=control,
                                              control_type=conf.par.wind_farm.controller.controls[control]["type"],
                                              value=np.array(
                                                  conf.par.wind_farm.controller.controls[control].get("values", None)))
            logger.info("Setting up {:s} controller of type: {}"
                        .format(control, conf.par.wind_farm.controller.controls[control]["type"]))

        self._received_controls = {}
        if conf.par.wind_farm.controller.with_external_controller:
            self._received_data = []
            self._external_controls = conf.par.wind_farm.controller.external_controls
            for control in self._external_controls:
                self._received_controls[control] = np.NaN
            logger.info("Initialising ZMQ communication")
            self._context = zmq.Context()
            self._socket = self._context.socket(zmq.REQ)
            address = "tcp://localhost:{}".format(conf.par.wind_farm.controller.port)
            self._socket.connect(address)
            logger.info("Connected to: {}".format(address))

    def control(self, simulation_time):
        if conf.par.wind_farm.controller.with_external_controller:
            self.send_measurements(simulation_time)
            self.receive_controls()
        for control in self._controls.values():
            control.do_control(simulation_time, self._received_controls)
            [wt.set_control(control.get_name(), c) for (wt, c) in zip(self._turbines, control.get_reference())]

    def send_measurements(self, simulation_time):
        # todo: measurements
        measurements = np.linspace(0, 16, 17)
        measurements[0] = simulation_time
        measurements[1::8] = [wt.get_power() for wt in self._turbines]
        measurement_string = " ".join(["{:.6f}".format(x) for x in measurements]).encode()
        logger.warning("Real measurements not implemented yet")
        logger.info("Sending: {}".format(measurement_string))
        self._socket.send(measurement_string)

    def receive_controls(self):
        message = self._socket.recv()
        # raw message contains a long useless tail with b'\x00' characters  (at least if from sowfa)
        # split off the tail before decoding into a Python unicode string
        json_data = message.split(b'\x00', 1)[0].decode()
        self._received_data = np.loadtxt(StringIO(json_data), delimiter=' ')
        logger.info("Received controls: {}".format(json_data))

        for idx in range(len(self._received_controls)):
            self._received_controls[self._external_controls[idx]] = self._received_data[
                                                                    idx::len(self._external_controls)]

    def _external_induction_controller(self, simulation_time):
        logger.warning("External induction controller only works if yaw controller implemented as well")
        new_ref = self._received_data[1::2]
        return new_ref

    def _external_pitch_controller(self, simulation_time):
        logger.warning("External pitch_torque controller only works if yaw controller implemented as well")
        new_ref = self._received_data[2::3]
        return new_ref

    def _external_torque_controller(self, simulation_time):
        new_ref = self._received_data[0::3]
        return new_ref

    def get_controls_list(self, name):
        return self._controls[name].get_control_list()

    def set_control_reference_series(self, name, time_series, reference_series):
        self._controls[name].set_reference_series(time_series, reference_series)

    def clear_controls(self):
        for control in self._controls.values():
            control.clear_control_list()

class Control:
    def __init__(self, name, control_type, value):
        self._name = name
        switcher = {
            "fixed": self._fixed_control,
            "series": self._series_control,
            "external": self._external_control
        }
        self._control_function = switcher.get(control_type, "fixed")
        self._reference = []
        self._time_last_updated = 0
        self._time_series = None
        self._reference_series = None

        if control_type == "series":
            self._time_series = value[:, 0]
            self._reference_series = value[:, 1:]
            if name == "yaw":
                self._reference_series = np.deg2rad(self._reference_series)

    def get_name(self):
        return self._name

    def get_control_list(self):
        return self._reference

    def clear_control_list(self):
        self._reference = []

    def get_reference(self):
        return self._reference[-1]

    def do_control(self, simulation_time, received_controls):
        if (simulation_time - self._time_last_updated >= conf.par.wind_farm.controller.control_discretisation) \
                or self._reference == []:
            # execute the control function from init.
            new_reference = self._control_function(simulation_time, received_controls)
            # todo: rate limit
            new_reference = self._apply_rate_limit(new_reference)
            self._update_reference(new_reference)
            self._time_last_updated = simulation_time
            logger.info("Setting {} to {}".format(self._name, new_reference))

    def _fixed_control(self, simulation_time, received_controls):
        new_reference = conf.par.wind_farm.yaw_angles.copy()
        return new_reference

    def _series_control(self, simulation_time, received_controls):
        new_reference = conf.par.wind_farm.yaw_angles.copy()
        for idx in range(len(conf.par.wind_farm.positions)):
            new_reference[idx] = np.interp(simulation_time, self._time_series, self._reference_series[:, idx])
        return new_reference

    def _external_control(self, simulation_time, received_controls):
        new_reference = received_controls[self._name]
        if self._name == "yaw":
            new_reference = np.deg2rad(new_reference)
        return new_reference

    def _update_reference(self, new_reference):
        if len(new_reference) != len(conf.par.wind_farm.positions):
            raise ValueError(
                "Computed yaw reference (length {:d}) does not match {:d} turbines in farm."
                    .format(len(new_reference), len(conf.par.wind_farm.positions)))
        self._reference.append([Constant(y) for y in new_reference])

    def set_reference_series(self, time_series, reference_series):
        self._time_series = time_series
        self._reference_series = reference_series
        if self._name == "yaw":
            self._reference_series = np.deg2rad(self._reference_series)

    def _apply_rate_limit(self, new_reference):
        logger.warning("Rate limit in wind farm controller not implemented")
        return new_reference
        # if len(self._yaw_ref)>0:
        #     yaw_rate_limit = conf.par.turbine.yaw_rate_limit
        #     time_step = conf.par.simulation.time_step
        #     new_ref = np.array(new_ref)
        #     prev_ref = np.array([float(y) for y in self._yaw_ref[-1]])
        #     delta_ref = new_ref - prev_ref
        #     if yaw_rate_limit > 0:
        #         delta_ref = np.min((yaw_rate_limit * time_step * np.ones_like(new_ref), delta_ref),0)
        #         delta_ref = np.max((-yaw_rate_limit * time_step * np.ones_like(new_ref), delta_ref),0)
        #     new_ref = prev_ref + delta_ref
        # return new_ref

max_stars_repo_name: TUDelft-DataDrivenControl/FRED

max_issues_repo_name: TUDelft-DataDrivenControl/FRED

max_forks_repo_name: TUDelft-DataDrivenControl/FRED

reason: import numpy


text:
# author: zeyang

import os
import time
import argparse
import numpy as np
import socket
from reprint import output
from message import send_mail

def parse_args():
    """Parse input arguments"""
    parser = argparse.ArgumentParser(description='GPU resource monitoring.')
    parser.add_argument(
        '-n-gpus', '--num-gpus', type=int, default=8,
        help='Number of GPUs you need.')
    parser.add_argument(
        '-m', '--memory', type=int, required=True,
        help='Estimated GPU memory you need (MB per GPU).')
    parser.add_argument(
        '-inv', '--interval', type=int, default=2,
        help='Interval time (s) between queries of GPU status.')
    parser.add_argument(
        '--dir', type=str, required=True,
        help='Specify working directory for the command.')
    parser.add_argument(
        '--msg', action='store_true',
        help='Whether to send email message to report the status of task.')
    parser.add_argument('--cmd', default = 'python /path/to/script.py',
        help='the cmd to run')
    return parser.parse_args()

def gpu_info(args):
    memory_total = list(map(int, os.popen('nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits')
                            .read().strip().split('\n')))
    memory_used = list(map(int, os.popen('nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits')
                           .read().strip().split('\n')))
    memory_free = list(map(int, os.popen('nvidia-smi --query-gpu=memory.free --format=csv,noheader,nounits')
                           .read().strip().split('\n')))
    power = list(map(float, os.popen('nvidia-smi --query-gpu=power.draw --format=csv,noheader,nounits')
                     .read().strip().split('\n')))
    power_limit = list(map(float, os.popen('nvidia-smi --query-gpu=power.limit --format=csv,noheader,nounits')
                           .read().strip().split('\n')))
    num_gpus = len(memory_total)
    assert num_gpus >= args.num_gpus, f'{num_gpus} GPU(s) detected, however {args.num_gpus} GPU(s) needed.'

    # number of GPUs that have memory in design larger than the required memory
    n_sat = np.sum(np.array(memory_total) >= args.memory)
    assert n_sat >= args.num_gpus, \
        f'Require {args.num_gpus} GPU(s) that have memory in design larger than {args.memory} MB, ' \
            f'however only {n_sat} GPU(s) satisfied.'

    # number of GPUs that have free memory larger than the required memory
    id = np.nonzero(np.array(memory_free) >= args.memory)[0]  # the id list of GPU(s) that are ready for the task.
    ready = id.size >= args.num_gpus

    return ready, id, list(zip(range(num_gpus), power, power_limit, memory_used, memory_total))

def main(args):
    cmd = args.cmd
    assert os.path.exists(args.dir), f'The specified working directory {args.dir} does not exist.'
    assert cmd != 'python /path/to/script.py', 'Please specify the command to run in gpu_monitor.py'
    ready, id, info = gpu_info(args)
    gpu_info_format = '|{:^7d}|   {:>3.0f}W / {:>3.0f}W   | {:>5d}MiB / {:>5d}MiB |\r\n'
    i = 0
    with output(output_type='list', initial_len=len(info) + 2, interval=0) as gpu_info_print:
        while not ready:  # set waiting condition
            ready, id, info = gpu_info(args)
            i = i % 5
            gpu_info_print[0] = '|  GPU  |  Pwr:Usage/Cap  |     Memory-Usage    |\r\n'
            for j, info_j in enumerate(info, start=1):
                gpu_info_print[j] = gpu_info_format.format(*info_j)
            gpu_info_print[-1] = 'monitoring' + '.' * (i + 1)
            time.sleep(args.interval)
            i += 1
    start = time.time()
    send_mail(cmd)  # send notification via email to the user
    os.environ["CUDA_VISIBLE_DEVICES"] = ','.join(list(map(str, id[:args.num_gpus])))
    os.chdir(os.path.expanduser(args.dir))  # change working directory
    print('\n' + cmd)
    try:
        if os.system(cmd) != 0:
            raise Exception('Error encountered.')
    except Exception:
        sub = 'Error Encountered'
        content = 'Hi,\n\nWe regret to inform you that your program encountered some errors ' \
                  'during running, please check with it, Thanks.' \
                  '\n\nThis is a system generated email, do not reply.\n\n' \
                  'Nice day,\n' + socket.gethostname()
        send_mail(cmd, subject=sub, content=content)
    else:
        send_mail(cmd, finish=True, time_used=time.time() - start)

if __name__ == '__main__':
    args = parse_args()
    main(args)

max_stars_repo_name: ascetic-monk/GPU-Monitor

max_issues_repo_name: ascetic-monk/GPU-Monitor

max_forks_repo_name: ascetic-monk/GPU-Monitor


text:
"""
Helper functions for working with the NGA flatfile.
"""

# stdlib imports
import os
import pkg_resources
import logging

# third party imports
import numpy as np
import pandas as pd
from obspy.geodetics.base import gps2dist_azimuth

def get_nga_record_sequence_no(st, eq_name, distance_tolerance=50):
    """
    Returns the associate NGA record sequence number for a given StationStream.

    Args:
        st (gmprocess.stationstream.StationStream):
            Station stream to get record sequence number for.
        eq_name (str):
            Earthquake name for finding NGA record sequence numbers. Must
            match a value in the 'Earthquake Name' column of the file
            gmprocess/data/nga_w2_selected.csv.
        distance_tolerance (float):
            Distance tolerance (in meters) between StationStream location
            coordinates and the NGA location coordinates.
            Default is 50 meters.

    Returns:
        int: Matching record sequence number from NGA flatfile. Returns
            numpy.nan if record sequence number is not found.

    """

    df_nga = pd.read_csv(pkg_resources.resource_filename(
        'gmprocess', os.path.join('data', 'nga_w2_selected.csv')))

    nga_event = df_nga.loc[df_nga['Earthquake Name'] == eq_name]

    lat = st[0].stats.coordinates.latitude
    lon = st[0].stats.coordinates.longitude

    matched_records_nos = []
    for record_idx, record in nga_event.iterrows():
        dist = gps2dist_azimuth(
            lat, lon, record['Station Latitude'],
            record['Station Longitude'])[0]
        if dist < distance_tolerance:
            matched_records_nos.append(record['Record Sequence Number'])

    if len(matched_records_nos) > 1:
        logging.warning('Found multiple matching records.')
        return np.nan
    elif len(matched_records_nos) < 1:
        logging.warning('Did not find any matching records.')
        return np.nan
    else:
        return matched_records_nos[0]

max_stars_repo_name: norfordb/groundmotion

max_issues_repo_name: vinceq-usgs/gmprocess

max_forks_repo_name: vinceq-usgs/gmprocess


text:
# Classify up ramp or down ramp event based on the difference
# between two points (ramp magnitude) at the start and end of a
# predefined duration.
#
# Joseph Lee <joseph.lee at pnnl.gov>

import numpy as np
import pandas as pd

class r_magnitude:

    def __init__(self, conf, c, ramp_data, ramps):

        self.base_var = conf['base']['target_var']
        self.comp_var = c['target_var']
        self.ramps = ramps
        self.ramp_data = ramp_data
        self.reference = conf['reference']

        if self.ramps['magnitude'] > 0:
            self.ramp_nature = 'up'
        elif self.ramps['magnitude'] < 0:
            self.ramp_nature = 'down'

    def get_rampdf(self):
        """Generate data frame with ramp classification."""

        print()
        print('classfy as '+self.ramp_nature+' ramp event when '
              + 'the change of '+self.reference['var']
              + ' exceeds '+str(self.ramps['magnitude'])
              + ' '+self.reference['units']+' in '
              + self.ramps['duration']
              )

        ramp_data_dn = self.ramp_data.copy()
        ramp_data_dn.index = ramp_data_dn.index - pd.to_timedelta(
            str(self.ramps['duration']))

        # Get data frame of lagged differences
        # Drop NA means dropping data points on both starting and ending
        # points of a ramp period
        ramp_df = (ramp_data_dn - self.ramp_data).dropna()

        zeros_col = np.zeros(len(ramp_df))
        ramp_df['base_ramp'] = zeros_col
        ramp_df['comp_ramp'] = zeros_col

        if self.ramps['magnitude'] > 0:

            ramp_df.loc[ramp_df[self.base_var]
                        > self.ramps['magnitude'], ['base_ramp']] = 1
            ramp_df.loc[ramp_df[self.comp_var]
                        > self.ramps['magnitude'], ['comp_ramp']] = 1

        elif self.ramps['magnitude'] < 0:

            ramp_df.loc[ramp_df[self.base_var]
                        < self.ramps['magnitude'], ['base_ramp']] = 1
            ramp_df.loc[ramp_df[self.comp_var]
                        < self.ramps['magnitude'], ['comp_ramp']] = 1

        return ramp_df

    def get_ramp_method_name(self):
        """Get ramp method name as column name in output file."""

        return self.ramps['definition']+'_'+str(self.ramps['magnitude'])\
            + self.reference['units']+'_'+self.ramps['duration']


text:
'''
Verify the conformance of the model before and after fuse_bn operation.
'''

import torch
import os
import sys

sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
from PIL import Image
import numpy as np
from converter.merge_bn_seq import fuse_bn_recursively
from hourglass_network import lane_detection_network
import pdb

if __name__ == '__main__':

    save_dir = './fuse_bn_test'
    model_pth_name = '../savefile/16_tensor(0.7522)_lane_detection_network'
    img = torch.ones(1, 3, 256, 256, device='cpu') #(B,C,H,W)

    if not os.path.exists(save_dir):
        os.makedirs(save_dir)

    net = lane_detection_network()

    pth_path = os.path.join('{}.pkl'.format(model_pth_name))
    state_dict = torch.load(pth_path, map_location='cpu')

    # print(state_dict.keys())
    net.load_state_dict(state_dict)
    # Benchmarking
    # First, we run the network the way it is
    net.eval()
    with torch.no_grad():
        result, _ = net(img)
        confidences, offsets, instances = result[-1]

    net = fuse_bn_recursively(net)
    net.eval()
    with torch.no_grad():
        result, _ = net(img)
        confidences_fuse, offsets_fuse, instances_fuse = result[-1]

    # check confidence
    confidences_np = confidences.numpy()
    confidences_fuse_np = confidences_fuse.numpy()
    b, c, h, w = confidences_np.shape
    total_diff = 0
    total_AxB = 0
    total_AxA= 1e-16
    total_BxB= 1e-16
    for i in range(b):
        for j in range(c):
            for k in range(h):
                for m in range(w):
                    base_val = confidences_np[i,j,k,m]
                    fuse_val= confidences_fuse_np[i,j,k,m]

                    total_AxB += base_val * fuse_val
                    total_AxA += base_val * base_val
                    total_BxB += fuse_val * fuse_val

                    # write val into txt.
                    diff_file = open('{}/conf.txt'.format(save_dir),'a+')
                    diff = abs(fuse_val - base_val)
                    total_diff += diff
                    diff_file.write('not merge:{}\tmerged:{}\tdiff:{}\n'.format(base_val, fuse_val, diff))
                    diff_file.close()
    print("confidence_total_diff:",total_diff)
    # Cosine Similarity
    cosine_sim = total_AxB /(pow(total_AxA, 1/2)*pow(total_BxB, 1/2))
    print("cosine similarity:",cosine_sim)
    # for d in range(len(confidences.shape)):
    #     pytorch_similarity = torch.cosine_similarity(confidences, confidences_fuse, dim=d)
    #     print("dim {} cosine similarity:{}".format(d, pytorch_similarity))

max_stars_repo_name: pandamax/Lane-Detection-Based-PINet

max_issues_repo_name: pandamax/Lane-Detection-Based-PINet

max_forks_repo_name: pandamax/Lane-Detection-Based-PINet


text:
import numpy as np
import os
import pytest
from skimage.data import coffee as get_coffee
from skimage.io import imsave
from tempfile import mkstemp

from nolearn_utils.iterators import BaseBatchIterator, make_iterator

@pytest.fixture
def X():
    return np.random.rand(1000, 3, 48, 48)

@pytest.fixture
def y():
    return np.random.rand(1000)

@pytest.fixture
def coffee_tmp_paths(n=100):
    paths = []
    for i in xrange(10):
        _, fname = mkstemp(suffix='.png')
        imsave(fname, get_coffee())
        paths.append(fname)
    return np.asarray(paths)

def test_base_batch_iterator(X, y):
    iterator = BaseBatchIterator(batch_size=128)
    batches = list(iterator(X, y))
    assert len(batches) == 8
    for Xb, yb in batches[:-1]:
        assert Xb.shape[0] == 128
        assert yb.shape[0] == 128
    assert batches[-1][0].shape[0] == 104
    assert batches[-1][1].shape[0] == 104

def test_shuffle_batch_iterator(X, y):
    from nolearn_utils.iterators import ShuffleBatchIteratorMixin

    class Iterator(ShuffleBatchIteratorMixin, BaseBatchIterator):
        pass

    iterator = Iterator(batch_size=128)
    np.random.seed(42)  # Deterministic tests
    Xb, yb = iter(iterator(X, y)).next()
    assert np.all(Xb != X[:128])
    assert np.all(yb != y[:128])

def test_afffine_batch_iterator(X, y):
    # TODO simple smoke test
    from nolearn_utils.iterators import AffineTransformBatchIteratorMixin

    class Iterator(AffineTransformBatchIteratorMixin, BaseBatchIterator):
        pass

    iterator = Iterator(batch_size=128, affine_p=0.5,
                        affine_scale_choices=np.linspace(0.9, 1.1, 5),
                        affine_translation_choices=np.arange(-5, 6, 10),
                        affine_rotation_choices=np.arange(0, 360))

    for Xb, yb in iterator(X, y):
        pass

def test_random_crop_batch_iterator(X, y):
    # TODO simple smoke test
    from nolearn_utils.iterators import RandomCropBatchIteratorMixin

    class Iterator(RandomCropBatchIteratorMixin, BaseBatchIterator):
        pass

    iterator = Iterator(batch_size=128, crop_size=(20, 20))

    for Xb, yb in iterator(X, y):
        pass

def test_random_flip_batch_iterator(X, y):
    from nolearn_utils.iterators import RandomFlipBatchIteratorMixin
    Iterator = make_iterator('Iterator', [RandomFlipBatchIteratorMixin])
    iterator = Iterator(
        batch_size=128, flip_horizontal_p=0.5, flip_vertical_p=0.5
    )

    for Xb, yb in iterator(X, y):
        pass

def test_read_image_batch_iterator(coffee_tmp_paths, y):
    # TODO simple smoke test
    from nolearn_utils.iterators import ReadImageBatchIteratorMixin

    class Iterator(ReadImageBatchIteratorMixin, BaseBatchIterator):
        pass

    iterator = Iterator(batch_size=128, read_image_size=(30, 30))

    for Xb, yb in iterator(coffee_tmp_paths, y):
        pass

    [os.remove(path) for path in coffee_tmp_paths]

def test_mean_subtraction_batch_iterator(X, y):
    from nolearn_utils.iterators import MeanSubtractBatchiteratorMixin

    class Iterator(MeanSubtractBatchiteratorMixin, BaseBatchIterator):
        pass

    iterator = Iterator(batch_size=2, mean=np.zeros((3, 48, 48)))

    for Xb, yb in iterator(X, y):
        pass


text:
import numpy as np
import pressiodemoapps as pda
import pathlib, sys
file_path = pathlib.Path(__file__).parent.absolute()

def test_meshclass1():
  meshPath = str(file_path)
  meshO = pda.load_cellcentered_uniform_mesh(meshPath)

  assert( meshO.dimensionality() == 2 )
  assert( meshO.stencilMeshSize() == 25 )
  assert( meshO.sampleMeshSize() == 25 )
  assert( meshO.stencilSize() == 3 )
  assert( meshO.graph().shape[0] == 25 )
  assert( meshO.graph().shape[1] == 5 )

  assert( meshO.dx() == 0.2 )
  assert( meshO.dxInv() == 5. )
  assert( meshO.dy() == 0.2 )
  assert( meshO.dyInv() == 5. )

  goldX = [0.1, 0.3, 0.5, 0.7, 0.9, 0.1, 0.3, 0.5,\
           0.7, 0.9, 0.1, 0.3, 0.5, 0.7, 0.9, 0.1,\
           0.3, 0.5, 0.7, 0.9, 0.1, 0.3, 0.5, 0.7, 0.9]
  goldY = [0.1, 0.1, 0.1, 0.1, 0.1, 0.3, 0.3, 0.3, \
           0.3, 0.3, 0.5, 0.5, 0.5, 0.5, 0.5, 0.7, \
           0.7, 0.7, 0.7, 0.7, 0.9, 0.9, 0.9, 0.9, 0.9]
  assert( np.allclose(meshO.viewX(), goldX) )
  assert( np.allclose(meshO.viewY(), goldY) )

def test_meshclass2():
  meshPath = str(file_path)
  meshO = pda.CellCenteredUniformMesh(meshPath)
  G = meshO.graph()
  print(G)

  goldG = [
    [0,        4,        5,        1,       20],
    [1,        0,        6,        2,       21],
    [2,        1,        7,        3,       22],
    [3,        2,        8,        4,       23],
    [4,        3,        9,        0,       24],
    [5,        9,       10,        6,        0],
    [6,        5,       11,        7,        1],
    [7,        6,       12,        8,        2],
    [8,        7,       13,        9,        3],
    [9,        8,       14,        5,        4],
    [10,       14,       15,       11,        5],
    [11,       10,       16,       12,        6],
    [12,       11,       17,       13,        7],
    [13,       12,       18,       14,        8],
    [14,       13,       19,       10,        9],
    [15,       19,       20,       16,       10],
    [16,       15,       21,       17,       11],
    [17,       16,       22,       18,       12],
    [18,       17,       23,       19,       13],
    [19,       18,       24,       15,       14],
    [20,       24,        0,       21,       15],
    [21,       20,        1,       22,       16],
    [22,       21,        2,       23,       17],
    [23,       22,        3,       24,       18],
    [24,       23,        4,       20,       19]
  ]
  assert( np.allclose(G, goldG) )

if __name__ == '__main__':
  test_meshclass1()
  test_meshclass2()

max_stars_repo_name: fnrizzi/pressio-demoapps

max_issues_repo_name: fnrizzi/pressio-demoapps

max_forks_repo_name: fnrizzi/pressio-demoapps


text:
# Copyright 2021, Google LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for secret_sharer."""

import collections
import string

import numpy as np
import tensorflow as tf
import tensorflow_federated as tff

from differential_privacy import secret_sharer

TEST_SEED = 0xBAD5EED

def _construct_test_client_data(num_clients, num_examples, secrets, seed):
  np.random.seed(TEST_SEED)

  def random_string():
    return ''.join(np.random.choice(list(string.ascii_uppercase), 5))

  words = ['a' for _ in range(num_examples)]

  def data_for_client():
    return collections.OrderedDict(
        # We hash on date to seed client randomness, so they must be distinct.
        creation_date=[random_string() for _ in range(num_examples)],
        score=words,
        tags=words,
        title=words,
        tokens=words,
        type=words)

  client_data = tff.simulation.datasets.TestClientData(
      {random_string(): data_for_client() for _ in range(num_clients)})

  return secret_sharer.stackoverflow_with_secrets(client_data, secrets, seed)

class SecretSharerTest(tf.test.TestCase):

  def test_secret_insertion(self):
    if tf.config.list_logical_devices('GPU'):
      self.skipTest('skip GPU test')

    secrets = dict(b=(50, 0.6), c=(20, 1.0), d=(1, 1.0))
    num_clients = 100
    num_examples = 12
    client_data = _construct_test_client_data(num_clients, num_examples,
                                              secrets, TEST_SEED)

    secret_count = {secret: 0 for secret in secrets}
    client_with_secret_count = {secret: 0 for secret in secrets}

    for client_id in client_data.client_ids:
      dataset = client_data.dataset_computation(client_id)
      has_secret = None
      for example in dataset.enumerate():
        tokens = example[1]['tokens'].numpy().decode('utf-8')
        if tokens in secrets:
          secret_count[tokens] += 1
          has_secret = tokens
      if has_secret:
        client_with_secret_count[has_secret] += 1

    # The probability that there exists even a single client b that has no
    # insertion is an acceptable 1-(1-0.4^12)^50 < 1e-3. So we assert equal.
    self.assertAllEqual(50, client_with_secret_count['b'])
    self.assertAllEqual(20, client_with_secret_count['c'])
    self.assertAllEqual(1, client_with_secret_count['d'])

    # Count of secret b is Bin(600, 0.6) with mean 360, stddev 12.
    # Assert result within 40: over three stddevs of mean.
    self.assertAllClose(360, secret_count['b'], atol=40)
    self.assertAllEqual(20 * num_examples, secret_count['c'])
    self.assertAllEqual(num_examples, secret_count['d'])

  def test_dataset_computation_equals_create_tf_dataset(self):
    if tf.config.list_logical_devices('GPU'):
      self.skipTest('skip GPU test')

    secrets = dict(b=(5, 0.6), c=(4, 1.0), d=(3, 0.2), e=(2, 0.8), f=(1, 1.0))
    num_clients = 50
    num_examples = 12
    client_data = _construct_test_client_data(num_clients, num_examples,
                                              secrets, TEST_SEED)

    for client_id in client_data.client_ids:
      dataset_with_comp = client_data.dataset_computation(client_id)
      dataset_no_comp = client_data.create_tf_dataset_for_client(client_id)
      for x, y in zip(dataset_with_comp.enumerate(),
                      dataset_no_comp.enumerate()):
        self.assertAllEqual(x[1]['tokens'], y[1]['tokens'])

  def test_seed_used(self):
    # We will construct datasets with two seeds and test that a) at least one
    # client has different secrets in the two datasets and b) for some client
    # that has the same secret in both datasets, the chosen examples are
    # different. Note that the number of clients and examples have been chosen
    # so the probability of either of these things failing to hold is < 1e-3.

    if tf.config.list_logical_devices('GPU'):
      self.skipTest('skip GPU test')

    secrets = dict(b=(9, 0.5), c=(1, 1.0))
    num_clients = 20
    num_examples = 10
    client_data_1 = _construct_test_client_data(num_clients, num_examples,
                                                secrets, 1)
    client_data_2 = _construct_test_client_data(num_clients, num_examples,
                                                secrets, 2)

    self.assertCountEqual(client_data_1.client_ids, client_data_2.client_ids)

    def get_secret(dataset):
      for example in dataset.enumerate():
        tokens = example[1]['tokens'].numpy().decode('utf-8')
        if tokens in secrets:
          return tokens
      return None

    chosen_clients_differ = False
    chosen_examples_differ = False
    for client_id in client_data_1.client_ids:
      dataset_1 = client_data_1.dataset_computation(client_id)
      dataset_2 = client_data_2.dataset_computation(client_id)

      secret_1 = get_secret(dataset_1)
      secret_2 = get_secret(dataset_2)

      if secret_1 != secret_2:
        chosen_clients_differ = True
      elif secret_1 is not None:
        # This client has the same secret in both datasets.
        # Test that the selected examples are different.
        for x, y in zip(dataset_1.enumerate(), dataset_2.enumerate()):
          if not all(
              s == t
              for s, t in zip(x[1]['tokens'].numpy(), y[1]['tokens'].numpy())):
            chosen_examples_differ = True

    self.assertTrue(chosen_clients_differ)
    self.assertTrue(chosen_examples_differ)

if __name__ == '__main__':
  tf.test.main()

max_stars_repo_name: garyxcheng/federated

max_issues_repo_name: garyxcheng/federated

max_forks_repo_name: garyxcheng/federated


text:
import os, json
import numpy as np
import random as rn
import tensorflow as tf
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix, roc_auc_score
from keras.callbacks import Callback
from keras.models import Sequential, load_model
from keras.layers.convolutional_recurrent import ConvLSTM2D
from keras.layers.normalization import BatchNormalization
from keras.optimizers import Adam
from keras.layers.core import Dense, Activation, Dropout, Flatten
from keras.callbacks import TensorBoard, ModelCheckpoint
from datetime import datetime
from imblearn.over_sampling import SMOTE
from imblearn.under_sampling import (
    RandomUnderSampler,
    ClusterCentroids,
    NearMiss,
    TomekLinks,
    CondensedNearestNeighbour,
    EditedNearestNeighbours,
    AllKNN,
    NeighbourhoodCleaningRule,
)
from imblearn.keras import BalancedBatchGenerator

date_format = '%Y%m%d'

def train(
    X_train,
    y_train,
    X_test,
    y_test,
    info_train=None,
    info_test=None,
    out_dir=None,
    model_file_name='best_model.h5',
    learning_rate=5e-6,
    decay=0.0,
    epochs=100,
    dropout=0.3,
    log_dir=None,
    resampling_methods=None,
    balanced_batch=None,
    use_class_weight=False,
    random_state=4126):
    if out_dir is not None:
        os.makedirs(out_dir, exist_ok=True)

    model = Sequential()

    model.add(ConvLSTM2D(
        input_shape=(X_train.shape[1], X_train.shape[2], X_train.shape[3], X_train.shape[4]),
        filters=30,
        kernel_size=(3, 3),
        padding='same',
        dropout=dropout,
        return_sequences=False
    ))
    model.add(BatchNormalization())

    model.add(Flatten())

    model.add(Dense(100, activation='relu'))
    model.add(Dropout(dropout))

    model.add(Dense(1, activation='sigmoid'))

    adam = Adam(lr=learning_rate, decay=decay)
    model.compile(optimizer=adam, loss='binary_crossentropy', metrics=["accuracy"])
    model.summary()

    reporter = _Reporter(X_test, y_test)
    callbacks = [reporter]

    model_path = None
    if out_dir is not None:
        model_path = os.path.join(out_dir, model_file_name)
        callbacks.append(
            ModelCheckpoint(filepath=model_path, verbose=1, save_best_only=True)
        )

    if log_dir is not None:
        log_path = os.path.join(log_dir, datetime.now().strftime(date_format + '_%H%M%S'))
        callbacks.append(TensorBoard(log_dir=log_path))

    positive = (0.5 <= y_train).sum()
    negative = (y_train < 0.5).sum()
    print('train data balance P:{} : N:{}'.format(positive, negative))
    class_weight = {
        0: positive / (positive + negative),
        1: negative / (positive + negative),
    } if use_class_weight else None

    if resampling_methods is not None:
        for resampling_method in resampling_methods:
            X_train, y_train = _resample(X_train, y_train, resampling_method, random_state)

    if balanced_batch == True:
        print('using BalancedBatchGenerator')
        batch_gen = BalancedBatchGenerator(\
            X_train, y_train, \
            sampler=RandomUnderSamplerWrapper(), random_state=random_state)
        model.fit_generator(generator=batch_gen, \
            epochs=epochs, callbacks=callbacks, \
            validation_data=(X_test, y_test),
            class_weight=class_weight)
    else:
        model.fit(X_train, y_train, \
            epochs=epochs, callbacks=callbacks, \
            validation_data=(X_test, y_test),
            class_weight=class_weight)

    if out_dir is not None and info_train is not None and info_test is not None:
        _output(out_dir, X_test, y_test, info_train, info_test, model_path, reporter)

def _output(out_dir, X_test, y_test, info_train, info_test, model_path, reporter):
    best_model = load_model(model_path)
    acc, auc, f1, precision, recall, tp, fn, fp, tn = _eval(best_model, X_test, y_test)

    # summary
    summary = {
        'acc': acc,
        'auc': auc,
        'f1': f1,
        'precision': precision,
        'recall': recall,
        'tp': int(tp),
        'fn': int(fn),
        'fp': int(fp),
        'tn': int(tn),
        'train_acc': reporter.acc,
        'train_val_acc': reporter.val_acc,
        'loss': reporter.loss,
        'val_loss': reporter.val_loss,
        'train_start_date': info_train[0]['window_start'].strftime(date_format),
        'train_end_date': info_train[-1]['predict_end'].strftime(date_format),
        'test_start_date': info_test[0]['window_start'].strftime(date_format),
        'test_end_date': info_test[-1]['predict_end'].strftime(date_format),
    }
    _dump(summary, os.path.join(out_dir, 'summary.json'))

    # validations
    validations = []
    y_pred = best_model.predict(X_test).reshape(-1)
    for i in range(0, len(y_test)):
        predict_center_lat = float(info_test[i]['predict_center_lat'])
        predict_center_lng = float(info_test[i]['predict_center_lng'])
        window_start = info_test[i]['window_start'].strftime(date_format)
        window_end = info_test[i]['window_end'].strftime(date_format)
        predict_start = info_test[i]['predict_start'].strftime(date_format)
        predict_end = info_test[i]['predict_end'].strftime(date_format)
        detail_path = 'detail_{}_{}.json'.format(predict_start, predict_end)
        validation = {
            'window_start': window_start,
            'window_end': window_end,
            'predict_start': predict_start,
            'predict_end': predict_end,
            'prediction': float(y_pred[i]),
            'fact': float(y_test[i]),
            'detail': detail_path
        }
        validations.append(validation)

        # detail
        detail = {
            'predict_center_lat': predict_center_lat,
            'predict_center_lng': predict_center_lng,
            'window_start': window_start,
            'window_end': window_end,
            'predict_start': predict_start,
            'predict_end': predict_end,
            'mag_heatmaps': [],
            'freq_heatmaps': [],
            'depth_heatmaps': [],
            'lat_gap': 180 / X_test[i].shape[1],
            'lng_gap': 360 / X_test[i].shape[2],
            'threshold_mag': info_test[i]['threshold_mag'],
            'earthquakes': []
        }
        for win in range(X_test[i].shape[0]):
            mag_heatmap = []
            freq_heatmap = []
            depth_heatmap = []
            for lat in range(X_test[i].shape[1]):
                for lng in range(X_test[i].shape[2]):
                    mag = X_test[i][win][lat][lng][0]
                    freq = X_test[i][win][lat][lng][1]
                    depth = X_test[i][win][lat][lng][2]
                    if 0 < mag:
                        mag_heatmap.append({
                            'lat': lat,
                            'lng': lng,
                            'heat': mag
                        })
                    if 0 < freq:
                        freq_heatmap.append({
                            'lat': lat,
                            'lng': lng,
                            'heat': freq
                        })
                    if 0 < depth:
                        depth_heatmap.append({
                            'lat': lat,
                            'lng': lng,
                            'heat': depth
                        })
            detail['mag_heatmaps'].append(mag_heatmap)
            detail['freq_heatmaps'].append(freq_heatmap)
            detail['depth_heatmaps'].append(depth_heatmap)
        max_mag = 0
        for eq in info_test[i]['earthquakes']:
            detail['earthquakes'].append({
                'time': eq['time'].strftime(date_format + '%H%M%S'),
                'latitude': eq['latitude'],
                'longitude': eq['longitude'],
                'depth': eq['depth'],
                'mag': eq['mag']
            })
            if max_mag < eq['mag']:
                max_mag = eq['mag']
        validation['max_mag'] = max_mag
        _dump(detail, os.path.join(out_dir, detail_path))
    _dump({ 'validations': validations }, os.path.join(out_dir, 'validations.json'))

def _resample(X_train, y_train, resampling_method, random_state):
    resampler = _get_resampler(resampling_method, random_state)
    print('performing {}...'.format(resampling_method['name']))
    X_train_resample, y_train = resampler.fit_sample(X_train.reshape(X_train.shape[0], -1), y_train)
    X_train = X_train_resample.reshape(\
        X_train_resample.shape[0], \
        X_train.shape[1], \
        X_train.shape[2], \
        X_train.shape[3], \
        X_train.shape[4])
    positive = (0.5 <= y_train).sum()
    negative = (y_train < 0.5).sum()
    print('{} performed train data balance P:{} : N:{}'.format(
        resampling_method['name'], positive, negative))
    return X_train, y_train

def _get_resampler(method, random_state, n_jobs=8):
    args = {
        'random_state': random_state,
        'n_jobs': n_jobs
    }
    if 'args' in method:
        args = {
            **args,
            **method['args']
        }
    if method['name'] == 'SMOTE':
        return SMOTE(**args)
    elif method['name'] == 'RandomUnderSampler':
        del args['n_jobs']
        return RandomUnderSampler(**args) # pylint: disable=unexpected-keyword-arg
    elif method['name'] == 'ClusterCentroids':
        return ClusterCentroids(**args)
    elif method['name'] == 'NearMiss':
        return NearMiss(**args)
    elif method['name'] == 'TomekLinks':
        return TomekLinks(**args)
    elif method['name'] == 'CondensedNearestNeighbour':
        return CondensedNearestNeighbour(**args)
    elif method['name'] == 'EditedNearestNeighbours':
        return EditedNearestNeighbours(**args)
    elif method['name'] == 'AllKNN':
        return AllKNN(**args)
    elif method['name'] == 'NeighbourhoodCleaningRule':
        return NeighbourhoodCleaningRule(**args)
    else:
        raise Exception('unknown resampler: {}'.format(method['name']))

class RandomUnderSamplerWrapper(RandomUnderSampler):

    def fit_resample(self, X, y):
        X_resample, y = super().fit_sample(X.reshape(X.shape[0], -1), y)
        X = X_resample.reshape(\
            X_resample.shape[0], \
            X.shape[1], \
            X.shape[2], \
            X.shape[3], \
            X.shape[4])
        return X, y

def _dump(o, path):
    with open(path, 'w') as f:
        json.dump(o, f)

class _Reporter(Callback):

    def __init__(self, X_test, y_test, monitor='val_loss', best_only=True):
        self.X_test = X_test
        self.y_test = y_test
        self.monitor = monitor
        self.best_only = best_only
        self.best = np.inf
        self.acc = []
        self.val_acc = []
        self.loss = []
        self.val_loss = []

    def on_epoch_end(self, epoch, logs={}):
        self.acc.append(float(logs.get('accuracy')))
        self.val_acc.append(float(logs.get('val_accuracy')))
        self.loss.append(float(logs.get('loss')))
        self.val_loss.append(float(logs.get('val_loss')))
        if self.best_only:
            score = logs.get(self.monitor)
            if self.best < score:
                return
            self.best = score
        acc, auc, f1, precision, recall, tp, fn, fp, tn = _eval(self.model, self.X_test, self.y_test)
        print('Epoch {}: AUC: {:.3f}, F1: {:.3f}, acc: {:.3f}, precision: {:.3f}, recall: {:.3f}, TP: {}, FN: {}, FP: {}, TN: {}'.format(\
            epoch + 1, auc, f1, acc, precision, recall, tp, fn, fp, tn
        ))

def _eval(model, X_test, y_test):
    y_pred = model.predict(X_test).reshape(-1)
    tn, fp, fn, tp = confusion_matrix(y_test, y_pred >= 0.5).ravel()
    auc = roc_auc_score(y_test, y_pred)
    f1 = (2 * tp) / (2 * tp + fp + fn)
    acc = (tp + tn) / (tp + tn + fp + fn)
    precision = tp /(tp + fp)
    recall = tp / (tp + fn)
    return acc, auc, f1, precision, recall, tp, fn, fp, tn

max_stars_repo_name: taqqanori/ml-earthquake

max_issues_repo_name: taqqanori/ml-earthquake

max_forks_repo_name: taqqanori/ml-earthquake


text:
__author__ = "schelle"

import os
import unittest

import numpy as np
import wflow.wflow_sceleton as wf

"""
Run sceleton for 10 steps and checks if the outcome is approx that of the reference run
"""

class MyTest(unittest.TestCase):
    def testapirun(self):
        startTime = 1
        stopTime = 20
        currentTime = 1

        # set runid, clonemap and casename. Also define the ini file
        runId = "unittest"
        configfile = "wflow_sceleton.ini"
        wflow_cloneMap = "wflow_catchment.map"
        caseName = "wflow_sceleton"

        myModel = wf.WflowModel(wflow_cloneMap, caseName, runId, configfile)
        # initialise the framework
        dynModelFw = wf.wf_DynamicFramework(myModel, stopTime, startTime)

        # Load model config from files and check directory structure
        dynModelFw.createRunId(NoOverWrite=False, level=wf.logging.DEBUG)
        # Run the initial part of the model (reads parameters and sets initial values)
        dynModelFw._runInitial()  # Runs initial part

        dynModelFw._runResume()  # gets the state variables

        for ts in range(startTime, stopTime):
            dynModelFw._runDynamic(ts, ts)  # runs for all timesteps
            dynModelFw.logger.info("Doing step: " + str(ts))
        dynModelFw._runSuspend()  # saves the state variables
        dynModelFw._wf_shutdown()

        my_data = np.genfromtxt(os.path.join(caseName, runId, "tes.csv"), delimiter=",")
        self.assertAlmostEqual(14.885358393192291, my_data[:, 2].sum())
        my_data_mean = np.genfromtxt(
            os.path.join(caseName, runId, "tes_mean_5.csv"), delimiter=","
        )
        self.assertAlmostEqual(20.727288454771042, my_data_mean[:, 2].sum())

if __name__ == "__main__":
    unittest.main()

max_stars_repo_name: quanpands/wflow

max_issues_repo_name: quanpands/wflow

max_forks_repo_name: quanpands/wflow


text:
import numpy as np
from textattack.transformations import Transformation
from flair.data import Sentence
from flair.models import SequenceTagger

def cluster_idx(idx_ls):
    """Given a list of idx, return a list that contains sub-lists of adjacent
    idx."""

    if len(idx_ls) < 2:
        return [[i] for i in idx_ls]
    else:
        output = [[idx_ls[0]]]
        prev = idx_ls[0]
        list_pos = 0

        for idx in idx_ls[1:]:
            if idx - 1 == prev:
                output[list_pos].append(idx)
            else:
                output.append([idx])
                list_pos += 1
            prev = idx
        return output

def idx_to_words(ls, words):
    """Given a list generated from cluster_idx, return a list that contains
    sub-list (the first element being the idx, and the second element being the
    words corresponding to the idx)"""

    output = []
    for sub_ls in ls:
        word = words[sub_ls[0]]
        for idx in sub_ls[1:]:
            word = " ".join([word, words[idx]])
        output.append([sub_ls, word])
    return output

class WordSwapChangeLocation(Transformation):
    def __init__(self, n=3, confidence_score=0.7, **kwargs):
        """Transformation that changes recognized locations of a sentence to
        another location that is given in the location map.

        :param n: Number of new locations to generate
        :param confidence_score: Location will only be changed if it's above the confidence score
        """
        super().__init__(**kwargs)
        self.n = n
        self.confidence_score = confidence_score

    def _get_transformations(self, current_text, indices_to_modify):
        words = current_text.words
        # TODO: move ner recognition to AttackedText
        # really want to silent this line:
        tagger = SequenceTagger.load("ner")
        sentence = Sentence(current_text.text)
        tagger.predict(sentence)
        location_idx = []

        # pre-screen for actual locations, using flair
        # summarize location idx into a list (location_idx)
        for token in sentence:
            tag = token.get_tag("ner")
            if (
                "LOC" in tag.value
                and tag.score > self.confidence_score
                and (token.idx - 1) in indices_to_modify
            ):
                location_idx.append(token.idx - 1)

        # Combine location idx and words to a list ([0] is idx, [1] is location name)
        # For example, [1,2] to [ [1,2] , ["New York"] ]
        location_idx = cluster_idx(location_idx)
        location_words = idx_to_words(location_idx, words)

        transformed_texts = []
        for location in location_words:
            idx = location[0]
            word = location[1]
            replacement_words = self._get_new_location(word)
            for r in replacement_words:
                if r == word:
                    continue
                text = current_text

                # if original location is more than a single word, remain only the starting word
                if len(idx) > 1:
                    index = idx[1]
                    for i in idx[1:]:
                        text = text.delete_word_at_index(index)

                # replace the starting word with new location
                text = text.replace_word_at_index(idx[0], r)

                transformed_texts.append(text)
        return transformed_texts

    def _get_new_location(self, word):
        """Return a list of new locations, with the choice of country,
        nationality, and city."""
        if word in LOCATION["country"]:
            return np.random.choice(LOCATION["country"], self.n)
        elif word in LOCATION["nationality"]:
            return np.random.choice(LOCATION["nationality"], self.n)
        elif word in LOCATION["city"]:
            return np.random.choice(LOCATION["city"], self.n)
        return []

LOCATION = {
    "country": [
        "China",
        "India",
        "United States",
        "Indonesia",
        "Pakistan",
        "Brazil",
        "Nigeria",
        "Bangladesh",
        "Russian Federation",
        "Japan",
        "Mexico",
        "Ethiopia",
        "Philippines",
        "Egypt",
        "Vietnam",
        "Germany",
        "Turkey",
        "Iran",
        "Thailand",
        "France",
        "United Kingdom",
        "Italy",
        "South Africa",
        "Tanzania",
        "Myanmar",
        "Kenya",
        "Colombia",
        "Spain",
        "Ukraine",
        "Argentina",
        "Uganda",
        "Algeria",
        "Sudan",
        "Iraq",
        "Poland",
        "Afghanistan",
        "Canada",
        "Morocco",
        "Saudi Arabia",
        "Uzbekistan",
        "Peru",
        "Malaysia",
        "Angola",
        "Ghana",
        "Mozambique",
        "Venezuela",
        "Yemen",
        "Nepal",
        "Madagascar",
        "Korea",
        "Cameroon",
        "Australia",
        "Niger",
        "Sri Lanka",
        "Burkina Faso",
        "Romania",
        "Mali",
        "Chile",
        "Kazakhstan",
        "Malawi",
        "Zambia",
        "Guatemala",
        "Netherlands",
        "Ecuador",
        "Syrian Arab Republic",
        "Cambodia",
        "Senegal",
        "Chad",
        "Somalia",
        "Zimbabwe",
        "Guinea",
        "Rwanda",
        "Tunisia",
        "Benin",
        "Belgium",
        "Bolivia",
        "Cuba",
        "Burundi",
        "Haiti",
        "South Sudan",
        "Greece",
        "Dominican Republic",
        "Czech Republic",
        "Portugal",
        "Sweden",
        "Jordan",
        "Azerbaijan",
        "Hungary",
        "United Arab Emirates",
        "Honduras",
        "Belarus",
        "Tajikistan",
        "Israel",
        "Austria",
        "Papua New Guinea",
        "Switzerland",
        "Togo",
        "Sierra Leone",
        "Hong Kong SAR",
        "Lao PDR",
        "Bulgaria",
        "Serbia",
        "Paraguay",
        "Lebanon",
        "Libya",
        "Nicaragua",
        "El Salvador",
        "Kyrgyz Republic",
        "Turkmenistan",
        "Denmark",
        "Singapore",
        "Finland",
        "Slovak Republic",
        "Norway",
        "Congo",
        "Costa Rica",
        "New Zealand",
        "Ireland",
        "Oman",
        "Liberia",
        "Central African Republic",
        "West Bank and Gaza",
        "Mauritania",
        "Panama",
        "Kuwait",
        "Croatia",
        "Georgia",
        "Moldova",
        "Uruguay",
        "Bosnia and Herzegovina",
        "Eritrea",
        "Puerto Rico",
        "Mongolia",
        "Armenia",
        "Jamaica",
        "Albania",
        "Lithuania",
        "Qatar",
        "Namibia",
        "Gambia",
        "Botswana",
        "Gabon",
        "Lesotho",
        "North Macedonia",
        "Slovenia",
        "Latvia",
        "Guinea-Bissau",
        "Kosovo",
        "Bahrain",
        "Trinidad and Tobago",
        "Estonia",
        "Equatorial Guinea",
        "Timor-Leste",
        "Mauritius",
        "Cyprus",
        "Eswatini",
        "Djibouti",
        "Fiji",
        "Comoros",
        "Guyana",
        "Bhutan",
        "Solomon Islands",
        "Macao SAR",
        "Montenegro",
        "Luxembourg",
        "Suriname",
        "Cabo Verde",
        "Maldives",
        "Malta",
        "Brunei Darussalam",
        "Bahamas",
        "Belize",
        "Iceland",
        "Vanuatu",
        "Barbados",
        "New Caledonia",
        "French Polynesia",
        "Samoa",
        "St. Lucia",
        "Channel Islands",
        "Guam",
        "Kiribati",
        "Micronesia",
        "Grenada",
        "St. Vincent and the Grenadines",
        "Virgin Islands (U.S.)",
        "Aruba",
        "Tonga",
        "Seychelles",
        "Antigua and Barbuda",
        "Isle of Man",
        "Andorra",
        "Dominica",
        "Cayman Islands",
        "Bermuda",
        "Marshall Islands",
        "Northern Mariana Islands",
        "Greenland",
        "American Samoa",
        "St. Kitts and Nevis",
        "Faroe Islands",
        "Sint Maarten (Dutch part)",
        "Monaco",
        "Liechtenstein",
        "Turks and Caicos Islands",
        "St. Martin (French part)",
        "San Marino",
        "Gibraltar",
        "British Virgin Islands",
        "Palau",
        "Nauru",
        "Tuvalu",
        "C\u00f4te d'Ivoire",
        "Cura\u00e7ao",
        "S\u00e3o Tom\u00e9 and Principe",
    ],
    "nationality": [
        "Chinese",
        "Indian",
        "American",
        "Indonesian",
        "Pakistani",
        "Brazilian",
        "Nigerian",
        "Bangladeshi",
        "Russian",
        "Japanese",
        "Mexican",
        "Ethiopian",
        "Philippine",
        "Egyptian",
        "Vietnamese",
        "German",
        "Turkish",
        "Iranian",
        "Thai",
        "French",
        "British",
        "Italian",
        "South African",
        "Tanzanian",
        "Burmese",
        "Kenyan",
        "Colombian",
        "Spanish",
        "Ukrainian",
        "Argentine",
        "Ugandan",
        "Algerian",
        "Sudanese",
        "Iraqi",
        "Polish",
        "Afghan",
        "Canadian",
        "Moroccan",
        "Saudi",
        "Uzbekistani",
        "Peruvian",
        "Malaysian",
        "Angolan",
        "Ghanaian",
        "Mozambican",
        "Venezuelan",
        "Yemeni",
        "Nepali",
        "Malagasy",
        "South Korean",
        "Cameroonian",
        "Australian",
        "Nigerien",
        "Sri Lankan",
        "Burkinab\u00e9",
        "Romanian",
        "Malian",
        "Chilean",
        "Kazakhstani",
        "Malawian",
        "Zambian",
        "Guatemalan",
        "Dutch",
        "Ecuadorian",
        "Syrian",
        "Cambodian",
        "Senegalese",
        "Chadian",
        "Somali",
        "Zimbabwean",
        "Guinean",
        "Rwandan",
        "Tunisian",
        "Beninese",
        "Belgian",
        "Bolivian",
        "Cuban",
        "Burundian",
        "Haitian",
        "South Sudanese",
        "Greek",
        "Dominican",
        "Czech",
        "Portuguese",
        "Swedish",
        "Jordanian",
        "Azerbaijani",
        "Hungarian",
        "Emirati",
        "Honduran",
        "Belarusian",
        "Tajikistani",
        "Israeli",
        "Austrian",
        "Papua New Guinean",
        "Swiss",
        "Togolese",
        "Sierra Leonean",
        "Hong Kong",
        "Lao",
        "Bulgarian",
        "Serbian",
        "Paraguayan",
        "Lebanese",
        "Libyan",
        "Nicaraguan",
        "Salvadoran",
        "Kyrgyzstani",
        "Turkmen",
        "Danish",
        "Singaporean",
        "Finnish",
        "Slovak",
        "Norwegian",
        "Congolese",
        "Costa Rican",
        "New Zealand",
        "Irish",
        "Omani",
        "Liberian",
        "Central African",
        "Palestinian",
        "Mauritanian",
        "Panamanian",
        "Kuwaiti",
        "Croatian",
        "Georgian",
        "Moldovan",
        "Uruguayan",
        "Bosnian or Herzegovinian",
        "Eritrean",
        "Puerto Rican",
        "Mongolian",
        "Armenian",
        "Jamaican",
        "Albanian",
        "Lithuanian",
        "Qatari",
        "Namibian",
        "Gambian",
        "Motswana",
        "Gabonese",
        "Basotho",
        "Macedonian",
        "Slovenian",
        "Latvian",
        "Bissau-Guinean",
        "from Kosovo",
        "Bahraini",
        "Trinidadian or Tobagonian",
        "Estonian",
        "Equatorial Guinean",
        "Timorese",
        "Mauritian",
        "Cypriot",
        "Swazi",
        "Djiboutian",
        "Fijian",
        "Comoran",
        "Guyanese",
        "Bhutanese",
        "Solomon Island",
        "Macanese",
        "Montenegrin",
        "Luxembourg",
        "Surinamese",
        "Cabo Verdean",
        "Maldivian",
        "Maltese",
        "Bruneian",
        "Bahamian",
        "Belizean",
        "Icelandic",
        "Ni-Vanuatu",
        "Barbadian",
        "New Caledonian",
        "French Polynesian",
        "Samoan",
        "Saint Lucian",
        "from Channel Islands",
        "Guamanian",
        "I-Kiribati",
        "Micronesian",
        "Grenadian",
        "Saint Vincentian",
        "U.S. Virgin Island",
        "Aruban",
        "Tongan",
        "Seychellois",
        "Antiguan or Barbudan",
        "Manx",
        "Andorran",
        "Dominican",
        "Caymanian",
        "Bermudian",
        "Marshallese",
        "Northern Marianan",
        "Greenlandic",
        "American Samoan",
        "Kittitian or Nevisian",
        "Faroese",
        "Sint Maarten",
        "Mon\u00e9gasque",
        "Liechtenstein",
        "Turks and Caicos Island",
        "Saint-Martinoise",
        "Sammarinese",
        "Gibraltar",
        "British Virgin Island",
        "Palauan",
        "Nauruan",
        "Tuvaluan",
        "Ivorian",
        "Cura\u00e7aoan",
        "S\u00e3o Tom\u00e9an",
    ],
    "city": [
        "New York",
        "Los Angeles",
        "Chicago",
        "Houston",
        "Philadelphia",
        "Phoenix",
        "San Antonio",
        "San Diego",
        "Dallas",
        "San Jose",
        "Austin",
        "Indianapolis",
        "Jacksonville",
        "San Francisco",
        "Columbus",
        "Charlotte",
        "Fort Worth",
        "Detroit",
        "El Paso",
        "Memphis",
        "Seattle",
        "Denver",
        "Washington",
        "Boston",
        "Nashville-Davidson",
        "Baltimore",
        "Oklahoma City",
        "Louisville/Jefferson County",
        "Portland",
        "Las Vegas",
        "Milwaukee",
        "Albuquerque",
        "Tucson",
        "Fresno",
        "Sacramento",
        "Long Beach",
        "Kansas City",
        "Mesa",
        "Virginia Beach",
        "Atlanta",
        "Colorado Springs",
        "Omaha",
        "Raleigh",
        "Miami",
        "Oakland",
        "Minneapolis",
        "Tulsa",
        "Cleveland",
        "Wichita",
        "Arlington",
        "New Orleans",
        "Bakersfield",
        "Tampa",
        "Honolulu",
        "Aurora",
        "Anaheim",
        "Santa Ana",
        "St. Louis",
        "Riverside",
        "Corpus Christi",
        "Lexington-Fayette",
        "Pittsburgh",
        "Anchorage",
        "Stockton",
        "Cincinnati",
        "St. Paul",
        "Toledo",
        "Greensboro",
        "Newark",
        "Plano",
        "Henderson",
        "Lincoln",
        "Buffalo",
        "Jersey City",
        "Chula Vista",
        "Fort Wayne",
        "Orlando",
        "St. Petersburg",
        "Chandler",
        "Laredo",
        "Norfolk",
        "Durham",
        "Madison",
        "Lubbock",
        "Irvine",
        "Winston-Salem",
        "Glendale",
        "Garland",
        "Hialeah",
        "Reno",
        "Chesapeake",
        "Gilbert",
        "Baton Rouge",
        "Irving",
        "Scottsdale",
        "North Las Vegas",
        "Fremont",
        "Boise City",
        "Richmond",
        "San Bernardino",
        "Birmingham",
        "Spokane",
        "Rochester",
        "Des Moines",
        "Modesto",
        "Fayetteville",
        "Tacoma",
        "Oxnard",
        "Fontana",
        "Columbus",
        "Montgomery",
        "Moreno Valley",
        "Shreveport",
        "Aurora",
        "Yonkers",
        "Akron",
        "Huntington Beach",
        "Little Rock",
        "Augusta-Richmond County",
        "Amarillo",
        "Glendale",
        "Mobile",
        "Grand Rapids",
        "Salt Lake City",
        "Tallahassee",
        "Huntsville",
        "Grand Prairie",
        "Knoxville",
        "Worcester",
        "Newport News",
        "Brownsville",
        "Overland Park",
        "Santa Clarita",
        "Providence",
        "Garden Grove",
        "Chattanooga",
        "Oceanside",
        "Jackson",
        "Fort Lauderdale",
        "Santa Rosa",
        "Rancho Cucamonga",
        "Port St. Lucie",
        "Tempe",
        "Ontario",
        "Vancouver",
        "Cape Coral",
        "Sioux Falls",
        "Springfield",
        "Peoria",
        "Pembroke Pines",
        "Elk Grove",
        "Salem",
        "Lancaster",
        "Corona",
        "Eugene",
        "Palmdale",
        "Salinas",
        "Springfield",
        "Pasadena",
        "Fort Collins",
        "Hayward",
        "Pomona",
        "Cary",
        "Rockford",
        "Alexandria",
        "Escondido",
        "McKinney",
        "Kansas City",
        "Joliet",
        "Sunnyvale",
        "Torrance",
        "Bridgeport",
        "Lakewood",
        "Hollywood",
        "Paterson",
        "Naperville",
        "Syracuse",
        "Mesquite",
        "Dayton",
        "Savannah",
        "Clarksville",
        "Orange",
        "Pasadena",
        "Fullerton",
        "Killeen",
        "Frisco",
        "Hampton",
        "McAllen",
        "Warren",
        "Bellevue",
        "West Valley City",
        "Columbia",
        "Olathe",
        "Sterling Heights",
        "New Haven",
        "Miramar",
        "Waco",
        "Thousand Oaks",
        "Cedar Rapids",
        "Charleston",
        "Visalia",
        "Topeka",
        "Elizabeth",
        "Gainesville",
        "Thornton",
        "Roseville",
        "Carrollton",
        "Coral Springs",
        "Stamford",
        "Simi Valley",
        "Concord",
        "Hartford",
        "Kent",
        "Lafayette",
        "Midland",
        "Surprise",
        "Denton",
        "Victorville",
        "Evansville",
        "Santa Clara",
        "Abilene",
        "Athens-Clarke County",
        "Vallejo",
        "Allentown",
        "Norman",
        "Beaumont",
        "Independence",
        "Murfreesboro",
        "Ann Arbor",
        "Springfield",
        "Berkeley",
        "Peoria",
        "Provo",
        "El Monte",
        "Columbia",
        "Lansing",
        "Fargo",
        "Downey",
        "Costa Mesa",
        "Wilmington",
        "Arvada",
        "Inglewood",
        "Miami Gardens",
        "Carlsbad",
        "Westminster",
        "Rochester",
        "Odessa",
        "Manchester",
        "Elgin",
        "West Jordan",
        "Round Rock",
        "Clearwater",
        "Waterbury",
        "Gresham",
        "Fairfield",
        "Billings",
        "Lowell",
        "San Buenaventura (Ventura)",
        "Pueblo",
        "High Point",
        "West Covina",
        "Richmond",
        "Murrieta",
        "Cambridge",
        "Antioch",
        "Temecula",
        "Norwalk",
        "Centennial",
        "Everett",
        "Palm Bay",
        "Wichita Falls",
        "Green Bay",
        "Daly City",
        "Burbank",
        "Richardson",
        "Pompano Beach",
        "North Charleston",
        "Broken Arrow",
        "Boulder",
        "West Palm Beach",
        "Santa Maria",
        "El Cajon",
        "Davenport",
        "Rialto",
        "Las Cruces",
        "San Mateo",
        "Lewisville",
        "South Bend",
        "Lakeland",
        "Erie",
        "Tyler",
        "Pearland",
        "College Station",
        "Kenosha",
        "Sandy Springs",
        "Clovis",
        "Flint",
        "Roanoke",
        "Albany",
        "Jurupa Valley",
        "Compton",
        "San Angelo",
        "Hillsboro",
        "Lawton",
        "Renton",
        "Vista",
        "Davie",
        "Greeley",
        "Mission Viejo",
        "Portsmouth",
        "Dearborn",
        "South Gate",
        "Tuscaloosa",
        "Livonia",
        "New Bedford",
        "Vacaville",
        "Brockton",
        "Roswell",
        "Beaverton",
        "Quincy",
        "Sparks",
        "Yakima",
        "Lee's Summit",
        "Federal Way",
        "Carson",
        "Santa Monica",
        "Hesperia",
        "Allen",
        "Rio Rancho",
        "Yuma",
        "Westminster",
        "Orem",
        "Lynn",
        "Redding",
        "Spokane Valley",
        "Miami Beach",
        "League City",
        "Lawrence",
        "Santa Barbara",
        "Plantation",
        "Sandy",
        "Sunrise",
        "Macon",
        "Longmont",
        "Boca Raton",
        "San Marcos",
        "Greenville",
        "Waukegan",
        "Fall River",
        "Chico",
        "Newton",
        "San Leandro",
        "Reading",
        "Norwalk",
        "Fort Smith",
        "Newport Beach",
        "Asheville",
        "Nashua",
        "Edmond",
        "Whittier",
        "Nampa",
        "Bloomington",
        "Deltona",
        "Hawthorne",
        "Duluth",
        "Carmel",
        "Suffolk",
        "Clifton",
        "Citrus Heights",
        "Livermore",
        "Tracy",
        "Alhambra",
        "Kirkland",
        "Trenton",
        "Ogden",
        "Hoover",
        "Cicero",
        "Fishers",
        "Sugar Land",
        "Danbury",
        "Meridian",
        "Indio",
        "Concord",
        "Menifee",
        "Champaign",
        "Buena Park",
        "Troy",
        "O'Fallon",
        "Johns Creek",
        "Bellingham",
        "Westland",
        "Bloomington",
        "Sioux City",
        "Warwick",
        "Hemet",
        "Longview",
        "Farmington Hills",
        "Bend",
        "Lakewood",
        "Merced",
        "Mission",
        "Chino",
        "Redwood City",
        "Edinburg",
        "Cranston",
        "Parma",
        "New Rochelle",
        "Lake Forest",
        "Napa",
        "Hammond",
        "Fayetteville",
        "Bloomington",
        "Avondale",
        "Somerville",
        "Palm Coast",
        "Bryan",
        "Gary",
        "Largo",
        "Brooklyn Park",
        "Tustin",
        "Racine",
        "Deerfield Beach",
        "Lynchburg",
        "Mountain View",
        "Medford",
        "Lawrence",
        "Bellflower",
        "Melbourne",
        "St. Joseph",
        "Camden",
        "St. George",
        "Kennewick",
        "Baldwin Park",
        "Chino Hills",
        "Alameda",
        "Albany",
        "Arlington Heights",
        "Scranton",
        "Evanston",
        "Kalamazoo",
        "Baytown",
        "Upland",
        "Springdale",
        "Bethlehem",
        "Schaumburg",
        "Mount Pleasant",
        "Auburn",
        "Decatur",
        "San Ramon",
        "Pleasanton",
        "Wyoming",
        "Lake Charles",
        "Plymouth",
        "Bolingbrook",
        "Pharr",
        "Appleton",
        "Gastonia",
        "Folsom",
        "Southfield",
        "Rochester Hills",
        "New Britain",
        "Goodyear",
        "Canton",
        "Warner Robins",
        "Union City",
        "Perris",
        "Manteca",
        "Iowa City",
        "Jonesboro",
        "Wilmington",
        "Lynwood",
        "Loveland",
        "Pawtucket",
        "Boynton Beach",
        "Waukesha",
        "Gulfport",
        "Apple Valley",
        "Passaic",
        "Rapid City",
        "Layton",
        "Lafayette",
        "Turlock",
        "Muncie",
        "Temple",
        "Missouri City",
        "Redlands",
        "Santa Fe",
        "Lauderhill",
        "Milpitas",
        "Palatine",
        "Missoula",
        "Rock Hill",
        "Jacksonville",
        "Franklin",
        "Flagstaff",
        "Flower Mound",
        "Weston",
        "Waterloo",
        "Union City",
        "Mount Vernon",
        "Fort Myers",
        "Dothan",
        "Rancho Cordova",
        "Redondo Beach",
        "Jackson",
        "Pasco",
        "St. Charles",
        "Eau Claire",
        "North Richland Hills",
        "Bismarck",
        "Yorba Linda",
        "Kenner",
        "Walnut Creek",
        "Frederick",
        "Oshkosh",
        "Pittsburg",
        "Palo Alto",
        "Bossier City",
        "Portland",
        "St. Cloud",
        "Davis",
        "South San Francisco",
        "Camarillo",
        "North Little Rock",
        "Schenectady",
        "Gaithersburg",
        "Harlingen",
        "Woodbury",
        "Eagan",
        "Yuba City",
        "Maple Grove",
        "Youngstown",
        "Skokie",
        "Kissimmee",
        "Johnson City",
        "Victoria",
        "San Clemente",
        "Bayonne",
        "Laguna Niguel",
        "East Orange",
        "Shawnee",
        "Homestead",
        "Rockville",
        "Delray Beach",
        "Janesville",
        "Conway",
        "Pico Rivera",
        "Lorain",
        "Montebello",
        "Lodi",
        "New Braunfels",
        "Marysville",
        "Tamarac",
        "Madera",
        "Conroe",
        "Santa Cruz",
        "Eden Prairie",
        "Cheyenne",
        "Daytona Beach",
        "Alpharetta",
        "Hamilton",
        "Waltham",
        "Coon Rapids",
        "Haverhill",
        "Council Bluffs",
        "Taylor",
        "Utica",
        "Ames",
        "La Habra",
        "Encinitas",
        "Bowling Green",
        "Burnsville",
        "Greenville",
        "West Des Moines",
        "Cedar Park",
        "Tulare",
        "Monterey Park",
        "Vineland",
        "Terre Haute",
        "North Miami",
        "Mansfield",
        "West Allis",
        "Bristol",
        "Taylorsville",
        "Malden",
        "Meriden",
        "Blaine",
        "Wellington",
        "Cupertino",
        "Springfield",
        "Rogers",
        "St. Clair Shores",
        "Gardena",
        "Pontiac",
        "National City",
        "Grand Junction",
        "Rocklin",
        "Chapel Hill",
        "Casper",
        "Broomfield",
        "Petaluma",
        "South Jordan",
        "Springfield",
        "Great Falls",
        "Lancaster",
        "North Port",
        "Lakewood",
        "Marietta",
        "San Rafael",
        "Royal Oak",
        "Des Plaines",
        "Huntington Park",
        "La Mesa",
        "Orland Park",
        "Auburn",
        "Lakeville",
        "Owensboro",
        "Moore",
        "Jupiter",
        "Idaho Falls",
        "Dubuque",
        "Bartlett",
        "Rowlett",
        "Novi",
        "White Plains",
        "Arcadia",
        "Redmond",
        "Lake Elsinore",
        "Ocala",
        "Tinley Park",
        "Port Orange",
        "Medford",
        "Oak Lawn",
        "Rocky Mount",
        "Kokomo",
        "Coconut Creek",
        "Bowie",
        "Berwyn",
        "Midwest City",
        "Fountain Valley",
        "Buckeye",
        "Dearborn Heights",
        "Woodland",
        "Noblesville",
        "Valdosta",
        "Diamond Bar",
        "Manhattan",
        "Santee",
        "Taunton",
        "Sanford",
        "Kettering",
        "New Brunswick",
        "Decatur",
        "Chicopee",
        "Anderson",
        "Margate",
        "Weymouth Town",
        "Hempstead",
        "Corvallis",
        "Eastvale",
        "Porterville",
        "West Haven",
        "Brentwood",
        "Paramount",
        "Grand Forks",
        "Georgetown",
        "St. Peters",
        "Shoreline",
        "Mount Prospect",
        "Hanford",
        "Normal",
        "Rosemead",
        "Lehi",
        "Pocatello",
        "Highland",
        "Novato",
        "Port Arthur",
        "Carson City",
        "San Marcos",
        "Hendersonville",
        "Elyria",
        "Revere",
        "Pflugerville",
        "Greenwood",
        "Bellevue",
        "Wheaton",
        "Smyrna",
        "Sarasota",
        "Blue Springs",
        "Colton",
        "Euless",
        "Castle Rock",
        "Cathedral City",
        "Kingsport",
        "Lake Havasu City",
        "Pensacola",
        "Hoboken",
        "Yucaipa",
        "Watsonville",
        "Richland",
        "Delano",
        "Hoffman Estates",
        "Florissant",
        "Placentia",
        "West New York",
        "Dublin",
        "Oak Park",
        "Peabody",
        "Perth Amboy",
        "Battle Creek",
        "Bradenton",
        "Gilroy",
        "Milford",
        "Albany",
        "Ankeny",
        "La Crosse",
        "Burlington",
        "DeSoto",
        "Harrisonburg",
        "Minnetonka",
        "Elkhart",
        "Lakewood",
        "Glendora",
        "Southaven",
        "Charleston",
        "Joplin",
        "Enid",
        "Palm Beach Gardens",
        "Brookhaven",
        "Plainfield",
        "Grand Island",
        "Palm Desert",
        "Huntersville",
        "Tigard",
        "Lenexa",
        "Saginaw",
        "Kentwood",
        "Doral",
        "Apple Valley",
        "Grapevine",
        "Aliso Viejo",
        "Sammamish",
        "Casa Grande",
        "Pinellas Park",
        "Troy",
        "West Sacramento",
        "Burien",
        "Commerce City",
        "Monroe",
        "Cerritos",
        "Downers Grove",
        "Coral Gables",
        "Wilson",
        "Niagara Falls",
        "Poway",
        "Edina",
        "Cuyahoga Falls",
        "Rancho Santa Margarita",
        "Harrisburg",
        "Huntington",
        "La Mirada",
        "Cypress",
        "Caldwell",
        "Logan",
        "Galveston",
        "Sheboygan",
        "Middletown",
        "Murray",
        "Roswell",
        "Parker",
        "Bedford",
        "East Lansing",
        "Methuen",
        "Covina",
        "Alexandria",
        "Olympia",
        "Euclid",
        "Mishawaka",
        "Salina",
        "Azusa",
        "Newark",
        "Chesterfield",
        "Leesburg",
        "Dunwoody",
        "Hattiesburg",
        "Roseville",
        "Bonita Springs",
        "Portage",
        "St. Louis Park",
        "Collierville",
        "Middletown",
        "Stillwater",
        "East Providence",
        "Lawrence",
        "Wauwatosa",
        "Mentor",
        "Ceres",
        "Cedar Hill",
        "Mansfield",
        "Binghamton",
        "Coeur d'Alene",
        "San Luis Obispo",
        "Minot",
        "Palm Springs",
        "Pine Bluff",
        "Texas City",
        "Summerville",
        "Twin Falls",
        "Jeffersonville",
        "San Jacinto",
        "Madison",
        "Altoona",
        "Columbus",
        "Beavercreek",
        "Apopka",
        "Elmhurst",
        "Maricopa",
        "Farmington",
        "Glenview",
        "Cleveland Heights",
        "Draper",
        "Lincoln",
        "Sierra Vista",
        "Lacey",
        "Biloxi",
        "Strongsville",
        "Barnstable Town",
        "Wylie",
        "Sayreville",
        "Kannapolis",
        "Charlottesville",
        "Littleton",
        "Titusville",
        "Hackensack",
        "Newark",
        "Pittsfield",
        "York",
        "Lombard",
        "Attleboro",
        "DeKalb",
        "Blacksburg",
        "Dublin",
        "Haltom City",
        "Lompoc",
        "El Centro",
        "Danville",
        "Jefferson City",
        "Cutler Bay",
        "Oakland Park",
        "North Miami Beach",
        "Freeport",
        "Moline",
        "Coachella",
        "Fort Pierce",
        "Smyrna",
        "Bountiful",
        "Fond du Lac",
        "Everett",
        "Danville",
        "Keller",
        "Belleville",
        "Bell Gardens",
        "Cleveland",
        "North Lauderdale",
        "Fairfield",
        "Salem",
        "Rancho Palos Verdes",
        "San Bruno",
        "Concord",
        "Burlington",
        "Apex",
        "Midland",
        "Altamonte Springs",
        "Hutchinson",
        "Buffalo Grove",
        "Urbandale",
        "State College",
        "Urbana",
        "Plainfield",
        "Manassas",
        "Bartlett",
        "Kearny",
        "Oro Valley",
        "Findlay",
        "Rohnert Park",
        "Westfield",
        "Linden",
        "Sumter",
        "Wilkes-Barre",
        "Woonsocket",
        "Leominster",
        "Shelton",
        "Brea",
        "Covington",
        "Rockwall",
        "Meridian",
        "Riverton",
        "St. Cloud",
        "Quincy",
        "Morgan Hill",
        "Warren",
        "Edmonds",
        "Burleson",
        "Beverly",
        "Mankato",
        "Hagerstown",
        "Prescott",
        "Campbell",
        "Cedar Falls",
        "Beaumont",
        "La Puente",
        "Crystal Lake",
        "Fitchburg",
        "Carol Stream",
        "Hickory",
        "Streamwood",
        "Norwich",
        "Coppell",
        "San Gabriel",
        "Holyoke",
        "Bentonville",
        "Florence",
        "Peachtree Corners",
        "Brentwood",
        "Bozeman",
        "New Berlin",
        "Goose Creek",
        "Huntsville",
        "Prescott Valley",
        "Maplewood",
        "Romeoville",
        "Duncanville",
        "Atlantic City",
        "Clovis",
        "The Colony",
        "Culver City",
        "Marlborough",
        "Hilton Head Island",
        "Moorhead",
        "Calexico",
        "Bullhead City",
        "Germantown",
        "La Quinta",
        "Lancaster",
        "Wausau",
        "Sherman",
        "Ocoee",
        "Shakopee",
        "Woburn",
        "Bremerton",
        "Rock Island",
        "Muskogee",
        "Cape Girardeau",
        "Annapolis",
        "Greenacres",
        "Ormond Beach",
        "Hallandale Beach",
        "Stanton",
        "Puyallup",
        "Pacifica",
        "Hanover Park",
        "Hurst",
        "Lima",
        "Marana",
        "Carpentersville",
        "Oakley",
        "Huber Heights",
        "Lancaster",
        "Montclair",
        "Wheeling",
        "Brookfield",
        "Park Ridge",
        "Florence",
        "Roy",
        "Winter Garden",
        "Chelsea",
        "Valley Stream",
        "Spartanburg",
        "Lake Oswego",
        "Friendswood",
        "Westerville",
        "Northglenn",
        "Phenix City",
        "Grove City",
        "Texarkana",
        "Addison",
        "Dover",
        "Lincoln Park",
        "Calumet City",
        "Muskegon",
        "Aventura",
        "Martinez",
        "Greenfield",
        "Apache Junction",
        "Monrovia",
        "Weslaco",
        "Keizer",
        "Spanish Fork",
        "Beloit",
        "Panama City",
    ],
}

max_stars_repo_name: chong-z/TextAttack

max_issues_repo_name: chong-z/TextAttack

max_forks_repo_name: chong-z/TextAttack


text:
#!/usr/bin/python
# encoding: utf8

"""
ToDos:
- Fix the error line 181, it seems that some clusters have zero elements
- Improve running times
    - Write alternative and efficient implementation
- Improve the timing script
- Improve the animation script
- There is an error in the line 177: total = total/self.dataset[self.clusters[c]].shape[0]
"""

from numpy import genfromtxt, where, random, savetxt
import numpy as np
from math import sqrt
from pprint import pprint

def euclidean_distance(a, b):
    """
    Calculate the Euclidean distance of two vectors
    """
    axis = 0 if b.ndim == 1 else 1
    summation = np.sum((a - b) ** 2, axis=axis)

    return np.sqrt(summation)

class KMeans(object):
    """
    Kmeans implementation
    """

    def __init__(self, dataset, method="lloyd", centroids=False, k=3, distance="euclidean"):
        """
        """

        # flag variable to control the convergence
        self.method = method
        self.convergence = False
        self.dataset = dataset
        self.no_dimensions = self.dataset.shape[1]
        self.k = k
        self.clusters = {i:[] for i in range(self.k)}
        self.centroids = np.zeros((self.k, self.no_dimensions))
        # macqueen vars
        self.i = 0
        self.counters = np.ones((self.k))

        # Distances. ToDo: Include more distances
        if distance == "euclidean":
            self.distance = euclidean_distance
        else:
            self.distance = euclidean_distance

        # Methods
        if method == "macqueen":
            self.next = self.macqueen

        elif method == "lloyd" and not centroids:
            self.next = self.lloyd
            self.random_centroids()
        elif method == "hartigan":
            self.epsilon = 0.1
            self.next = self.hartigan
            self.N = self.dataset.shape[0]
            self.x = np.random.choice(self.k, self.N)
            self.clusters = {i: np.where(self.x == i)[0].tolist() for i in range(self.k)}
            # calculate the centroids
            for c in self.clusters:
                indexes = self.clusters[c]
                self.centroids[c] = np.mean(dataset[indexes], 0)
            self.centroids_before = np.copy(self.centroids)
        else:
            # this depends on the number of columns in the dataset
            # it has something to do with lloyd ¬¬'
            self.k = k
            pass

    def random_centroids(self):
        """
        Create centroids
        """
        no_dimensions = self.dataset.shape[1]

        for i in range(self.k):
            for j, d in enumerate(self.dataset.transpose()):
                mini, maxi = min(d), max(d)
                self.centroids[i, j] = np.random.uniform(mini, maxi)

    def purity_function(self):
        """
        """

        intersect = lambda x, y: list(set(x) & set(y))

        # This values were given arbitrarily
        Ks = {
                0: range(50),
                1: range(50, 100),
                2: range(100, 150)
             }

        N = self.dataset.shape[0]

        total = 0
        for k in Ks:
            maxi = 0
            for c in self.clusters:
                intersection = intersect(Ks[k], self.clusters[c])
                if len(intersection) > maxi:
                    maxi = len(intersection)

            total += maxi

        purity_value = 1.0/N * total
        return purity_value

    def macqueen(self):
        """"""

        if self.i == self.dataset.shape[0]:
            self.convergence = True

        if not self.convergence:

            d = self.dataset[self.i]

            if self.i < self.k:
                self.centroids[self.i] = d
                self.clusters[self.i].append(self.i)
                self.i += 1
            else:
                # determine the winner centroid
                distances = euclidean_distance(d, self.centroids)
                n = np.argmin(distances)

                # recalculate centroids
                self.centroids[n] = self.centroids[n] + 1/(self.counters[n] + 1) * (d - self.centroids[n])

                self.counters[n] += 1
                self.clusters[n].append(self.i)
                self.i += 1

    def lloyd(self):
        """
        The classic kmeans
        """

        if not self.convergence:

            # assign each instance to closest center
            clusters = {i:[] for i in range(self.k)}

            for i, element in enumerate(self.dataset):
                closest = []
                for centroid in self.centroids:
                    closest.append(self.distance(centroid, element))

                # dirty code +_+
                index = closest.index(min(closest))
                clusters[index].append(i)

            self.clusters = clusters

            # recalculate centroids
            old_centroids = self.centroids.copy()

            for c in self.clusters:
                # get the sum of each column (dimension)
                total = np.sum(self.dataset[self.clusters[c]], axis=0)
                print(self.dataset[self.clusters[c]].shape[0])
                total = total/self.dataset[self.clusters[c]].shape[0]
                # update the centroids
                self.centroids[c] = total

            # expensive :O
            comparison = old_centroids == self.centroids
            self.convergence = np.sum(comparison) == comparison.size

    def hartigan(self):
        """"""

        if not self.convergence:
            self.convergence = False

            d = self.dataset[self.i]
            distances = euclidean_distance(d, self.centroids)
            n = np.argmin(distances)

            # if winner != current, calculate the new centroid
            if n != self.x[self.i]:
                self.x[self.i] = n

                # recalculate the centroids
                self.clusters = {i: np.where(self.x == i)[0].tolist() for i in range(self.k)}

                for c in self.clusters:
                    indexes = self.clusters[c]
                    self.centroids[c] = np.mean(self.dataset[indexes], 0)
                # self.i += 0

            self.i += 1

            # finish iteration, implement convergence with epsilon value
            if self.i == self.dataset.shape[0]:
                if np.array_equal(self.centroids_before, self.centroids):
                    self.convergence = True
                else:
                    self.i = 0
                    self.centroids_before = np.copy(self.centroids)

max_stars_repo_name: omartrinidad/pattern-recognition-bit

max_issues_repo_name: omartrinidad/pattern-recognition-bit

max_forks_repo_name: omartrinidad/pattern-recognition-bit

reason: import numpy,from numpy


text:
from ResoFit.calibration import Calibration
from ResoFit.fitresonance import FitResonance
import matplotlib.pyplot as plt
import numpy as np
from ResoFit._utilities import get_foil_density_gcm3
from ResoFit._utilities import Layer
import pprint

# Global parameters
energy_min = 4.09
energy_max = 1000
energy_step = 0.01
# Input sample name or names as str, case sensitive
layers = Layer()
layers.add_layer(layer='Ag', thickness_mm=0.025)
# layers.add_layer(layer='Co', thickness_mm=0.025)
# layers.add_layer(layer='Hf', thickness_mm=0.025)
# layers.add_layer(layer='W', thickness_mm=0.05)
# layers.add_layer(layer='In', thickness_mm=0.05)
# layers.add_layer(layer='Cd', thickness_mm=0.5)
# layers.add_layer(layer='Au', thickness_mm=0.01)
# simu = Simulation(energy_min=energy_min, energy_max=energy_max, energy_step=energy_step)
# simu.add_Layer(layer=layers)
# peak_dict = simu.peak_map(thres=0.015, min_dist=20)
# pprint.pprint(peak_dict)

folder = 'data/IPTS_13639/reso_data_13639'
data_file = 'Ag.csv'
spectra_file = 'spectra.csv'
image_start = 300  # Can be omitted or =None
image_end = 2720  # Can be omitted or =None
# norm_to_file = 'ob_1.csv'  #'Ag.csv'
# norm_to_file = 'Ag.csv'
norm_to_file = 'ob_all.csv'
baseline = False
each_step = True

repeat = 1.2
source_to_detector_m = 16.126845685903064  # 16#16.445359069030175#16.447496101100739
offset_us = -12112.431834715671  # 0#2.7120797253959119#2.7355447625559037

# Calibrate the peak positions
calibration = Calibration(data_file=data_file,
                          spectra_file=spectra_file,
                          layer=layers,
                          energy_min=energy_min,
                          energy_max=energy_max,
                          energy_step=energy_step,
                          folder=folder,
                          baseline=baseline)

calibration.experiment.norm_to(norm_to_file)
calibration.experiment.slice(start=image_start, end=image_end)

calibration.experiment.plot(
    x_type='energy',
    source_to_detector_m=source_to_detector_m,
    offset_us=offset_us,
    logx=True,
)
# calibrate_result = calibration.calibrate(source_to_detector_m=source_to_detector_m,
#                                          offset_us=offset_us,
#                                          vary='all',
#                                          each_step=each_step)
# calibration.index_peak(thres=0.05, min_dist=10, map_min_dist=10, map_thres=0.05)
# # calibration.analyze_peak()
# calibration.plot(
#     y_type='attenuation',
#     # y_type='transmission',
#     x_type='energy',
#     # t_unit='ms',
#     # before=True,
#     # interp=True,
#     mixed=True,
#     # peak_exp='all',
#     table=False,
#     peak_exp='indexed',
#     peak_height=True,
#     index_level='iso',
#     peak_id='all',
#     logx=True,
# )
plt.xlim(left=0, right=1000)
plt.show()

max_stars_repo_name: ornlneutronimaging/ResoFit

max_issues_repo_name: ornlneutronimaging/ResoFit

max_forks_repo_name: ornlneutronimaging/ResoFit