text:
import cv2
import cPickle, os
import numpy as np
from faster_rcnn.datasets.factory import get_imdb

# up to 5
det_file_dirs = ['/home/yang/Faster-RCNN-Refinement/output/faster_rcnn_end2end/voc_2007_test/faster_rcnn_100000']
visualization_output_dir = 'visualization'
# orange, yellow, green, blue, red
colors = [(66, 164, 244), (65, 235, 244), (0, 204, 0), (244, 181, 65), (65, 82, 244)]

def vis_detections(im, class_name, dets, model_idx, thresh=0.8, text=False):
    """Visual debugging of detections."""
    for i in range(np.minimum(10, dets.shape[0])):
        bbox = tuple(int(np.round(x)) for x in dets[i, :4])
        score = dets[i, -1]
        if score > thresh:
            cv2.rectangle(im, bbox[0:2], bbox[2:4], colors[model_idx], 2)
            if text:
                cv2.putText(im, '%s: %.3f' % (class_name, score), (bbox[0], bbox[1] + 15), cv2.FONT_HERSHEY_PLAIN,
                            1.0, (0, 0, 255), thickness=1)
    return im

def visualize(imdb):
    """Test a Fast R-CNN network on an image database."""
    num_images = len(imdb.image_index)
    # all detections are collected into:
    #    all_boxes[cls][image] = N x 5 array of detections in
    #    (x1, y1, x2, y2, score)
    detection_results = []
    for det_file_dir in det_file_dirs:
        det_file = os.path.join(det_file_dir, 'detections.pkl')
        with open(det_file, 'rb') as f:
            all_boxes = cPickle.load(f)
        detection_results.append(all_boxes)

    for i in range(num_images):
        im = cv2.imread(imdb.image_path_at(i))
        im2show = np.copy(im)

        for model_idx, detection_result in enumerate(detection_results):
            # skip j = 0, because it's the background class
            for j in xrange(1, imdb.num_classes):
                cls_dets = detection_result[j][i]
                im2show = vis_detections(im2show, imdb.classes[j], cls_dets, model_idx)

        cv2.imwrite(os.path.join(visualization_output_dir, str(i) + '.png'), im2show)
        if (i+1) % 100 == 0:
            print('im_write: %d/%d' % (i+1, num_images))

if __name__ == '__main__':
    imdb_name = 'voc_2007_test'
    imdb = get_imdb(imdb_name)
    imdb.competition_mode(on=True)
    visualize(imdb)


text:
import numpy as np
import operator
def classify0(inX, dataset, labels, k):
    dataset_size = dataset.shape[0]
    diff_mat = np.tile(inX, (dataset_size,1)) - dataset
    diff_mat_square = diff_mat ** 2
    sq_distances = np.sum(diff_mat_square, axis=1)
    distances =  sq_distances ** 0.5
    sorted_distances = distances.argsort()
    class_count = {}
    for i in range(k):
        vote_label = labels[sorted_distances[i]]
        # Have some questions about it...
        class_count[vote_label] = class_count.get(vote_label,0) + 1
        # Take cautions over here, I changed it into python3 standard dict implementation...
    sorted_class_count = sorted(class_count.items(), key= lambda x: x[1],reverse=True)
    return sorted_class_count[0][0]
def file2matrix(filename):
    fr = open(filename)
    arrayOLines = fr.readlines()
    line_numbers = len(arrayOLines)
    return_mat = np.zeros((line_numbers,3))
    class_label_vector = []
    index = 0
    for line in arrayOLines:
        line = line.strip()
        listFromLine = line.split('\t')
        return_mat[index, :] = listFromLine[0:3]
        class_label_vector.append(int(listFromLine[-1]))
        index+=1
    return return_mat, class_label_vector
def autoNorm(dataset):
    min_value = dataset.min(0)
    max_value = dataset.max(0)
    ranges = max_value - min_value
    m = dataset.shape[0]
    norm_dataset = np.zeros(np.shape(dataset))
    norm_dataset = dataset - np.tile(min_value, (m, 1))
    norm_dataset = norm_dataset / np.tile(ranges, (m, 1))
    return norm_dataset, ranges, min_value
def datingClassTest():
    ho_ratio = 0.1
    error = 0
    dating_data, dating_labels = file2matrix('datingTestSet2.txt')
    norm_dating, _,_ = autoNorm(dating_data)
    m = len(norm_dating)
    num_test_vec = int(ho_ratio * m)
    for i in range(num_test_vec):
        result = classify0(norm_dating[i],norm_dating[num_test_vec: -1,:]
                           ,dating_labels[num_test_vec:-1], 3)
        print('The classifier came back with: %d, the real answer is: %d'%(result, dating_labels[i]))

        if (result != dating_labels[i]):
            error +=1
    print('total error rate is %d'%(error/num_test_vec))
datingClassTest()

ext: py

reason: import numpy


text:
"""Send a numpy array roundtrip to Octave using an m-file.
"""
from oct2py import octave
import numpy as np

if __name__ == '__main__':
    x = np.array([[1, 2], [3, 4]], dtype=float)
    out, oclass = octave.roundtrip(x)
    import pprint
    pprint.pprint([x, x.dtype, out, oclass, out.dtype])


text:
'''
A stateful LSTM for text generation
Modification of https://github.com/fchollet/keras/blob/master/examples/lstm_text_generation.py
Example script to generate text from Nietzsche's writings.
At least 20 epochs are required before the generated text
starts sounding coherent.
It is recommended to run this script on GPU, as recurrent
networks are quite computationally intensive.
If you try this script on new data, make sure your corpus
has at least ~100k characters. ~1M is better.
'''

from __future__ import print_function

import random

import numpy as np
from keras.layers import Dense, Activation
from keras.layers import LSTM
from keras.models import Sequential
from keras.optimizers import RMSprop
from keras.utils import generic_utils

# Beginning of Sequence
BOS = '$'
# End Of SEQuence
EOSEQ = "$$$"

BATCH_SIZE = 1024

# Generators return EOS to indicate end of generation
def sentence_generator(lines):
    random.shuffle(lines)
    for line in lines:
        if len(line) > 5:
            yield line
    yield EOSEQ

def char_pair_generator(sentence):
    context_char = BOS
    for char in sentence:
        yield context_char, char
        context_char = char
    yield EOSEQ, EOSEQ

def get_char_dict(lines, max_chars=999999):
    assert (max_chars > 2)
    char_set = set([BOS])
    sentences = 0
    for line in sentence_generator(lines):
        char_set.update(line)
        sentences += 1
    char_indices = dict((c, i) for i, c in enumerate(char_set))
    indices_char = dict((i, c) for i, c in enumerate(char_set))
    return char_indices, indices_char, sentences

output = open("statefull_out.smi",'w')
lines = open("all_smiles.smi", 'r').readlines()

char_indices, indices_char, training_sentences = get_char_dict(lines)

print('total chars:', len(char_indices))
print('total sentences:', training_sentences)

# Train arrays
X = np.zeros((BATCH_SIZE, 1, len(char_indices)), dtype=np.bool)
Y = np.zeros((BATCH_SIZE, len(char_indices)), dtype=np.bool)

# build the model: a single LSTM
print('Build model...')
model = Sequential()
model.add(LSTM(128, batch_input_shape=(BATCH_SIZE, 1, len(char_indices)), stateful=True))
model.add(Dense(len(char_indices)))
model.add(Activation('softmax'))

optimizer = RMSprop(lr=0.01)
model.compile(loss='categorical_crossentropy', optimizer=optimizer)

def sample(preds, temperature=1.0):
    # helper function to sample an index from a probability array
    preds = np.asarray(preds).astype('float64')
    preds = np.log(preds) / temperature
    exp_preds = np.exp(preds)
    preds = exp_preds / np.sum(exp_preds)
    probas = np.random.multinomial(1, preds, 1)
    return np.argmax(probas)

# train the model, output generated text after each iteration
for iteration in range(1, 40):
    print()
    print('-' * 50)
    print('Iteration', iteration)
    processed_sentences = 0
    progbar = generic_utils.Progbar(training_sentences)

    # TRAINING
    sentence_gen = sentence_generator(lines)

    char_pair_generators = []
    for i in range(BATCH_SIZE):
        # possible bug when having less then batch size sentences (neglected)
        char_pair_generators.append(char_pair_generator(next(sentence_gen)))
        processed_sentences += 1

    finished_iteration = False
    losses = []

    while not finished_iteration:
        for i in range(BATCH_SIZE):
            context_char, char = next(char_pair_generators[i])

            # refill char pair generator
            if context_char == EOSEQ:
                sentence = next(sentence_gen)
                # the iteration finished, no new sentence can be retrieved
                if sentence == EOSEQ:
                    finished_iteration = True
                    break
                char_pair_generators[i] = char_pair_generator(sentence)
                context_char, char = next(char_pair_generators[i])
                processed_sentences += 1

                if processed_sentences % 100 == 0:
                    progbar.update(processed_sentences, values=[("loss", np.mean(losses))])
                    losses = []
            X[i, 0, char_indices[context_char]] = 1
            Y[i, char_indices[char]] = 1

        loss = model.train_on_batch(X, Y)
        losses.append(loss)
        X.fill(0)
        Y.fill(0)

    # SAMPLING
    diversities = [0.2, 0.5, 1.0, 1.2]
    generated = len(diversities) * [BOS]

    for i in range(4000):
        X.fill(0)
        for k in range(len(diversities)):
            X[k, 0, char_indices[generated[k][-1]]] = 1.

        preds = model.predict(X, batch_size=BATCH_SIZE, verbose=0)

        for k in range(len(diversities)):
            next_index = sample(preds[k], diversities[k])
            char = indices_char[next_index]
            generated[k] += char

    for k in range(len(diversities)):
        print()
        print("diversity:", diversities[k])
        print(generated[k][1:])
        output.write(generated[k][1:])
        print()

max_stars_repo_name: ArqiSoft/ml-services

max_issues_repo_name: ArqiSoft/ml-services

max_forks_repo_name: ArqiSoft/ml-services


text:
from legacypipe.survey import LegacySurveyData

class DecamSurvey(LegacySurveyData):
    def filter_ccd_kd_files(self, fns):
        return [fn for fn in fns if 'decam' in fn]
    def filter_ccds_files(self, fns):
        return [fn for fn in fns if 'decam' in fn]
    def filter_annotated_ccds_files(self, fns):
        return [fn for fn in fns if 'decam' in fn]
    def get_default_release(self):
        return 9008

class NinetyPrimeMosaic(LegacySurveyData):
    def filter_ccd_kd_files(self, fns):
        return [fn for fn in fns if ('90prime' in fn) or ('mosaic' in fn)]
    def filter_ccds_files(self, fns):
        return [fn for fn in fns if ('90prime' in fn) or ('mosaic' in fn)]
    def filter_annotated_ccds_files(self, fns):
        return [fn for fn in fns if ('90prime' in fn) or ('mosaic' in fn)]
    def get_default_release(self):
        return 9009

class M33SurveyData(DecamSurvey):
    def ccds_for_fitting(self, brick, ccds):
        import numpy as np
        from astrometry.libkd.spherematch import match_radec
        I, _, _ = match_radec(ccds.ra, ccds.dec, np.array(23.462121), np.array(30.659925), 0.55, nearest=True)
        #I = np.delete(I, np.where((ccds.filter[I] == 'g') * (ccds.expnum[I] != 661055))[0])
        #I = np.delete(I, np.where((ccds.filter[I] == 'z') * (ccds.expnum[I] != 790242))[0])
        return I

class OdinData(LegacySurveyData):
    #def filter_ccd_kd_files(self, fns):
    #    return [fn for fn in fns if ('90prime' in fn) or ('mosaic' in fn)]
    def filter_ccds_files(self, fns):
        return [fn for fn in fns if ('odin' in fn)]
    def filter_annotated_ccds_files(self, fns):
        return [fn for fn in fns if ('odin' in fn)]
    def get_default_release(self):
        return 200

class HscData(LegacySurveyData):
    #def filter_ccd_kd_files(self, fns):
    #    return [fn for fn in fns if ('90prime' in fn) or ('mosaic' in fn)]
    def filter_ccds_files(self, fns):
        return [fn for fn in fns if ('hsc' in fn)]
    def filter_annotated_ccds_files(self, fns):
        return [fn for fn in fns if ('hsc' in fn)]
    def get_default_release(self):
        return 200

runs = {
    'decam': DecamSurvey,
    '90prime-mosaic': NinetyPrimeMosaic,
    'south': DecamSurvey,
    'north': NinetyPrimeMosaic,
    'm33': M33SurveyData,
    'odin': OdinData,
    'hsc': HscData,
    None: LegacySurveyData,
}

def get_survey(name, **kwargs):
    survey_class = runs[name]
    survey = survey_class(**kwargs)
    return survey

max_stars_repo_name: legacysurvey/pipeline

max_issues_repo_name: legacysurvey/pipeline

max_forks_repo_name: legacysurvey/pipeline


text:
import numpy as np
import torchvision
from torch.utils.data import Dataset, DataLoader, ConcatDataset
from torchvision import transforms
import os
from PIL import Image
from collections import Counter

# Data transformation with augmentation
def get_data_transform(split, rgb_mean, rbg_std, image_size):
    data_transforms = {
        'train': transforms.Compose([
            transforms.RandomResizedCrop(size = image_size),
            transforms.RandomHorizontalFlip(),
            transforms.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4, hue=0),
            transforms.ToTensor(),
            transforms.Normalize(mean = rgb_mean, std=rbg_std)
        ]),
        'val': transforms.Compose([
            transforms.Resize(256),
            transforms.CenterCrop([image_size,image_size]),
            transforms.ToTensor(),
            transforms.Normalize(rgb_mean, rbg_std)
        ]),
        'test': transforms.Compose([
            transforms.Resize(256),
            transforms.CenterCrop([image_size,image_size]),
            transforms.ToTensor(),
            transforms.Normalize(rgb_mean, rbg_std)
        ])
    }
    return data_transforms[split]

# Dataset
class LT_Loader(Dataset):

    def __init__(self, root, ann_file, phase='train', image_size =224):
        self.img_path = []
        self.labels = []
        self.n_classes = 365
        mean =  [0.485, 0.456, 0.406]
        std = [0.229, 0.224, 0.225]
        self.transform = get_data_transform(phase, mean, std, image_size)
        with open(ann_file) as f:
            for line in f:
                self.img_path.append(os.path.join(root, line.split()[0]))
                self.labels.append(int(line.split()[1]))

    def __len__(self):
        return len(self.labels)

    def get_cls_num_list(self):
        cls_num_list = []
        cls_num_dict = Counter(self.labels)
        for k in range(self.n_classes):
            cls_num_list.append(cls_num_dict[k])
        return cls_num_list

    def get_balanced_weight(self):
        cls_num_list = self.get_cls_num_list()

        beta = (np.sum(cls_num_list)-1)/np.sum(cls_num_list)
        effective_num = 1.0 - np.power(beta, cls_num_list)
        per_cls_weights = (1.0 - beta) / np.array(effective_num)
        per_cls_weights = per_cls_weights / np.sum(per_cls_weights) * len(cls_num_list)
        return per_cls_weights

    def __getitem__(self, index):

        path = self.img_path[index]
        label = self.labels[index]

        sample = Image.open(path).convert('RGB')
        # import ipdb;ipdb.set_trace()
        # if self.transform is not None:
        sample = self.transform(sample)

        return sample, label

max_stars_repo_name: GT-RIPL/UNO-IC

max_issues_repo_name: GT-RIPL/UNO-IC

max_forks_repo_name: GT-RIPL/UNO-IC


text:
#!/usr/bin/python3

import itertools
import os
import sys

import numpy as np
from mp2 import *
from PIL import Image

np.set_printoptions(threshold=sys.maxsize)
np.set_printoptions(linewidth=1000)

script_dir = os.path.dirname(__file__)
images_dir = os.path.join(script_dir, "test_images")
images = sorted([os.path.join(images_dir, f) for f in os.listdir(images_dir)])

se_identity_1 = np.array([[True]])
se_identity_3 = np.array(
    [[False, False, False], [False, True, False], [False, False, False]]
)
se_identity_5 = np.array(
    [
        [False, False, False, False, False],
        [False, False, False, False, False],
        [False, False, True, False, False],
        [False, False, False, False, False],
        [False, False, False, False, False],
    ]
)
se_cross_3 = np.array([[False, True, False], [True, True, True], [False, True, False]])
se_north_3 = np.array(
    [[False, True, False], [False, True, False], [False, False, False]]
)

if __name__ == "__main__":
    print("Andrew Quinn - EECS 332 - MP#2, Testing Framework")
    print("=" * 80)

    for image in images:
        print(".-*-" * 20)
        print(".-*-" * 20)
        print(".-*-" * 20)
        print("\n\n")
        print(image)
        img_in = np.array(Image.open(image).convert("1"))
        print(img_in)
        print("")

        print("\n\n\n============ EROSION TEST ==============\n\n\n")
        print("SE 1x1 identity (should all be True):")
        print(img_in == erode(img_in, se_identity_1))
        print("SE 3x3 identity (should all be True):")
        print(img_in == erode(img_in, se_identity_3))
        print("SE 5x5 identity (should all be True):")
        print(img_in == erode(img_in, se_identity_5))

        print("SE 3x3 cross (only Trues which have True on all sides):")
        print("\n   original ... ")
        print(img_in)
        print("\n   cross erode ... ")
        print(erode(img_in, se_cross_3))

        print("SE 3x3 north (only Trues which have True above as well):")
        print("\n   original ... ")
        print(img_in)
        print("\n   north erode ... ")
        print(erode(img_in, se_north_3))

        print("\n\n\n============ DILATION TEST ==============\n\n\n")
        print("SE 1x1 identity (should all be True):")
        print(img_in == dilate(img_in, se_identity_1))
        print("SE 3x3 identity (should all be True):")
        print(img_in == dilate(img_in, se_identity_3))
        print("SE 5x5 identity (should all be True):")
        print(img_in == dilate(img_in, se_identity_5))

        print("SE 3x3 cross (only Trues which have True on any sides):")
        print("\n   original ... ")
        print(img_in)
        print("\n   cross dilate ... ")
        print(dilate(img_in, se_cross_3, verbose=False))

        print("SE 3x3 north (Trues may also have True above):")
        print("\n   original ... ")
        print(img_in)
        print("\n   north dilate ... ")
        print(dilate(img_in, se_north_3))

        print("\n\n\n============ OPENING TEST ==============\n\n\n")
        print("SE 1x1 identity (should all be True):")
        print(img_in == opening(img_in, se_identity_1))
        print("SE 3x3 identity (should all be True):")
        print(img_in == opening(img_in, se_identity_3))
        print("SE 5x5 identity (should all be True):")
        print(img_in == opening(img_in, se_identity_5))

        print("SE 3x3 cross (only Trues which have True on any sides):")
        print("\n   original ... ")
        print(img_in)
        print("\n   cross opening ... ")
        print(opening(img_in, se_cross_3, verbose=False))

        print("SE 3x3 north (Trues may also have True above):")
        print("\n   original ... ")
        print(img_in)
        print("\n   north opening ... ")
        print(opening(img_in, se_north_3))

        print("\n\n\n============ CLOSING TEST ==============\n\n\n")
        print("SE 1x1 identity (should all be True):")
        print(img_in == closing(img_in, se_identity_1))
        print("SE 3x3 identity (should all be True):")
        print(img_in == closing(img_in, se_identity_3))
        print("SE 5x5 identity (should all be True):")
        print(img_in == closing(img_in, se_identity_5))

        print("SE 3x3 cross (only Trues which have True on any sides):")
        print("\n   original ... ")
        print(img_in)
        print("\n   cross closing ... ")
        print(closing(img_in, se_cross_3, verbose=False))

        print("SE 3x3 north (Trues may also have True above):")
        print("\n   original ... ")
        print(img_in)
        print("\n   north closing ... ")
        print(closing(img_in, se_north_3))

    print("\n\n\n============ BOUNDARY TEST ==============\n\n\n")
    print("SE 1x1 identity (should all be True):")
    print(img_in == boundary(img_in, se_identity_1))
    print("SE 3x3 identity (should all be True):")
    print(img_in == boundary(img_in, se_identity_3))
    print("SE 5x5 identity (should all be True):")
    print(img_in == boundary(img_in, se_identity_5))

    print("SE 3x3 cross (only Trues which have True on any sides):")
    print("\n   original ... ")
    print(img_in)
    print("\n   cross closing ... ")
    print(boundary(img_in, se_cross_3, verbose=False))

    print("SE 3x3 north (Trues may also have True above):")
    print("\n   original ... ")
    print(img_in)
    print("\n   north boundary ... ")
    print(boundary(img_in, se_north_3))


text:
import base64
import cv2
import numpy as np
import scipy.spatial as spatial

def read_b64_img(b64):
    img = base64.b64decode(b64)
    npimg = np.fromstring(img, dtype=np.uint8)
    return cv2.imdecode(npimg, 1)

class BBox(object):
    def __init__(self, x1, y1, x2, y2):
        # (x1, y1) is the upper left corner,
        # (x2, y2) is the lower right corner,
        if x1 > x2: x1, x2 = x2, x1
        if y1 > y2: y1, y2 = y2, y1
        self.x1 = x1
        self.y1 = y1
        self.x2 = x2
        self.y2 = y2

    def taxicab_diagonal(self):
        return self.x2 - self.x1 + self.y2 - self.y1

    def overlaps(self, other):
        # Return True if self and other overlap.
        return not ((self.x1 > other.x2) or (self.x2 < other.x1) or (self.y1 > other.y2) or (self.y2 < other.y1))

def makeClassFromCont(contours):
    bboxes = []
    for c in contours:
        x,y,w,h = cv2.boundingRect(c)
        if w < 5 or h < 5:
            continue
        bboxes.append(BBox(x, y, x+w, y+h))
    return bboxes

def remove_overlaps(contours):
    #This function returns a set of bboxes after removing the overlapping contours
    bboxes = makeClassFromCont(contours)

    corners = []
    ulcorners = []

    # dict mapping corners to Bboxes.
    bbox_map = {}

    for bbox in bboxes:
        ul = (bbox.x1, bbox.y1)
        lr = (bbox.x2, bbox.y2)
        bbox_map[ul] = bbox
        bbox_map[lr] = bbox
        ulcorners.append(ul)
        corners.append(ul)
        corners.append(lr)

    try:
        tree = spatial.KDTree(np.asarray(corners))
    except Exception:
        return 0
    for corner in ulcorners:
        bbox = bbox_map[corner]
        # Find all points which are within a taxicab distance of corner
        indices = tree.query_ball_point(corner, bbox_map[corner].taxicab_diagonal(), p = 1)
        for near_corner in tree.data[indices]:
            near_bbox = bbox_map[tuple(near_corner)]
            if bbox != near_bbox and bbox.overlaps(near_bbox):
                # Expand both the bboxes
                bbox.x1 = near_bbox.x1 = min(bbox.x1, near_bbox.x1)
                bbox.y1 = near_bbox.y1 = min(bbox.y1, near_bbox.y1)
                bbox.x2 = near_bbox.x2 = max(bbox.x2, near_bbox.x2)
                bbox.y2 = near_bbox.y2 = max(bbox.y2, near_bbox.y2)
    return set(bbox_map.values())

max_stars_repo_name: kargaranamir/aim

max_issues_repo_name: kargaranamir/aim

max_forks_repo_name: kargaranamir/aim

reason: import numpy,import scipy


text:
from pathlib import Path

import tqdm
import numpy as np
import pandas as pd
from osgeo import gdal

AOI_NAMES = [
    'AOI_7_Moscow',
    'AOI_8_Mumbai',
    'AOI_9_San_Juan',
]

def get_coords(aoi_name, data_folder):
    rows = []

    image_dir = (Path(data_folder) / "PS-MS")
    n_images = len(list(image_dir.glob('./*.tif')))
    if n_images == 0:
        raise RuntimeError(f"Can't find GeoTIFF images: data_folder={data_folder}")

    for path in image_dir.glob('./*.tif'):
        imname = path.stem.split('_')[-1]  # e.g. "chip100"

        src = gdal.Open(str(path))
        ulx, xres, xskew, uly, yskew, yres = src.GetGeoTransform()
        lrx = ulx + (src.RasterXSize * xres)
        lry = uly + (src.RasterYSize * yres)
        rows.append({
            'aoi_name': aoi_name,
            'imname': imname,
            'mode': 'test',
            'lrx': lrx,
            'ulx': ulx,
            'lry': lry,
            'uly': uly,
        })

    return pd.DataFrame(rows)

def main(aoi_data_path_mapping):
    df_list = []
    for aoi_name in sorted(aoi_data_path_mapping.keys()):
        df = get_coords(aoi_name, aoi_data_path_mapping[aoi_name])
        if len(df) == 0:
            raise RuntimeError("Failed to parse AOI coordinates")

        df['ulx'] = df['ulx'] * 1000 * 1000
        df['lrx'] = df['lrx'] * 1000 * 1000
        df['uly'] = df['uly'] * 1000 * 1000
        df['lry'] = df['lry'] * 1000 * 1000

        xdiff = df.sort_values(by='ulx').ulx.drop_duplicates().diff().dropna().value_counts().index[0]
        xmin = df.sort_values(by='ulx').ulx.drop_duplicates().min()
        df['ix'] = ((df.ulx - xmin) / xdiff).apply(lambda x: round(x))
        assert df.groupby('ix').ulx.agg(['min', 'max', 'nunique'])['nunique'].max() == 1

        ydiff = df.sort_values(by='uly').uly.drop_duplicates().diff().dropna().value_counts().index[0]
        ymin = df.sort_values(by='uly').uly.drop_duplicates().min()
        df['iy'] = ((df.uly - ymin) / ydiff).apply(lambda x: round(x))
        df['iy'] = df['iy'].max() - df['iy']
        assert df.groupby('iy').uly.agg(['min', 'max', 'nunique'])['nunique'].max() == 1

        df_list.append(df)

    df = pd.concat(df_list, sort=False)
    # df.to_csv(out_chiplocations, index=False)
    return df

if __name__ == '__main__':
    # out_chiplocations = 'data/working/sn5/stat/AOI_all_bigmap_chip_locations.csv'
    df = main(aoi_data_path_mapping)
    assert len(df) > 0


text:
"""Create and load data to be used by the CNNs."""
import datetime
import glob
import logging
import os

import numpy as np
import xarray as xr

from wildfire import multiprocessing
from wildfire.data import goes_level_2

_logger = logging.getLogger(__name__)

def get_patch_indices(max_index, length, stride):
    """Compute path indices.

    Return indices such that each every the last range would have the full `length`
    (i.e. always returns `max_index - length` as the last element).

    Examples
    --------
    get_patch_indices(10, 2, 2) = [0, 2, 4, 6, 8]
    get_patch_indices(9, 2, 3) = [0, 3, 6, 7]
    get_patch_indices(7, 3, 3) = [0, 3, 4]

    Parameters
    ----------
    max_index :int
    length: int
    stride : int

    Returns
    -------
    list
    """
    if max_index < length:
        raise ValueError("Max index must be larger than stride")
    last_patch = max_index - length
    indices = np.arange(0, last_patch, stride)
    return np.append(indices, [last_patch])

def extract_patches_2d(arr, height, width, stride):
    """Extract 2d patches from array.

    if stride = width, then shape of the returned object will be:
        arr_height, arr_width, _ = arr.shape
        ceil(arr_heigh / height) * ceil(arr_width / width)

    Parameters
    ----------
    arr : array-like
    height : int
    width : int
    stride : int

    Returns
    -------
    array-like
    """
    max_height, max_width = arr.shape[:2]

    height_indices = get_patch_indices(max_index=max_height, length=height, stride=stride)
    width_indices = get_patch_indices(max_index=max_width, length=width, stride=stride)

    patches = []
    for height_idx in height_indices:
        for width_idx in width_indices:
            patches.append(
                arr[height_idx : height_idx + height, width_idx : width_idx + width]
            )
    return np.array(patches)

def process_file(
    level_2_filepath, level_1_directory, height, width, stride, persist_directory
):
    """Create training data from a GOES level 2 fire dataset.

    For a given GOES L2 fire product, find the accompanying GOES L1 data, and return the
    pixels with fire in it.

    Parameters
    ----------
    level_2_filepath : str
    level_1_directory : str
    height : int
    width : int
    stride : int

    Returns
    -------
    array-like
        shape: (num_fire_patches, height, width, 17)
    """
    _logger.info("Processing %s...", level_2_filepath)

    level_2 = xr.load_dataset(level_2_filepath)
    level_1 = goes_level_2.utilities.match_level_1(
        level_2=level_2, level_1_directory=level_1_directory
    )

    two_km_coords = {
        "x": level_1["band_16"].dataset.x.values,
        "y": level_1["band_16"].dataset.y.values,
    }

    level_1 = xr.concat(
        objs=[
            band.rescale_to_2km().normalize().assign_coords(**two_km_coords)
            for _, band in level_1.iteritems()
        ],
        dim="band",
    )

    # shape = (x, y, 17)
    data = np.concatenate(
        [level_1.values, np.expand_dims(level_2.Temp.values, axis=0)]
    ).transpose([1, 2, 0])

    # shape = (num_patches, height, width, 17)
    data = extract_patches_2d(arr=data, height=height, width=width, stride=stride)
    fire_indices = np.any(np.isfinite(data[:, :, :, -1]), axis=(1, 2))

    # shape = (num_fire_pathces, height, width, 17)
    data = data[fire_indices]

    now = datetime.datetime.utcnow().strftime("%Y%m%d%H%M%S")
    basename = os.path.basename(level_2_filepath)
    persist_filepath = os.path.join(persist_directory, f"cnn_training_c{now}_{basename}")

    data = xr.Dataset(
        {
            "abi": xr.DataArray(data[:, :, :, :-1].astype(np.float32)),
            "fire_temp": xr.DataArray(data[:, :, :, -1].astype(np.float32)),
        }
    )
    data.to_netcdf(persist_filepath)
    _logger.info("Saved training data to file: %s", persist_filepath)
    return data

def create_goes_level_2_training_data(
    level_2_directory,
    level_1_directory,
    persist_directory,
    height,
    width,
    stride,
    pbs=False,
    **cluster_kwargs,
):
    """Create GOES L2 training data for the CNN to predict wildfire presence.

    Parameters
    ----------
    level_2_directory : str
    level_1_directory : str
    persist_directory : str
    height : int
    width : int
    stride : int
    """
    goes_l2_filepaths = glob.glob(
        os.path.join(level_2_directory, "**", "*.nc"), recursive=True
    )

    num_filepaths = len(goes_l2_filepaths)
    _logger.info(
        "Creating training data from %d file for the DNN using %d processes...",
        num_filepaths,
        os.cpu_count(),
    )
    multiprocessing.map_function(
        function=process_file,
        function_args=[
            goes_l2_filepaths,
            [level_1_directory] * num_filepaths,
            [height] * num_filepaths,
            [width] * num_filepaths,
            [stride] * num_filepaths,
            [persist_directory] * num_filepaths,
        ],
        pbs=pbs,
        **cluster_kwargs,
    )
    _logger.info("Saved training data to directory: %s", persist_directory)

max_stars_repo_name: Ferrumofomega/goes

max_issues_repo_name: joyprojects/wildfire

max_forks_repo_name: Ferrumofomega/wildfire


text:
#!/usr/bin/env python

import rospy
import numpy as np
from std_msgs.msg import Float32MultiArray

x_y_theta_t = [[0, 0, 0, 0]]
r_wheel = 0.025
wheel_sep = 0.210

def calc_pos(msg):
    if (len(x_y_theta_t) == 1):
        x_y_theta_t.append([0, 0, 0, msg.data[2]])
    wl = msg.data[0]*np.pi/180
    wr = msg.data[1]*np.pi/180
    t_ard = msg.data[2]

    delta_theta = (-wl * r_wheel * (t_ard - x_y_theta_t[-1][3]) + wr * r_wheel * (t_ard - x_y_theta_t[-1][3])) / wheel_sep
    delta_s = (wl * r_wheel * (t_ard - x_y_theta_t[-1][3]) + wr * r_wheel * (t_ard - x_y_theta_t[-1][3])) * 0.5
    x = x_y_theta_t[-1][0] + delta_s * np.cos(x_y_theta_t[-1][2]*np.pi/180 + delta_theta * 0.5)
    y = x_y_theta_t[-1][1] + delta_s * np.sin(x_y_theta_t[-1][2]*np.pi/180 + delta_theta * 0.5)
    theta = x_y_theta_t[-1][2] + delta_theta*180/np.pi #FIX ME: theta needs to stay between -359 and +359 degrees

    x_y_theta_t.append([x, y, theta, t_ard])

def write_to_csv():
    np.savetxt('data.csv', x_y_theta_t, fmt="%1.3f", delimiter=',')

def init_node():
    rospy.init_node('arduino_data_collector', anonymous=True)  # anonymous=True appends a unique ID to the node name
    rospy.Subscriber("arduino_data", Float32MultiArray, calc_pos)
    rospy.on_shutdown(write_to_csv)
    rospy.spin()

if __name__ == '__main__':
    init_node()

max_stars_repo_path: code used for testing/odometry-calculation.py

max_stars_repo_name: ongaroleandro/autonomous-mobile-robot

max_issues_repo_name: ongaroleandro/autonomous-mobile-robot

max_forks_repo_name: ongaroleandro/autonomous-mobile-robot


text:
from model import common
import torch
import torch.nn as nn
import scipy.io as sio

# 创建模型
def make_model(args, parent=False):
    return BSR(args)

def matrix_init():
    a = torch.linspace(1, 15, steps=15) - 8
    a = a.float()

    mat_1 = torch.mul(a, a)
    mat_1, mat_3 = torch.meshgrid([mat_1, mat_1])
    a = a.view(15, 1)
    mat_2 = torch.mul(a, a.t())
    mat_1 = mat_1.contiguous().view(1, 1, 225, 1, 1)
    mat_2 = mat_2.contiguous().view(1, 1, 225, 1, 1)
    mat_3 = mat_3.contiguous().view(1, 1, 225, 1, 1)

    return torch.cat((mat_1, mat_2, mat_3), 1).cuda()

# BSR 模型
class BSR(nn.Module):
    def __init__(self, args, conv=common.default_conv):
        super(BSR, self).__init__()
        n_resblocks = args.n_resblocks
        n_feats = args.n_feats
        kernel_size = 3  # 卷积核大小
        self.scale_idx = 0

        act = nn.ReLU(True)  # 激活函数

        self.DWT = common.DWT()  # 二维离散小波
        self.IWT = common.IWT()  # 逆向的二维离散小波

        n = 3
        # downsample的第一层，维度变化4->16
        m_head = [common.BBlock(conv, 4, 160, 3, act=act)]
        d_l1 = []
        for _ in range(n):
            d_l1.append(common.BBlock(conv, 160, 160, 3, act=act))

        # downsample的第二层，维度变化640->256（默认的feature map == 64）
        d_l2 = [common.BBlock(conv, 640, n_feats * 4, 3, act=act)]
        for _ in range(n):
            d_l2.append(
                common.BBlock(conv, n_feats * 4, n_feats * 4, 3, act=act))

        # downsample的第三层，并与upsample进行连接，也是upsample的第三层
        # 维度变化1024->256，256->1024
        pro_l3 = [common.BBlock(conv, n_feats * 16, n_feats * 4, 3, act=act)]
        for _ in range(n * 2):
            pro_l3.append(
                common.BBlock(conv, n_feats * 4, n_feats * 4, 3, act=act))
        pro_l3.append(
            common.BBlock(conv, n_feats * 4, n_feats * 16, 3, act=act))

        # upsample的第二层，维度变化256->640
        i_l2 = []
        for _ in range(n):
            i_l2.append(
                common.BBlock(conv, n_feats * 4, n_feats * 4, 3, act=act))
        i_l2.append(common.BBlock(conv, n_feats * 4, 640, 3, act=act))

        # upsample的第一层，维度变化160->4
        i_l1 = []
        for _ in range(n):
            i_l1.append((common.BBlock(conv, 160, 160, 3, act=act)))
        m_tail = [conv(160, 4, 3)]

        # downsample的第一层
        self.head = nn.Sequential(*m_head)
        self.d_l1 = nn.Sequential(*d_l1)
        # downsample的第二层
        self.d_l2 = nn.Sequential(*d_l2)
        # 第三层连接层
        self.pro_l3 = nn.Sequential(*pro_l3)
        # upsample的第二层
        self.i_l2 = nn.Sequential(*i_l2)
        # upsample的第一层
        self.i_l1 = nn.Sequential(*i_l1)
        self.tail = nn.Sequential(*m_tail)

    def forward(self, x):
        # downsample的第一层
        x1 = self.d_l1(self.head(self.DWT(x)))
        # downsample的第二层
        x2 = self.d_l2(self.DWT(x1))
        # upsample的第三层，并且使用了short cut的结构，将对应的downsample的第二层
        # 加到upsample对应的层上。
        x_ = self.IWT(self.pro_l3(self.DWT(x2))) + x2
        # upsample的第二层
        x_ = self.IWT(self.i_l2(x_)) + x1
        # upsample的第一层
        x = self.IWT(self.tail(self.i_l1(x_))) + x
        # x = self.add_mean(x)
        return x

    def set_scale(self, scale_idx):
        self.scale_idx = scale_idx

max_stars_repo_name: yanglixiaoshen/SAP-Net

max_issues_repo_name: yanglixiaoshen/SAP-Net

max_forks_repo_name: yanglixiaoshen/SAP-Net

reason: import scipy


text:
import numpy as np
import pytest
from numpy.testing import assert_allclose

import astropy.units as u
from astropy.timeseries.periodograms.lombscargle import LombScargle
from astropy.timeseries.periodograms.lombscargle._statistics import (fap_single, inv_fap_single,
                                                                     METHODS)
from astropy.timeseries.periodograms.lombscargle.utils import convert_normalization, compute_chi2_ref  # noqa: E501

from astropy.utils.compat.optional_deps import HAS_SCIPY

METHOD_KWDS = dict(bootstrap={'n_bootstraps': 20, 'random_seed': 42})
NORMALIZATIONS = ['standard', 'psd', 'log', 'model']

def make_data(N=100, period=1, theta=[10, 2, 3], dy=1, rseed=0, units=False):
    """Generate some data for testing"""
    rng = np.random.default_rng(rseed)
    t = 5 * period * rng.random(N)
    omega = 2 * np.pi / period
    y = theta[0] + theta[1] * np.sin(omega * t) + theta[2] * np.cos(omega * t)
    dy = dy * (0.5 + rng.random(N))
    y += dy * rng.standard_normal(N)

    fmax = 5

    if units:
        return t * u.day, y * u.mag, dy * u.mag, fmax / u.day
    else:
        return t, y, dy, fmax

def null_data(N=1000, dy=1, rseed=0, units=False):
    """Generate null hypothesis data"""
    rng = np.random.default_rng(rseed)
    t = 100 * rng.random(N)
    dy = 0.5 * dy * (1 + rng.random(N))
    y = dy * rng.standard_normal(N)
    fmax = 40

    if units:
        return t * u.day, y * u.mag, dy * u.mag, fmax / u.day
    else:
        return t, y, dy, fmax

@pytest.mark.parametrize('normalization', NORMALIZATIONS)
@pytest.mark.parametrize('with_errors', [True, False])
@pytest.mark.parametrize('units', [False, True])
def test_distribution(normalization, with_errors, units):
    t, y, dy, fmax = null_data(units=units)

    if not with_errors:
        dy = None

    ls = LombScargle(t, y, dy, normalization=normalization)
    freq, power = ls.autopower(maximum_frequency=fmax)
    z = np.linspace(0, power.max(), 1000)

    # Test that pdf and cdf are consistent
    dz = z[1] - z[0]
    z_mid = z[:-1] + 0.5 * dz
    pdf = ls.distribution(z_mid)
    cdf = ls.distribution(z, cumulative=True)
    if isinstance(dz, u.Quantity):
        dz = dz.value
    assert_allclose(pdf, np.diff(cdf) / dz, rtol=1E-5, atol=1E-8)

    # psd normalization without specified errors produces bad results
    if not (normalization == 'psd' and not with_errors):
        # Test that observed power is distributed according to the theoretical pdf
        hist, bins = np.histogram(power, 30, density=True)
        midpoints = 0.5 * (bins[1:] + bins[:-1])
        pdf = ls.distribution(midpoints)
        assert_allclose(hist, pdf, rtol=0.05, atol=0.05 * pdf[0])

@pytest.mark.parametrize('N', [10, 100, 1000])
@pytest.mark.parametrize('normalization', NORMALIZATIONS)
def test_inverse_single(N, normalization):
    fap = np.linspace(0, 1, 11)

    z = inv_fap_single(fap, N, normalization)
    fap_out = fap_single(z, N, normalization)
    assert_allclose(fap, fap_out)

@pytest.mark.parametrize('normalization', NORMALIZATIONS)
@pytest.mark.parametrize('use_errs', [True, False])
@pytest.mark.parametrize('units', [False, True])
def test_inverse_bootstrap(normalization, use_errs, units):
    t, y, dy, fmax = null_data(units=units)
    if not use_errs:
        dy = None

    fap = np.linspace(0, 1, 11)
    method = 'bootstrap'
    method_kwds = METHOD_KWDS['bootstrap']

    ls = LombScargle(t, y, dy, normalization=normalization)

    z = ls.false_alarm_level(fap, maximum_frequency=fmax,
                             method=method, method_kwds=method_kwds)
    fap_out = ls.false_alarm_probability(z, maximum_frequency=fmax,
                                         method=method,
                                         method_kwds=method_kwds)

    # atol = 1 / n_bootstraps
    assert_allclose(fap, fap_out, atol=0.05)

@pytest.mark.parametrize('method', sorted(set(METHODS) - {'bootstrap'}))
@pytest.mark.parametrize('normalization', NORMALIZATIONS)
@pytest.mark.parametrize('use_errs', [True, False])
@pytest.mark.parametrize('N', [10, 100, 1000])
@pytest.mark.parametrize('units', [False, True])
def test_inverses(method, normalization, use_errs, N, units, T=5):
    if not HAS_SCIPY and method in ['baluev', 'davies']:
        pytest.skip("SciPy required")

    t, y, dy, fmax = make_data(N, rseed=543, units=units)
    if not use_errs:
        dy = None
    method_kwds = METHOD_KWDS.get(method, None)

    fap = np.logspace(-10, 0, 11)

    ls = LombScargle(t, y, dy, normalization=normalization)
    z = ls.false_alarm_level(fap, maximum_frequency=fmax,
                             method=method,
                             method_kwds=method_kwds)
    fap_out = ls.false_alarm_probability(z, maximum_frequency=fmax,
                                         method=method,
                                         method_kwds=method_kwds)
    assert_allclose(fap, fap_out)

@pytest.mark.parametrize('method', sorted(METHODS))
@pytest.mark.parametrize('normalization', NORMALIZATIONS)
@pytest.mark.parametrize('units', [False, True])
def test_false_alarm_smoketest(method, normalization, units):
    if not HAS_SCIPY and method in ['baluev', 'davies']:
        pytest.skip("SciPy required")

    kwds = METHOD_KWDS.get(method, None)
    t, y, dy, fmax = make_data(rseed=42, units=units)

    ls = LombScargle(t, y, dy, normalization=normalization)
    freq, power = ls.autopower(maximum_frequency=fmax)
    Z = np.linspace(power.min(), power.max(), 30)

    fap = ls.false_alarm_probability(Z, maximum_frequency=fmax,
                                     method=method, method_kwds=kwds)

    assert len(fap) == len(Z)
    if method != 'davies':
        assert np.all(fap <= 1)
        assert np.all(fap[:-1] >= fap[1:])  # monotonically decreasing

@pytest.mark.parametrize('method', sorted(METHODS))
@pytest.mark.parametrize('use_errs', [True, False])
@pytest.mark.parametrize('normalization', sorted(set(NORMALIZATIONS) - {'psd'}))
@pytest.mark.parametrize('units', [False, True])
def test_false_alarm_equivalence(method, normalization, use_errs, units):
    # Note: the PSD normalization is not equivalent to the others, in that it
    # depends on the absolute errors rather than relative errors. Because the
    # scaling contributes to the distribution, it cannot be converted directly
    # from any of the three normalized versions.
    if not HAS_SCIPY and method in ['baluev', 'davies']:
        pytest.skip("SciPy required")

    kwds = METHOD_KWDS.get(method, None)
    t, y, dy, fmax = make_data(units=units)
    if not use_errs:
        dy = None

    ls = LombScargle(t, y, dy, normalization=normalization)
    freq, power = ls.autopower(maximum_frequency=fmax)
    Z = np.linspace(power.min(), power.max(), 30)
    fap = ls.false_alarm_probability(Z, maximum_frequency=fmax,
                                     method=method, method_kwds=kwds)

    # Compute the equivalent Z values in the standard normalization
    # and check that the FAP is consistent
    Z_std = convert_normalization(Z, len(t),
                                  from_normalization=normalization,
                                  to_normalization='standard',
                                  chi2_ref=compute_chi2_ref(y, dy))
    ls = LombScargle(t, y, dy, normalization='standard')
    fap_std = ls.false_alarm_probability(Z_std, maximum_frequency=fmax,
                                         method=method, method_kwds=kwds)

    assert_allclose(fap, fap_std, rtol=0.1)

max_stars_repo_name: MatiasRepetto/astropy

max_issues_repo_name: MatiasRepetto/astropy

max_forks_repo_name: MatiasRepetto/astropy

reason: import numpy,from numpy,import astropy,from astropy


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Aug 17 15:38:02 2018

@author: bmoseley
"""
import numpy as np
from projectq import MainEngine
from projectq.ops import All, Measure, H, Rx

from helper import measureAllQubits

class VQE:
    ''' A VQE object for evaluating the expected energy of a parameterised trial state.
    The expected energy can either be estimated by carrying out repeated measurements of the state (slow)
    or by cheating and calculating it exactly using the hidden state amplitudes in the simulator (fast)
    '''

    def __init__(self, hamiltonian, Hamiltonian, Ansatz):
        self.hamiltonian = hamiltonian
        self.Hamiltonian = Hamiltonian
        self.Ansatz = Ansatz
        self.n_qubits = Ansatz.n_qubits

        # basic error check
        for hamiltonian_term in hamiltonian:
            if self.n_qubits != len(hamiltonian_term)-2:
                raise Exception("Error: n_qubits does not match length of hamiltonian term (%i, %s)"%(self.n_qubits, hamiltonian_term))

    def _subtermMeasurement(self, qubits, engine, hamiltonian_term):
        ''' Performs a single measurement of one of the Pauli
            matrix terms in the Hamiltonian.
        '''
        # Changes basis such that measuring Pauli term becomes
        # equivalent to measuring in the computational basis.
        maskArray = np.zeros(len(qubits))
        for i,qubit in enumerate(qubits):
            gate = hamiltonian_term[i+2]
            if gate == 'I':
                maskArray[i] = 0
            elif gate == 'X':
                H | qubit
                maskArray[i] = 1
            elif gate == 'Y':
                Rx(np.pi/2) | qubit
                maskArray[i] = 1
            else:
                maskArray[i] = 1
        result = measureAllQubits(qubits, engine)
        parityArray = maskArray*result# Calculate parity of relevant qubits in term
        paritySum = np.sum(parityArray)
        parityMeasurement = (-1)**paritySum

        return parityMeasurement

    def _runSubtermExpectation(self, parameters, n_repeats, hamiltonian_term):
        '''Find the expectation value of a single subterm in the Hamiltonian'''
        # TODO: this step could be distributed across many CPU nodes, but need to handle the random seed in MainEngine carefully
        n_qubits = self.n_qubits

        expectationSum = 0.
        for _ in range(n_repeats):
            engine = MainEngine()# Note each time this is called a new random seed is used by engine
            qubits = engine.allocate_qureg(n_qubits)
            self.Ansatz(parameters, qubits, engine)
            measuredValue = self._subtermMeasurement(qubits, engine, hamiltonian_term)
            expectationSum += measuredValue

        return expectationSum/n_repeats

    def runExpectation(self, parameters, n_repeats):
        ''' Estimates the expected energy of the state defined by the parameters by
            performing many state initialisations and measurements.
        '''
        hamiltonian = self.hamiltonian
        results = [self._runSubtermExpectation(parameters, n_repeats, hamiltonian_term) for hamiltonian_term in hamiltonian]
        energy = np.sum([results[i]*hamiltonian[i][1] for i in range(len(hamiltonian))])

        return energy

    def runCheatExpectation(self, parameters):
        ''' Uses ProjectQ cheat functions to quickly measure the exact energy
            expectation value.
        '''
        n_qubits = self.n_qubits
        Hamiltonian = self.Hamiltonian

        engine = MainEngine() # define engine
        qubits = engine.allocate_qureg(n_qubits)
        self.Ansatz(parameters, qubits, engine)

        engine.flush()# need to flush before accessing quantum results
        energy = engine.backend.get_expectation_value(Hamiltonian, qubits)

        All(Measure) | qubits# just to avoid deallocation error
        return energy

if __name__ == "__main__":

    # TEST

    import time
    from HamiltonianFile import HamiltonianFile
    from H2Ansatz import H2Ansatz

    # get hamiltonian
    Ham = HamiltonianFile('hamiltonians/H2at075.txt')
    Hamiltonian, hamiltonian = Ham.Hamiltonian, Ham.hamiltonian

    H2 = VQE(hamiltonian=hamiltonian, Hamiltonian=Hamiltonian, Ansatz=H2Ansatz)

    parameters = np.array([0.0, 0.0, 0.11483322638407246])

    start = time.time()
    energy = H2.runExpectation(parameters, n_repeats=1000)
    delta = time.time()-start
    print("runExpectation energy: %.7f (%.2f s  (%.0f min))"%(energy, delta, delta/60))

    start = time.time()
    energy = H2.runCheatExpectation(parameters)
    delta = time.time()-start
    print("runCheatExpectation energy: %.7f (%.2f s  (%.0f min))"%(energy, delta, delta/60))


text:
from numpy import ndarray, array, nonzero, array
import numpy as N
from pg import DB
from odincal.config import config

class db(DB):
    def __init__(self):
        DB.__init__(self, dbname=config.get('database', 'dbname'),
                    user=config.get('database', 'user'),
                    host=config.get('database', 'host'),
                    passwd=config.get('database', 'passwd'),
                    )

def freq(lofreq, skyfreq, LO):
    n = 896
    f = N.zeros(shape=(n,))
    seq = [1, 1, 1, -1, 1, 1, 1, -1, 1, -1, 1, 1, 1, -1, 1, 1]
    m = 0
    for adc in range(8):
        if seq[2 * adc]:
            k = seq[2 * adc] * 112
            df = 1.0e6 / seq[2 * adc]
            if seq[2 * adc + 1] < 0:
                df = -df
            for j in range(k):
                f[m + j] = LO[adc / 2] + j * df
            m += k
    fdata = N.zeros(shape=(n,))
    if skyfreq >= lofreq:
        for i in range(n):
            v = f[i]
            v = lofreq + v
            v /= 1.0e9
            fdata[i] = v
    else:
        for i in range(n):
            v = f[i]
            v = lofreq - v
            v /= 1.0e9
            fdata[i] = v
    return fdata

def decode_data(con, row):
    data = {
        'backend': [],
        'frontend': [],
        'version': [],
        'intmode': [],
        'sourcemode': [],
        'freqmode': [],
        'ssb_fq': [],
        'mean_spectra': [],
        'median_spectra': [],
        'channels': [],
        'skyfreq': [],
        'lofreq': [],
        'altitude_range': [],
        'hotload_range': [],
    }

    for item in data.keys():
        if item == 'mean_spectra' or item == 'median_spectra':
            data[item] = N.ndarray(shape=(row['channels'],), dtype='float64',
                                   buffer=row[item])
        elif item == 'ssb_fq' or item == 'altitude_range' or item == 'hotload_range':
            data[item] = N.array(row[item])
        else:
            data[item] = row[item]
    return data

class Fit_spectrum():
    def __init__(self, data, f):
        self.ssb_fq = data['ssb_fq']
        self.medianTb = data['median_spectra']
        self.intmode = data['intmode']
        self.backend = data['backend']
        self.data = data
        self.f = f

    def filter_spectrum(self, freq, spectrum):
        '''low pass filter'''
        # find indexes of the spectrum medianTb
        # that are "contaminated by atmospheric lines"
        # these indexes will not be used for the low pass fit

        # if medianTb spectrum change by more than 0.3 K between
        # five channels we flag this channel to be contaminated
        index = N.nonzero(N.abs(spectrum[5:-5] -
                                spectrum[10::]) > .3)[0] + 5
        # flag also the 15 closest channels on each side
        non_ok_index = []
        for ind in index:
            zr = 15
            non_ok_index.extend(
                N.array(range(N.max([0, ind - zr]), N.min([ind + zr, 112 * 2]))))

        # find indexes of "dead" channels
        non_ok = N.nonzero((spectrum == 0))[0]
        non_ok.shape = (non_ok.shape[0],)

        # combine indexes that are contamintated by lines or dead
        non_ok_index.extend(N.array(non_ok))
        non_ok_index = N.unique(N.array(non_ok_index))
        return non_ok_index

    def fit_spectrum(self):

        self.get_ssb_module_indexes()
        n = 112 * 2  # number of channels per module

        # first fit data from each ssb module independently
        A = []
        Err = []
        for ind, module in enumerate(self.module_indexes):

            # find indexes for the current module
            spectrum_indexes = range(ind * n, (ind + 1) * n)
            self.freq_indexes = range(module * n, (module + 1) * n)
            module_data = N.array(self.medianTb[spectrum_indexes])
            module_freq = N.array(self.f[self.freq_indexes])

            # sort data on frequency
            sort_freq = N.argsort(module_freq)

            # now apply a low pass filter on frequency sorted data
            non_ok_index = self.filter_spectrum(
                module_freq[sort_freq], module_data[sort_freq])

            # take care of that the filter was applied on sorted data
            if len(non_ok_index) > 0:
                non_ok_index = sort_freq[non_ok_index]
            ok_index = N.setdiff1d(range(n), non_ok_index)

            # now fit data (part 1)
            # fit the function y=a+b*sin(c*freq+d)
            # for a range of fixed c parameters
            a, err = self.fit_module_data(
                module_freq[ok_index], module_data[ok_index], module)
            A.append(a)
            Err.append(err)
        # now find out which c value that gave the least error
        # when considering data from all modules
        Err = N.array(Err)
        min_err_ind = N.argmin(N.sum(N.array(Err), 0))

        # now get the spectrum fit
        median_fit = []
        for ind, module in enumerate(self.module_indexes):
            spectrum_indexes = range(ind * n, (ind + 1) * n)
            freq_indexes = range(module * n, (module + 1) * n)
            a = A[ind][min_err_ind]
            Tbfit = a[0] + a[1] * N.sin(2 * N.pi / a[2] * (
                self.f[freq_indexes] - N.median(self.f[freq_indexes])) + a[3])
            find = N.nonzero(self.medianTb[spectrum_indexes] == 0)[0]
            Tbfit[find] = 0
            median_fit.extend(Tbfit)

        self.median_fit = N.array(median_fit)
        Tdata = {'median': self.medianTb,
                 'highaltfit': self.median_fit}
        print str(self.data['backend'] + ' ' + self.data['frontend'] + ' ' +
                  self.data['sourcemode'] + ' ' +
                  str(self.data['freqmode']) + ' ' +
                  str(self.data['intmode']) +
                  str(self.data['hotload_range']))
        return Tdata

    def plot_fit(self):

        import matplotlib.pyplot as plt
        plt.plot(self.freq, self.medianTb, '.r',
                 self.freq, self.median_fit, '.g')
        plt.title(self.data['backend'] + ' ' + self.data['frontend'] + ' ' +
                  self.data['sourcemode'] + ' ' +
                  str(self.data['freqmode']) + ' ' +
                  str(self.data['intmode']) +
                  str(self.data['hotload_range']))
        plt.legend(['original', 'fit'])
        plt.ylabel('Tb [K]')
        plt.xlabel('Frquency [GHz]')
        plt.ylim([-2, 2])
        plt.grid(True)
        plt.minorticks_on()
        plt.title
        plt.show()

    def fit_module_data(self, freq, spectrum, module):
        '''fit the function y=a+b*sin(c*freq+d), for a range
          of fixed c parameter, to spectrum'''

        problem = 0
        if len(spectrum) < 56:
            problem = 1
        if self.backend == 'AC1' and module == 0:
            problem = 1
        a = []  # an empty list of fit parameters
        err = []  # an empty list with errors for each fit

        # loop over a range of fixed 'c' parameters
        for ind in N.arange(0.01, 1, 0.001):
            if problem == 1:
                a.append([0, 0, 1, 0])
                err.append(9e4)
                continue
            ff = freq - N.median(self.f[self.freq_indexes])

            k1 = N.ones(len(freq))
            k2 = N.sin(2 * N.pi / ind * ff)
            k3 = N.cos(2 * N.pi / ind * ff)
            k = N.array([k1, k2, k3])
            k = k.transpose()
            m = N.linalg.lstsq(k, spectrum)[0]

            a0 = m[0]
            a1 = N.sqrt(m[1]**2 + m[2]**2)
            a2 = float(ind)
            a3 = N.arctan2(m[2], m[1])
            fa = a0 + a1 * N.sin(2 * N.pi / a2 * ff + a3)

            a.append([a0, a1, a2, a3])
            err.append(N.sum((fa - spectrum)**2))

        return a, err

    def get_ssb_module_indexes(self):
        n = 2 * 112
        if self.intmode == 511:
            module_indexes = range(0, 4)
            self.freq = self.f
        elif self.intmode == 1023:
            module_indexes = N.argsort(self.ssb_fq)[2:4]
            if module_indexes[0] > module_indexes[1]:
                module_indexes = [module_indexes[1], module_indexes[0]]
            freq_indexes = range(module_indexes[0] * n,
                                 (module_indexes[0] + 1) * n)
            freq_indexes.extend(range(module_indexes[1] * n,
                                      (module_indexes[1] + 1) * n))
            self.freq = self.f[freq_indexes]
        elif self.intmode == 2047:
            module_indexes = N.argsort(self.ssb_fq)[0:2]
            if module_indexes[0] > module_indexes[1]:
                module_indexes = [module_indexes[1], module_indexes[0]]
            freq_indexes = range(module_indexes[0] * n,
                                 (module_indexes[0] + 1) * n)
            freq_indexes.extend(range(module_indexes[1] * n,
                                      (module_indexes[1] + 1) * n))
            self.freq = self.f[freq_indexes]
        self.module_indexes = module_indexes

def main():
    '''fit data found in the ac_level1b_average table
       and import the fits to the ac_cal_level1c table'''

    con = db()
    version = 8
    temp = [version, '{80000,120000}']
    # find out for which modes we have data in the ac_level1b_average table
    query = con.query(''' select backend,frontend,
                  version,intmode,
                  sourcemode,freqmode,ssb_fq,hotload_range,
                  altitude_range,median_spectra,mean_spectra,
                  skyfreq,lofreq,channels
                  from ac_level1b_average
                  where version={0} and altitude_range='{1}'
                  order by backend,sourcemode,freqmode,
                  frontend,ssb_fq,hotload_range
                  '''.format(*temp))
    result = query.dictresult()

    # fit data for each mode and import to table ac_cal_level1c
    for row in result:
        data = decode_data(con, row)
        f = freq(data['lofreq'], data['skyfreq'],
                 data['ssb_fq'] * 1e6)

        f = Fit_spectrum(data, f)
        Tdata = f.fit_spectrum()
        if 0:
            f.plot_fit()

        if 1:
            temp = {
                'backend': row['backend'],
                'frontend': row['frontend'],
                'version': row['version'],
                'intmode': row['intmode'],
                'sourcemode': row['sourcemode'],
                'freqmode': row['freqmode'],
                'ssb_fq': row['ssb_fq'],
                'hotload_range': row['hotload_range'],
                'altitude_range': row['altitude_range'],
                'median_spectra': Tdata['median'].tostring(),
                'median_fit': Tdata['highaltfit'].tostring(),
                'channels': len(Tdata['median']),
            }

            tempkeys = [temp['backend'], temp['frontend'], temp['version'],
                        temp['intmode'], temp['sourcemode'], temp['freqmode'],
                        temp['ssb_fq'], temp['hotload_range'],
                        temp['altitude_range']]
            con.query('''delete from ac_cal_level1c
                     where backend='{0}' and frontend='{1}'
                     and version={2} and intmode={3}
                     and sourcemode='{4}' and freqmode={5}
                     and ssb_fq='{6}' and
                     hotload_range='{7}' and
                     altitude_range='{8}' '''.format(*tempkeys))
            con.insert('ac_cal_level1c', temp)
        else:
            pass

    con.close()

max_stars_repo_name: Odin-SMR/odincal

max_issues_repo_name: Odin-SMR/odincal

max_forks_repo_name: Odin-SMR/odincal

reason: import numpy,from numpy


text:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

dataset = pd.read_csv('Salary_Data.csv')
X = dataset.iloc[:, :-1].values
y = dataset.iloc[:, 1].values

from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 1/3, random_state = 0)

'''from sklearn.preprocessing import MinMaxScaler
sc_X = MinMaxScaler(feature_range = (0,1))
X_train = sc_X.fit_transform(X_train)
X_test = sc_X.transform(X_test)
sc_y = MinMaxScaler(feature_range = (0, 1))
y_train = sc_y.fit_transform(y_train)
y_test = sc_y.transform(y_test)'''

from sklearn.linear_model import LinearRegression
regressor = LinearRegression()
regressor.fit(X_train, y_train)

y_pred = regressor.predict(X_test)

plt.scatter(X_train, y_train, color = 'red')
plt.plot(X_train, regressor.predict(X_train), color = 'blue')
plt.title('Salary vs Experience(Training Set)')
plt.xlabel('Years of Experience')
plt.ylabel('Salary')
plt.legend()
plt.savefig('SimpleLinearRegression.png')
plt.show()

plt.scatter(X_test, y_test, color = 'red')
plt.plot(X_train, regressor.predict(X_train), color = 'blue')
plt.title('Salary vs Experience(Test Set)')
plt.xlabel('Years of Experience')
plt.ylabel('Salary')
plt.legend()
plt.savefig('SimpleLinearRegression_Test.png')
plt.show()

max_stars_repo_path: Machine Learning A-Z/Part 2 - Regression/Section 4 - Simple Linear Regression/my_simple_regression.py


text:
"""
`utils.py`
----------

Module that should process a given project and
build the index of relevant source code files.

@author: Dylan Bourgeois (@dtsbourg)

License: CC-BY 4.0
"""

import pickle
import os
import numpy as np
import astor
import json

import project_crawler

def save(ast, filename, format='pickle'):
    directory = os.path.dirname(filename)
    if not os.path.exists(directory):
        os.makedirs(directory)

    if format=='pickle':
        with open(filename, 'wb') as handle:
            pickle.dump(ast, handle, protocol=pickle.HIGHEST_PROTOCOL)
    elif format=='txt':
        with open(filename, 'w') as handle:
            handle.write(ast)

def load(filename, format='pickle'):
    if format == 'pickle':
        try:
            with open(filename, 'rb') as handle:
                return pickle.load(handle)
        except:
            print("[ERROR] Could not open file", filename)

    elif format == 'txt':
        print("[LOAD] Parsing file ...")
    else:
        print("[ERROR] Please specify a valid format.")
        return None

def load_asts(path, verbose=False):
    try:
        asts = project_crawler.crawl(path, filetype='.ast')
        all_asts = []

        for idx, ast in enumerate(asts):
            loaded = load(ast)
            all_asts.append(loaded)
            if verbose:
                print("\r[LOAD]  --- Loading parsed AST {0}/{1} ...".format(idx+1,len(asts)), end='\r')
        print()
        return all_asts
    except:
        print("[ERROR] Could not load pre-generated ASTs from", path, ".")
        print("[ERROR] Please run again with the --preprocess flag enabled.")

def one_hot_encoder(x, size, max=None, min=None):
    x = np.array(x)
    if max is None:
        max = np.max(x)
    if min is None:
        min = np.min(x)
    n_types =  max + 1 - min
    x_one_hot = np.zeros((size, n_types))
    x_one_hot[np.arange(size), x] = 1
    return x_one_hot

def parse_file(filename, verbose=False):
    return astor.code_to_ast.parse_file(filename)

def save_json(obj, save_dir, filename):
    with open(os.path.join(save_dir, filename), 'w') as fout:
        fout.write(json.dumps(obj))
        print("[AST]  --- Saved", fout.name)

def invert_var_map(var_map):
    inv_var_map = {}
    for node_id, var_name in var_map.items():
        if inv_var_map.get(var_name, None) is None:
            inv_var_map[var_name] = [node_id]
        else:
            inv_var_map[var_name].append(node_id)
    return inv_var_map

def create_dir(d):
    if not os.path.exists(d):
        os.makedirs(d)

max_stars_repo_name: dtsbourg/codegraph-fmt

max_issues_repo_name: dtsbourg/SAGE-fmt

max_forks_repo_name: dtsbourg/SAGE-fmt


text:
import unittest
import random
import numpy as np
import torch
from networks.softmax_matcher import SoftmaxMatcher

class TestMatching(unittest.TestCase):
    def test_basic(self):
        B = 2
        N = 1024
        C = 248
        H = 512
        W = 512

        src_scores = torch.ones(B, 1, N)
        src_desc = torch.randn(B, C, N)
        tgt_scores_dense = torch.ones(B, 1, H, W)
        tgt_desc_dense = torch.randn(B, C, H, W)

        # "Hide" the src descriptors within the dense target descriptor map
        L = list(range(0, H * W))
        random.seed(0)
        random.shuffle(L)
        L = L[:N]
        row_col = [(index % H, index // H) for index in L]
        for i in range(N):
            x = src_desc[0, :, i]
            x = x.reshape(1, C)
            tgt_desc_dense[1, :, row_col[i][0], row_col[i][1]] = x
        # Now we have known correspondences: i <--> row_col

        config = {'networks': {'matcher_block': {'softmax_temp': 0.01, 'score_comp': True}}}
        config['window_size'] = 2
        config['gpuid'] = "cpu"
        model = SoftmaxMatcher(config)
        model.eval()

        pseudo_coords, _, _ = model.forward(src_scores, src_desc, tgt_scores_dense, tgt_desc_dense)

        # Now we can check whether the pseudo coordinates are the same as our known correspondences:
        outliers = 0
        outlier_threshold = 1e-4
        for i in range(N):
            dx = pseudo_coords[0, i, 0].item() - row_col[i][1]
            dy = pseudo_coords[0, i, 1].item() - row_col[i][0]
            d = np.sqrt(dx**2 + dy**2)
            if d > outlier_threshold:
                outliers += 1
        print('outliers: {}'.format(outliers))

        self.assertTrue(outliers / N < 0.001)

if __name__ == '__main__':
    unittest.main()


text:
from psydac.feec.derivatives import Grad, Curl, Div
from psydac.fem.tensor       import TensorFemSpace
from psydac.fem.splines      import SplineSpace
from psydac.linalg.stencil   import StencilVector
from psydac.linalg.block     import BlockVector,ProductSpace
from psydac.fem.vector       import ProductFemSpace
from psydac.feec.utilities   import Interpolation, interpolation_matrices
import numpy as np
from mpi4py import MPI
comm = MPI.COMM_WORLD

p = 3
grid_1 = np.linspace(0., 1., 6)
grid_2 = np.linspace(0., 1., 6)

# ... first component
V1 = SplineSpace(p, grid=grid_2)
V2 = SplineSpace(p, grid=grid_2)

H1    = TensorFemSpace(V1, V2, comm=comm)
Hcurl = ProductFemSpace(H1.reduce_degree([0]), H1.reduce_degree([1]))
L2    = H1.reduce_degree([0,1])

Int = Interpolation(H1=H1, Hcurl=Hcurl, L2=L2)
M0, M1, M2 = interpolation_matrices(H1)
# ... H1
f = lambda x,y: x*(1.-x)*y*(1.-y)
F = Int('H1', f)

# ... Hcurl
g0 = lambda x,y: (1.-2.*x)*y*(1.-y)
g1 = lambda x,y: x*(1.-x)*(1.-2.*y)
G = Int('Hcurl', [g0, g1])

F =  M0.solve(F)
G =  M1.solve(G)

grad = Grad(H1, Hcurl.vector_space)
curl = Curl(H1, Hcurl.vector_space, L2.vector_space)

x    = BlockVector(ProductSpace(H1.vector_space))
y    = BlockVector(Hcurl.vector_space)

x[0]._data = F._data
x = grad(x).toarray()
y = np.concatenate(tuple(g.toarray() for g in G))
print(np.abs(x-y).max())

max_stars_repo_name: GabrielJie/psydac

max_issues_repo_name: GabrielJie/psydac

max_forks_repo_name: GabrielJie/psydac


text:
import argparse

import numpy as np
import pandas as pd
from alibi_detect.od import Mahalanobis
from alibi_detect.utils.data import create_outlier_batch
from alibi_detect.utils.saving import load_detector

np.random.seed(112)

parser = argparse.ArgumentParser()
parser.add_argument('data_file_path', type=str)
parser.add_argument('outlier_detector_path', type=str)
args = parser.parse_args()

df = pd.read_csv(args.data_file_path, index_col=0)
X = df.drop(columns='target').values

od = load_detector(args.outlier_detector_path)
print(od)

preds = od.predict(
    X,
    return_instance_score=True
)

df_detections = pd.DataFrame(preds['data'])
perc_outl = df_detections[df_detections['is_outlier'] == 1]['is_outlier'].count() / df_detections['is_outlier'].count() * 100
print("% outliers: {}".format(perc_outl))

max_stars_repo_name: winderai/pachyderm-seldon

max_issues_repo_name: winderresearch/pachyderm-seldon

max_forks_repo_name: winderresearch/pachyderm-seldon


text:
import os
import time
import numpy as np
import pandas as pd
import tables as tb

def date_to_timestamp_ms(date_obj):
    timestamp_in_nanoseconds = date_obj.astype('int64')
    timestamp_in_ms = (timestamp_in_nanoseconds / 1000000).astype('int64')
    return timestamp_in_ms

def latitude_longitude_to_meters(lat, lng):
    # https://github.com/bokeh/datashader/blob/master/examples/taxi_preprocessing_example.py
    radius_of_earth = 6378137  # in meters
    origin_shift = 2 * np.pi * radius_of_earth / 2.0
    mx = lng * origin_shift / 180.0
    my = np.log(np.tan((90 + lat) * np.pi / 360.0)) / (np.pi / 180.0)
    my = my * origin_shift / 180.0
    return mx, my

def fill_table(table, mapping, df):
    num_records = df.shape[0]  # it's equal to the chunksize used in read_csv
    row = table.row
    for i in range(num_records):
        row['vendor_id'] = df[mapping['vendor_id']].values[i]

        pickup_ms = date_to_timestamp_ms(df[mapping['pickup_datetime']].values[i])
        row['pickup_timestamp_ms'] = pickup_ms
        dropoff_ms = date_to_timestamp_ms(df[mapping['dropoff_datetime']].values[i])
        row['dropoff_timestamp_ms'] = dropoff_ms

        row['passenger_count'] = df['passenger_count'].values[i]
        row['trip_distance'] = df['trip_distance'].values[i]

        row['pickup_longitude'] = df['pickup_longitude'].values[i]
        row['pickup_latitude'] = df['pickup_latitude'].values[i]
        row['dropoff_longitude'] = df['dropoff_longitude'].values[i]
        row['dropoff_latitude'] = df['dropoff_latitude'].values[i]

        row['pickup_x'], row['pickup_y'] = latitude_longitude_to_meters(
            df['pickup_latitude'].values[i], df['pickup_longitude'].values[i])
        row['dropoff_x'], row['dropoff_y'] = latitude_longitude_to_meters(
            df['dropoff_latitude'].values[i], df['dropoff_longitude'].values[i])

        row['fare_amount'] = df['fare_amount'].values[i]
        row['tip_amount'] = df['tip_amount'].values[i]
        row['total_amount'] = df['total_amount'].values[i]

        row['payment_type'] = df['payment_type'].values[i]
        row.append()
    table.flush()

def get_year_and_month(file_name):
    # we could also use a regex like '[0-9]){4}-([0-9]){2}\.csv'
    a, b, c = file_name.split('_')
    period, extension = c.split('.')
    year, month = period.split('-')
    return year, month

def get_csv_mapping(year, month):
    """CSV files from different year/month have some different field names.

    Parameters
    ----------
    year : int
    month : int

    Returns
    -------
    mapping : dict

    See Also
    --------
    data dictionary for NY yellow taxi CSV files
    http://www.nyc.gov/html/tlc/downloads/pdf/data_dictionary_trip_records_yellow.pdf
    """
    # To the left, the key we want to use. To the right, the key in the CSV file
    if year == '2014':
        mapping = {
            'vendor_id': 'vendor_id',
            'pickup_datetime': 'pickup_datetime',
            'dropoff_datetime': 'dropoff_datetime',
            # 'rate_code_id': 'rate_code',
        }
    elif year == '2015' or (year == '2016' and 1 <= int(month) <= 6):
        mapping = {
            'vendor_id': 'VendorID',
            'pickup_datetime': 'tpep_pickup_datetime',
            'dropoff_datetime': 'tpep_dropoff_datetime',
            # sometimes it's RateCodeID, others RatecodeID...
            # 'rate_code_id': 'RateCodeID',
        }
    else:
        raise NotImplementedError('data dictionary not defined for this period')

    return mapping

def main():
    here = os.path.abspath(os.path.dirname(__file__))
    data_dir = os.path.abspath(os.path.join(here, '..', 'data'))
    h5_file_path = os.path.join(data_dir, 'NYC-yellow-taxis-10k.h5')
    nyc_dir = os.path.join(data_dir, 'nyctaxi')
    years = os.listdir(nyc_dir)

    # Open the HDF5 file in 'a'ppend mode and populate the table with CSV data
    with tb.open_file(filename=h5_file_path, mode='a') as f:
        for year in years:
            year_dir = os.path.join(data_dir, 'nyctaxi', year)
            csv_files = os.listdir(year_dir)

            for csv_file in csv_files:
                year, month = get_year_and_month(csv_file)
                mapping = get_csv_mapping(year, month)
                # define the dtype to use when reading the CSV with pandas (this
                # has nothing to do with the HDF5 table)
                dtype = {
                    mapping['vendor_id']: 'category',
                    'store_and_fwd_flag': 'category',
                    'payment_type': 'category',
                }
                parse_dates = [
                    mapping['pickup_datetime'], mapping['dropoff_datetime']
                ]

                table_where = '/yellow_{}_{}'.format(year, month)
                table = f.get_node(where=table_where)

                t0 = time.time()
                print('Processing {} started'.format(csv_file))
                csv_file_path = os.path.join(year_dir, csv_file)

                # read in chunks because these CSV files are too big
                chunksize = 10000
                for chunk in pd.read_csv(
                    csv_file_path, chunksize=chunksize, dtype=dtype,
                    skipinitialspace=True, parse_dates=parse_dates):
                    df = chunk.reset_index(drop=True)
                    # print(df.describe())
                    # print(df.head())
                    fill_table(table, mapping, df)
                    # debugging tip: put a break here, so you can inspect what
                    # has been written in the table
                    break

                t1 = time.time()
                print('Processing {} took {:.2f}s'.format(csv_file, (t1 - t0)))

if __name__ == '__main__':
    main()


text:
"""
Loading scripts for HF type datasets
"""
import os
import sys
from typing import Optional, Union

from datasets import (Dataset, Features, IterableDataset, Value,
                      interleave_datasets, load_dataset)
from wasabi import msg

def load_tweets(dedupe=False):
    cwd = os.getcwd()
    f_path = os.path.dirname(os.path.abspath(__file__))
    data_path = os.path.join(f_path, "..", "data")
    sys.path.append(data_path)
    from dedupe import min_hash_deduper, duplicate_filter
    from utils import to_datetime

    os.chdir(data_path)

    dataset = load_dataset("HopeTweet", streaming=True)
    ds = dataset["train"]

    ds = ds.map(to_datetime)
    if dedupe:
        ds = ds.map(min_hash_deduper, batched=True, batch_size=50_000)
        ds = ds.map(duplicate_filter, batched=True, batch_size=50_000)
    os.chdir(cwd)
    return ds

def load_news():
    cwd = os.getcwd()
    f_path = os.path.dirname(os.path.abspath(__file__))
    data_path = os.path.join(f_path, "..", "data")
    os.chdir(data_path)
    dataset = load_dataset("DaNews", streaming=True)
    ds = dataset["train"]
    os.chdir(cwd)

    def format_news(example: dict) -> dict:
        example["text"] = ""
        if example["heading"].strip(" "):
            example["text"] = example["heading"]
        if example["subheading"].strip(" "):
            example["text"] += f'\n {example["subheading"]}'
        if example["publishdate"].strip(" "):
            example["text"] += f'\n {example["publishdate"]}'
        if example["publishdate"].strip(" "):
            example["text"] += f'\n\n {example["paragraph"]}'
        if example["body"].strip(" "):
            example["text"] += f'\n\n {example["body"]}'
        return example

    ds = ds.map(format_news)

    return ds

def load_dagw(filter_danavis: bool = True, streaming: bool = False):
    dataset = load_dataset(
        "DDSC/partial-danish-gigaword-no-twitter", streaming=streaming
    )
    ds = dataset["train"]
    if filter_danavis:

        def filter_(examples):
            i = 0
            while i < len(examples["source"]):
                s = examples["source"][i]
                if s == "danavis":
                    for k in examples:
                        examples[k].pop(i)
                else:
                    i += 1
            return examples

        # not possible to use filter with a streamed dataset
        ds = ds.map(
            filter_,
            batched=True,
        )
    return ds

def load_reddit(streaming=False):
    dataset = load_dataset("DDSC/reddit-da", streaming=streaming)
    ds = dataset["train"]
    return ds

def load_lexdk(streaming: bool = False) -> Union[Dataset, IterableDataset]:
    """Load the Lex.dk dataset as a Hugging Face Dataset object.

    Args:
        streaming (bool, optional):
            Whether to stream the dataset. Defaults to False.

    Returns:
        Dataset or IterableDataset: The loaded Hugging Face dataset.
    """
    features = Features(
        dict(
            url=Value("string"),
            title=Value("string"),
            clarification=Value("string"),
            authors=[Value("string")],
            date=Value("string"),
            text=Value("string"),
        )
    )
    return load_dataset(
        path="dfm/data/lexdk", features=features, streaming=streaming, split="train"
    )

def load_tokenizer_ds():
    """
    script used for training the tokenizer. Load a balances set of data to train the tokenizer on.
    """

    def word_count(example):
        example["n_words"] = len(list(filter(lambda x: x, example["text"].split(" "))))
        return example

    import numpy as np

    tweets = load_tweets(dedupe=False)
    avg_words_pr_tweet = np.mean(
        [a["n_words"] for a in list(tweets.map(word_count).take(100_000))]
    )
    tweet_n_words = 300_000_000
    n_tweets = tweet_n_words // avg_words_pr_tweet

    news = load_news()

    avg_words_pr_article = np.mean(
        [a["n_words"] for a in list(news.map(word_count).take(100_000))]
    )
    news_n_words = 300_000_000
    n_news = news_n_words // avg_words_pr_article

    reddit = load_reddit(streaming=False)
    dagw = load_dagw(streaming=False)
    n_reddit = len(reddit)  # 1 908 887
    n_dagw = len(dagw)  # 666 437

    reddit = load_reddit(streaming=True)  # use all tokens ~86M
    dagw = load_dagw(
        streaming=True
    )  # use all tokens (-danavis, -twitter) ~1 000M tokens

    # calculate proportion of each dataset
    arr = np.array([n_tweets, n_news, n_dagw, n_reddit])
    prob = arr / sum(arr)

    ds = interleave_datasets([tweets, news, dagw, reddit], probabilities=prob.tolist())
    n = 10_000_000
    msg.info(f"Limiting dataset to {n} samples.")
    ds = ds.take(n)
    return ds

def load_dfm_dataset(dataset: str, **kwargs) -> Dataset:
    """load a predefined dfm dataset

    Args:
        dataset (str): A predefined dataset

    Returns:
        Dataset: a Huggingface dataset
    """
    dataset_loaders = {
        "tokenization": load_tokenizer_ds,
        "hopetweets": load_tweets,
        "reddit": load_reddit,
        "danews": load_news,
        "dagw": load_dagw,
        "lexdk": load_lexdk,
    }

    if dataset in dataset_loaders:
        return dataset_loaders[dataset.lower()](**kwargs)
    else:
        raise ValueError(
            f"invalid dataset. Valid datasets include {dataset_loaders.keys()}"
        )

max_stars_repo_name: martbern/danish-foundation-models

max_issues_repo_name: martbern/danish-foundation-models

max_forks_repo_name: martbern/danish-foundation-models


text:
from PIL import Image
import pyautogui
import numpy as np

# =====坐标=====
# 屏幕宽度,高度

win_width, win_height = pyautogui.size()
# 图片宽度,高度
img_width, img_height = (1200, 640)
# 基准xy(偏移量 )
base_xy = ((win_width - img_width) / 2, 103)
# 图片区域
img_region = (base_xy[0], base_xy[1], img_width, img_height)

img = pyautogui.screenshot('screen.png', img_region)

ar = np.matrix([
    [0, 1], [1, 0]
])

print(ar)
print(ar.I)


text:
import os

import numpy as np
import pybullet as p

from assistive_gym.envs.agents.agent import Agent
from assistive_gym.envs.env import AssistiveEnv

CONTAINER_Y_POSITIONS = [3.6, 4, 4.4]
CONTAINER_COLORS = ["RED", "GREEN", "BLUE"]
COLOR_VALUES = dict(RED=[1, 0, 0, 1], GREEN=[0, 1, 0, 1], BLUE=[0, 0, 1, 1])

def create_hydration_container(env: AssistiveEnv, pos: list, color: list) -> Agent:
    visual_filename = os.path.join(
        env.directory, "dinnerware", "plastic_coffee_cup.obj"
    )
    collision_filename = os.path.join(
        env.directory, "dinnerware", "plastic_coffee_cup_vhacd.obj"
    )

    mass = 0.1
    mesh_scale = [0.05] * 3

    orient = [0, 0, 0, 1]

    tool_visual = p.createVisualShape(
        shapeType=p.GEOM_MESH,
        fileName=visual_filename,
        meshScale=mesh_scale,
        rgbaColor=color,
        physicsClientId=env.id,
    )
    tool_collision = p.createCollisionShape(
        shapeType=p.GEOM_MESH,
        fileName=collision_filename,
        meshScale=mesh_scale,
        physicsClientId=env.id,
    )
    tool = p.createMultiBody(
        baseMass=mass,
        baseCollisionShapeIndex=tool_collision,
        baseVisualShapeIndex=tool_visual,
        basePosition=pos,
        baseOrientation=orient,
        useMaximalCoordinates=False,
        physicsClientId=env.id,
    )

    tool_agent = Agent()
    tool_agent.init(tool, env.id, env.np_random, indices=-1)
    return tool_agent

def setup_hydration_staging(env: AssistiveEnv):
    positions = np.array(CONTAINER_Y_POSITIONS)
    colors = np.array(CONTAINER_COLORS)

    env.np_random.shuffle(colors)
    env.np_random.shuffle(positions)

    for y, color in zip(positions.tolist(), colors.tolist()):
        pos = [-3.8, y, 0.9]
        col = COLOR_VALUES[color]
        create_hydration_container(env, pos, col)

    print(env.np_random.choice(colors))


text:
import os
import sys
import pandas as pd
import numpy as np
import openrouteservice as ors
import folium
import geopandas as gpd
import networkx as nx
import osmnx as ox
import geojson
from tqdm import tqdm, trange
import random as rn
from sklearn.metrics.pairwise import haversine_distances
from math import radians

ox.config(use_cache=True,
          log_file=True,
          log_console=False,
          log_filename='download-lisboa',
          cache_folder='cache')

print(ox.__version__)
print(nx.__version__)

ors_profiles = [
    'driving-car',
    'driving-hgv',
    'foot-walking',
    'foot-hiking',
    'cycling-regular',
    'cycling-road',
    'cycling-mountain',
    'cycling-electric']

color_dict = {
    'driving-car': '#F0D954',
    'driving-hgv': '#854F21',
    'foot-walking': '#328BCF',
    'foot-hiking': '#328BCF',
    'cycling-regular': '#2E8533',
    'cycling-road': '#2E8533',
    'cycling-mountain': '#2E8533',
    'cycling-electric': '#2E8533'
}

class ORS(object):
    def __init__(self, server_addr=None, ors_key=None, profiles=ors_profiles):
        self.profiles = profiles
        if server_addr is None:
            self.server = ors.Client(key=ors_key)
        else:
            self.server = ors.Client(base_url=server_addr)

    def compute_route(self, profile, coordinates):
        route = self.server.directions(
            coordinates=coordinates,
            profile=profile,
            format='geojson',
            extra_info=["steepness","suitability","surface","waycategory","waytype","tollways","traildifficulty","osmid","roadaccessrestrictions","countryinfo","green","noise"],
            validate=False)
        try:
            route_distance = route['features'][0]['properties']['summary']['distance']
            route_duration = route['features'][0]['properties']['summary']['duration']
        except :
             route_distance = 0
             route_duration = 0

        # print(profile, '\n==Distance:', route_distance, '\n==Duration:', route_duration, '\n')

        return route, route_distance, route_duration

    def compute_haversine_distance(self, point_a, point_b):
        """Computes the Haversine (Great Circle) distance between pointA and pointB."""

        point_a_in_radians = [radians(_) for _ in point_a]
        point_b_in_radians = [radians(_) for _ in point_b]
        result = haversine_distances([point_a_in_radians, point_b_in_radians])
        result = result * 6371000 / 1000 # to get km

        return result[0,1]

def random_batch_points_generator(nodes_data, nbr_points):
    nodes = pd.read_csv(nodes_data)
    nodes = gpd.GeoDataFrame(nodes, geometry=gpd.points_from_xy(nodes.x, nodes.y))

    points = []

    total_nodes = nodes.shape[0]
    with trange(nbr_points) as t:
        for i in t:
            t.set_description('Point %i' % (i+1))

            int_A = rn.randrange(total_nodes)
            int_B = rn.randrange(total_nodes)

            # make sure point B differs from point A
            while (nodes.iloc[int_A].x == nodes.iloc[int_B].x) & (nodes.iloc[int_A].y == nodes.iloc[int_B].y):
                int_B = rn.randrange(total_nodes)

            point_a = [nodes.iloc[int_A].x, nodes.iloc[int_A].y]
            point_b = [nodes.iloc[int_B].x, nodes.iloc[int_B].y]

            points.append([point_a, point_b])

    return points

def read_random_points(file, nbr_points):
    if not os.path.exists(file):
        print('File {} does not exist to read points from. Please recheck the file\'s path'.format(file))
    points = []

    data = pd.read_csv(file, index_col=0, nrows=nbr_points)
    data['point_A'] = data['point_A'].str[1:-1].str.split(", ").apply(lambda x: (list(map(float, x))))
    data['point_B'] = data['point_B'].str[1:-1].str.split(", ").apply(lambda x: (list(map(float, x))))

    with trange(data.shape[0]) as t:
        for i in t:
            point_a = data['point_A'][i]
            point_b = data['point_B'][i]

            points.append([point_a, point_b])

    return points

def main():

    nodes_data = 'data/Lisbon_node_list_simplified.csv'
    output_file_sufix = '2020'
    output_file = 'data/dist_time_lisbon_{}.csv'.format(output_file_sufix)
    points_drawn = True, 'data/dist_time_lisbon_2020.csv'
    # points_drawn = False, ''

    server_addr = 'http://10.0.28.126:10020/ors'
    nbr_points = 100000

    print('Creating output pd.Dataframe\n')
    columns = ['point_A', 'point_B', 'haversine_dist']
    for profile in ors_profiles:
        columns.append(profile+'_dist')
        columns.append(profile + '_time')
    data = pd.DataFrame(columns=columns)

    print('Connecting to ORS server\n')
    ors_obj = ORS(server_addr=server_addr)

    if points_drawn[0] and os.path.exists(points_drawn[1]):
        print('Reading previous selected {} points\n'.format(nbr_points))
        points = read_random_points(points_drawn[1], nbr_points)
    else:
        print('Selecting random {} points\n'. format(nbr_points))
        points = random_batch_points_generator(nodes_data, nbr_points)

    print('Iterating over points...')

    with trange(nbr_points) as t:
        for i in t:
            t.set_description('Point %i' % (i+1))

            map = folium.Map(location=[-9.153140, 38.767118].copy()[::-1], tiles='cartodbpositron', zoom_start=13)

            point_a = points[i][0]
            point_b = points[i][1]

            coordinates = []
            coordinates.append(point_a)
            coordinates.append(point_b)
            data_row = {}
            data_row['point_A'] = point_a
            data_row['point_B'] = point_b

            for profile in ors_profiles:
                try:
                    route, route_distance, route_duration = ors_obj.compute_route(profile, coordinates)
                    data_row[profile+'_dist'] = route_distance
                    data_row[profile+'_time'] = route_duration
                    folium.PolyLine(locations=[list(reversed(coord))
                                               for coord in route['features'][0]['geometry']['coordinates']],
                                    color=color_dict[profile]).add_to(map)
                    route_file = 'routes/'+output_file_sufix+'_'+str(i)+'_'+profile+'.geojson'
                    with open(route_file, 'w') as f:
                        geojson.dump(route, f)

                except ors.exceptions.ApiError:
                    continue

            haversine = ors_obj.compute_haversine_distance(coordinates[0].copy()[::-1], coordinates[-1].copy()[::-1])
            data_row['haversine_dist']  = haversine

            data = data.append(data_row, ignore_index=True, sort=False)

            #map.save('html/index'+str(i)+'.html')
            #map.save('html/'+output_file_sufix+'/index'+str(i)+'.html')

    #data.to_csv(output_file)
    print('Completed! File shape:', data.shape)

if __name__ == "__main__":
    main()

max_stars_repo_name: U-Shift/circuity-lisbon

max_issues_repo_name: U-Shift/circuity-lisbon

max_forks_repo_name: U-Shift/circuity-lisbon

reason: import numpy,import networkx