text:
import streamlit as st
from PIL import Image, ImageDraw
from pathlib import Path
import os
import imagededup
import numpy as np
from io import StringIO
import requests
import sys, io
from blinker import signal
from shutil import copy, move

st.title('Image Deduper')

# should come from user
    #image_dir = Path('/Users/amnaik/Documents/pprojects/pyfindimagedupes/imagededupml/test/')

st.sidebar.title("Setup")

folder_path = st.sidebar.text_input('folder', \
'/Users/amnaik/Documents/test/')

destination_path = Path('/Users/amnaik/Documents/to_delete/')

option = st.sidebar.selectbox('Choose algorithm', ('pHash', 'aHash', 'dHash', 'wHash', 'CNN'))

max_distance = st.sidebar.slider('distance threshold', 0, 64, 10)

search_method = st.sidebar.selectbox('Search', ('bktree','brute_force_cython'))

scores = st.sidebar.checkbox("show scores")

if st.sidebar.button("Start"):
    image_dir = folder_path

# -----------------------------------------------------------------------------#
status = st.empty()
status.text("Waiting...")
progress_bar = st.progress(0)

send_data = signal('send-data')
send_progress = signal('progress')

# -----------------------------------------------------------------------------#
from imagededup.methods import PHash, CNN, AHash, DHash, WHash

@send_data.connect
def receive_data(sender, **kw):
    #print("Caught signal from %r, data %r" % (sender, kw))
    status.text(kw['message'])
    return 'received!'

@send_progress.connect
def receive_progress(sender, **kw):
    #print("Caught signal from %r, data %r" % (sender, kw))
    progress_bar.progress(int(round(kw['percent'])))
    #status.text(kw['message'])
    return 'received!'

if option == "pHash":
    try:
        phasher = PHash()
        duplicates = phasher.find_duplicates(image_dir, max_distance_threshold=max_distance, \
                                         scores=True)
    except Exception as e:
        duplicates = {}
        st.write(str(e))
elif option == "CNN":
    try:
        cnn_encoder = CNN()
        duplicates = cnn_encoder.find_duplicates(image_dir=image_dir, scores=True)
    except Exception as e:
        duplicates = {}
        st.write(str(e))
elif option == "aHash":
    try:
        ahasher = AHash()
        duplicates = ahasher.find_duplicates(image_dir, max_distance_threshold=max_distance, \
                                         scores=True)
    except Exception as e:
        duplicates = {}
        st.write(str(e))
elif option == 'dHash':
    try:
        dhasher = DHash()
        duplicates = dhasher.find_duplicates(image_dir, max_distance_threshold=max_distance, \
                                         scores=True)
    except Exception as e:
        duplicates = {}
        st.write(str(e))
elif option == 'wHash':
    try:
        whasher = WHash()
        duplicates = whasher.find_duplicates(image_dir, max_distance_threshold=max_distance, \
                                         scores=True)
    except Exception as e:
        duplicates = {}
        st.write(str(e))
else:
    duplicates = {}
    st.write("Algorithm not chosen...")

# -----------------------------------------------------------------------------#

img_select = {}
for k, v in duplicates.items():
    if (v != []):
        img_select[k] = [x[0] for x in v]

st.write(img_select)

# -----------------------------------------------------------------------------#
st.subheader("Results")
i = 0
img_selected = []
to_delete = []
for k in img_select.keys():
    captions = []
    images = []
    # check if master image not in the similar image
    if k not in img_selected:
        i = i+1
        st.write("Record: %d" % (i))
        images.append(str(image_dir) + '/' + k)
        #copy for deletion
        copy(str(image_dir) + '/' + k, destination_path)
        to_delete.append(k)
        captions.append(k)
        for vi in img_select[k]:
            images.append(str(image_dir) + '/' + vi)
            captions.append(vi)
            img_selected.append(vi)
            #copy for deletion
            copy(str(image_dir) + '/' + vi, destination_path)
            to_delete.append(vi)

    st.image(images, caption=captions, width=300)

# -----------------------------------------------------------------------------#
num_files = len([f for f in os.listdir(folder_path) if f.endswith('.JPG') and os.path.isfile(os.path.join(folder_path, f))])

if len(img_select):
    summary = ("Found " + str(len(list(set(to_delete)))) + " similar images in total " + str(num_files) + " images")
    outcome = st.sidebar.text_area("Summary", summary)
else:
    outcome = st.sidebar.text_area("Summary", "")

st.subheader("For Deletion")
deletion_summary = st.write ("These files will be deleted from source folder", list(set(to_delete)))

for k in list(set(to_delete)):
    if os.path.isfile(str(image_dir) + '/' + k):
        os.remove(str(image_dir) + '/' + k)
    else:
        couldnt_delete = st.write("could not delete file", k)

max_stars_repo_name: amolnaik/dupre

max_issues_repo_name: amolnaik/dupre

max_forks_repo_name: amolnaik/dupre


text:
# -*- coding: utf-8 -*-
"""
@file
@brief Implements optimized k-nn.
"""
import random
import numpy
from .kppv import NuagePoints

class NuagePointsLaesa (NuagePoints):
    """
    Implémente l'algorithme des plus proches voisins,
    version :ref:`LAESA <space_metric_algo_laesa_prime>`_
    """

    def __init__(self, nb_pivots):
        """
        Construit la classe

        @param      nb_pivots       number of pivots
        """
        NuagePoints.__init__(self)
        self.nb_pivots = nb_pivots

    def fit(self, X, y=None):
        """
        Follows sklearn API.

        @param      X   training set
        @param      y   labels
        """
        self.nuage = X
        self.labels = y
        self.selection_pivots(self.nb_pivots)

    def selection_pivots(self, nb):
        """
        Sélectionne *nb* pivots aléatoirements.

        @param      nb      nombre de pivots
        """
        nb = min(nb, self.nuage.shape[0])
        if nb == 1:
            self.pivots = [2]
        else:
            self.pivots = set()
            while len(self.pivots) < nb:
                i = random.randint(0, self.nuage.shape[0] - 1)
                if i not in self.pivots:
                    self.pivots.add(i)
            self.pivots = list(sorted(self.pivots))

        # on calcule aussi la distance de chaque éléments au pivots
        self.dist = numpy.zeros((self.nuage.shape[0], len(self.pivots)))
        for i in range(self.nuage.shape[0]):
            for j in range(len(self.pivots)):  # pylint: disable=C0200
                self.dist[i, j] = self.distance(
                    self.nuage[i, :], self.nuage[self.pivots[j], :])

    def ppv(self, obj):
        """
        Retourne l'élément le plus proche de obj et sa distance avec obj,
        utilise la sélection à l'aide pivots

        @param      obj     object
        @return             ``tuple(distance, index)``
        """

        # initialisation
        dp = [(self.distance(obj, self.nuage[p, :]), p, i)
              for i, p in enumerate(self.pivots)]

        # pivots le plus proche
        dm, im, _ = min(dp)

        # améliorations
        for i in range(0, self.nuage.shape[0]):

            # on regarde si un pivot permet d'éliminer l'élément i
            calcul = True
            for d, p, ip in dp:
                delta = abs(d - self.dist[i, ip])
                if delta > dm:
                    calcul = False
                    break

            # dans le cas contraire on calcule la distance
            if calcul:
                d = self.distance(obj, self.nuage[i, :])
                if d < dm:
                    dm = d
                    im = i

        return dm, im

ext: py

max_stars_repo_name: Pandinosaurus/mlstatpy

max_issues_repo_name: Pandinosaurus/mlstatpy

max_forks_repo_name: Pandinosaurus/mlstatpy

reason: import numpy


text:
#Author: John, Leo, & Viktoria
#Project: Connect 4
#Date: 12/12/2017

import numpy
import pygame

def createboard(rows,columns):
    """ Creates a string given rows and columns desired
        that can be converted into a matrix through numpy

    >>> createboard(5,4)
    '0,0,0,0,0; 0,0,0,0,0; 0,0,0,0,0; 0,0,0,0,0'
    >>> createboard(3,7)
    '0,0,0; 0,0,0; 0,0,0; 0,0,0; 0,0,0; 0,0,0; 0,0,0'
    """
    row_size = ''
    for rows in range(rows):
        if rows == 0:
            row_size = row_size + '0'
        else:
            row_size = row_size + ',0'
    fullmatrix = ''
    for cols in range(columns):
        if cols == 0:
            fullmatrix = fullmatrix + row_size
        else:
            fullmatrix = fullmatrix + '; ' + row_size
    return fullmatrix

def look_through_rows(board, column, player):
    """ Given a matrix, a column of the matrix, and a key,
    This function will look through the column bottom to top,
    and find the first empty slot indicated by a 0 and place
    a piece there (1 or 2)
    >>> look_through_rows(numpy.matrix('0,0,0,0,0; 0,0,0,0,0; 0,0,0,0,0; 0,0,0,0,0'), 2, 1)
    matrix([[0, 0, 0, 0, 0],
            [0, 0, 0, 0, 0],
            [0, 0, 0, 0, 0],
            [0, 0, 1, 0, 0]])
    >>> look_through_rows(numpy.matrix('0,0,0; 0,0,0; 0,0,0; 0,0,0; 0,0,0; 0,0,0; 0,0,0'), 1, 2)
    matrix([[0, 0, 0],
            [0, 0, 0],
            [0, 0, 0],
            [0, 0, 0],
            [0, 0, 0],
            [0, 0, 0],
            [0, 2, 0]])
    """
    if board.shape[1] > column:
        count = board.shape[0] - 1
        count2 = 1
        while count >= 0 and count2 == 1:
            if board[count,column] == 0:
                board[count,column] = player
                count2 = count2 - 1
            else:
                count = count - 1
        return board
    else:
        print('Improper Column Given')

if __name__ == "__main__":
    import doctest
    doctest.testmod(verbose=False)

max_stars_repo_name: feberhardt/Project-Digital

max_issues_repo_name: feberhardt/Project-Digital

max_forks_repo_name: feberhardt/Project-Digital


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""This module contains functions to be used with the scipy.optimize package in order to fit the 3DMM to a target depth map.
"""

import numpy as np
from ..utils.mesh import generateFace
from ..utils.transform import rotMat2angle
from .derivative import dR_dpsi, dR_dtheta, dR_dphi

def initialShapeCost(param, target, model, w = (1, 1)):
    # Shape eigenvector coefficients
    idCoef = param[: model.numId]
    expCoef = param[model.numId: model.numId + model.numExp]

    # Landmark fitting cost
    source = generateFace(param, model, ind = model.sourceLMInd)

    rlan = (source - target.T).flatten('F')
    Elan = np.dot(rlan, rlan) / model.sourceLMInd.size

    # Regularization cost
    Ereg = np.sum(idCoef ** 2 / model.idEval) + np.sum(expCoef ** 2 / model.expEval)

    return w[0] * Elan + w[1] * Ereg

def initialShapeGrad(param, target, model, w = (1, 1)):
    # Shape eigenvector coefficients
    idCoef = param[: model.numId]
    expCoef = param[model.numId: model.numId + model.numExp]

    # Rotation Euler angles, translation vector, scaling factor
    angles = param[model.numId + model.numExp:][:3]
    R = rotMat2angle(angles)
    t = param[model.numId + model.numExp:][3: 6]
    s = param[model.numId + model.numExp:][6]

    # The eigenmodel, before rigid transformation and scaling
    shape = model.idMean[:, model.sourceLMInd] + np.tensordot(model.idEvec[:, model.sourceLMInd, :], idCoef, axes = 1) + np.tensordot(model.expEvec[:, model.sourceLMInd, :], expCoef, axes = 1)

    # After rigid transformation and scaling
    source = s*np.dot(R, shape) + t[:, np.newaxis]

    rlan = (source - target.T).flatten('F')

    drV_dalpha = s*np.tensordot(R, model.idEvec[:, model.sourceLMInd, :], axes = 1)
    drV_ddelta = s*np.tensordot(R, model.expEvec[:, model.sourceLMInd, :], axes = 1)
    drV_dpsi = s*np.dot(dR_dpsi(angles), shape)
    drV_dtheta = s*np.dot(dR_dtheta(angles), shape)
    drV_dphi = s*np.dot(dR_dphi(angles), shape)
    drV_dt = np.tile(np.eye(3), [model.sourceLMInd.size, 1])
    drV_ds = np.dot(R, shape)

    Jlan = np.c_[drV_dalpha.reshape((source.size, idCoef.size), order = 'F'), drV_ddelta.reshape((source.size, expCoef.size), order = 'F'), drV_dpsi.flatten('F'), drV_dtheta.flatten('F'), drV_dphi.flatten('F'), drV_dt, drV_ds.flatten('F')]

    return 2 * (w[0] * np.dot(Jlan.T, rlan) / model.sourceLMInd.size + w[1] * np.r_[idCoef / model.idEval, expCoef / model.expEval, np.zeros(7)])

def shapeCost(param, model, target, targetLandmarks, NN, w = (1, 1, 1), calcID = True):
    # Shape eigenvector coefficients
    idCoef = param[: model.numId]
    expCoef = param[model.numId: model.numId + model.numExp]

    # Transpose target if necessary
    if targetLandmarks.shape[0] != 3:
        targetLandmarks = targetLandmarks.T

    # After rigid transformation and scaling
    source = generateFace(param, model)

    # Find the nearest neighbors of the target to the source vertices
    distance, ind = NN.kneighbors(source.T)
    targetNN = target[ind.squeeze(axis = 1), :].T

    # Calculate resisduals
    rver = (source - targetNN).flatten('F')
    rlan = (source[:, model.sourceLMInd] - targetLandmarks).flatten('F')

    # Calculate costs
    Ever = np.dot(rver, rver) / model.numVertices
    Elan = np.dot(rlan, rlan) / model.sourceLMInd.size

    if calcID:

        Ereg = np.sum(idCoef ** 2 / model.idEval) + np.sum(expCoef ** 2 / model.expEval)

    else:

        Ereg = np.sum(expCoef ** 2 / model.expEval)

    return w[0] * Ever + w[1] * Elan + w[2] * Ereg

def shapeGrad(param, model, target, targetLandmarks, NN, w = (1, 1, 1), calcID = True):
    # Shape eigenvector coefficients
    idCoef = param[: model.numId]
    expCoef = param[model.numId: model.numId + model.numExp]

    # Rotation Euler angles, translation vector, scaling factor
    angles = param[model.numId + model.numExp:][:3]
    R = rotMat2angle(angles)
    t = param[model.numId + model.numExp:][3: 6]
    s = param[model.numId + model.numExp:][6]

    # Transpose if necessary
    if targetLandmarks.shape[0] != 3:
        targetLandmarks = targetLandmarks.T

    # The eigenmodel, before rigid transformation and scaling
    shape = model.idMean + np.tensordot(model.idEvec, idCoef, axes = 1) + np.tensordot(model.expEvec, expCoef, axes = 1)

    # After rigid transformation and scaling
    source = s*np.dot(R, shape) + t[:, np.newaxis]

    # Find the nearest neighbors of the target to the source vertices
    distance, ind = NN.kneighbors(source.T)
    targetNN = target[ind.squeeze(axis = 1), :].T

    # Calculate resisduals
    rver = (source - targetNN).flatten('F')
    rlan = (source[:, model.sourceLMInd] - targetLandmarks).flatten('F')

    drV_ddelta = s*np.tensordot(R, model.expEvec, axes = 1)
    drV_dpsi = s*np.dot(dR_dpsi(angles), shape)
    drV_dtheta = s*np.dot(dR_dtheta(angles), shape)
    drV_dphi = s*np.dot(dR_dphi(angles), shape)
    drV_dt = np.tile(np.eye(3), [model.numVertices, 1])
    drV_ds = np.dot(R, shape)

    if calcID:

        drV_dalpha = s*np.tensordot(R, model.idEvec, axes = 1)

        Jver = np.c_[drV_dalpha.reshape((source.size, idCoef.size), order = 'F'), drV_ddelta.reshape((source.size, expCoef.size), order = 'F'), drV_dpsi.flatten('F'), drV_dtheta.flatten('F'), drV_dphi.flatten('F'), drV_dt, drV_ds.flatten('F')]

        Jlan = np.c_[drV_dalpha[:, model.sourceLMInd, :].reshape((targetLandmarks.size, idCoef.size), order = 'F'), drV_ddelta[:, model.sourceLMInd, :].reshape((targetLandmarks.size, expCoef.size), order = 'F'), drV_dpsi[:, model.sourceLMInd].flatten('F'), drV_dtheta[:, model.sourceLMInd].flatten('F'), drV_dphi[:, model.sourceLMInd].flatten('F'), drV_dt[:model.sourceLMInd.size * 3, :], drV_ds[:, model.sourceLMInd].flatten('F')]

        return 2 * (w[0] * np.dot(Jver.T, rver) / model.numVertices + w[1] * np.dot(Jlan.T, rlan) / model.sourceLMInd.size + w[2] * np.r_[idCoef / model.idEval, expCoef / model.expEval, np.zeros(7)])

    else:

        Jver = np.c_[drV_ddelta.reshape((source.size, expCoef.size), order = 'F'), drV_dpsi.flatten('F'), drV_dtheta.flatten('F'), drV_dphi.flatten('F'), drV_dt, drV_ds.flatten('F')]

        Jlan = np.c_[drV_ddelta[:, model.sourceLMInd, :].reshape((targetLandmarks.size, expCoef.size), order = 'F'), drV_dpsi[:, model.sourceLMInd].flatten('F'), drV_dtheta[:, model.sourceLMInd].flatten('F'), drV_dphi[:, model.sourceLMInd].flatten('F'), drV_dt[:model.sourceLMInd.size * 3, :], drV_ds[:, model.sourceLMInd].flatten('F')]

        return 2 * (np.r_[np.zeros(idCoef.size), w[0] * np.dot(Jver.T, rver) / model.numVertices] + np.r_[np.zeros(idCoef.size), w[1] * np.dot(Jlan.T, rlan) / model.sourceLMInd.size] + w[2] * np.r_[np.zeros(idCoef.size), expCoef / model.expEval, np.zeros(7)])

max_stars_repo_name: leon-nn/face-fitting

max_issues_repo_name: leon-nn/face-fitting

max_forks_repo_name: leon-nn/face-fitting


text:
import numpy as np
import matplotlib.pyplot as plt

### Para copiar archivo adentro de la libreria de Odrive ###
if __name__ == '__main__':
    import shutil
    shutil.copy(__file__, 'env/Lib/site-packages/odrive/roboDrive.py')

import odrive.utils as utils

def print_group_trajs(chosen):
    time_axis = []
    accumulated_time = 0
    estimatess = []
    inputss = []
    errorss = []
    for indiv in chosen:
        time_axis.extend([t+accumulated_time for t in indiv.data['time_axis']])
        accumulated_time = time_axis[-1]
        e = indiv.data['pos_estimate_a1']
        estimatess.extend(e)
        i = indiv.data['input_pos']
        inputss.extend(i)
        errorss.extend(list(np.subtract(np.array(i),np.array(e))))
    plt.plot(time_axis, estimatess)
    plt.plot(time_axis, inputss)
    plt.plot(time_axis, errorss)
    plt.xlabel("Muestreo")
    plt.ylabel("Posición")
    plt.legend(["Posición Actual", "Referencia", "Error"])
    plt.show()

def pos_plotter(odrvAxis):
     utils.start_liveplotter(lambda: [odrvAxis.encoder.pos_estimate,
     odrvAxis.controller.pos_setpoint,
     odrvAxis.controller.pos_setpoint - odrvAxis.encoder.pos_estimate])

def change_plotter_rate(new_data_rate, new_plot_rate, new_num_samples):
    utils.data_rate = new_data_rate
    utils.plot_rate = new_plot_rate
    utils.num_samples = new_num_samples

change_plotter_rate(100, 100, 500)

max_stars_repo_name: valenmgama/robo-control

max_issues_repo_name: valenmgama/robo-control

max_forks_repo_name: valenmgama/robo-control


text:
import pytest

import fenics
import numpy
from fenics_numpy import fenics_to_numpy, numpy_to_fenics

@pytest.mark.parametrize(
    "test_input,expected",
    [
        (fenics.Constant(0.66), numpy.asarray(0.66)),
        (fenics.Constant([0.5, 0.66]), numpy.asarray([0.5, 0.66])),
    ],
)
def test_fenics_to_numpy_constant(test_input, expected):
    assert numpy.allclose(fenics_to_numpy(test_input), expected)

def test_fenics_to_numpy_function():
    # Functions in DG0 have nodes at centers of finite element cells
    mesh = fenics.UnitIntervalMesh(10)
    V = fenics.FunctionSpace(mesh, "DG", 0)
    test_input = fenics.interpolate(fenics.Expression("x[0]", degree=1), V)
    expected = numpy.linspace(0.05, 0.95, num=10)
    assert numpy.allclose(fenics_to_numpy(test_input), expected)

def test_fenics_to_numpy_mixed_function():
    # Functions in DG0 have nodes at centers of finite element cells
    mesh = fenics.UnitIntervalMesh(10)
    vec_dim = 4
    V = fenics.VectorFunctionSpace(mesh, "DG", 0, dim=vec_dim)
    test_input = fenics.interpolate(
        fenics.Expression(vec_dim * ("x[0]",), element=V.ufl_element()), V
    )
    expected = numpy.linspace(0.05, 0.95, num=10)
    expected = numpy.reshape(numpy.tile(expected, (4, 1)).T, V.dim())
    assert numpy.allclose(fenics_to_numpy(test_input), expected)

def test_fenics_to_numpy_vector():
    # Functions in DG0 have nodes at centers of finite element cells
    mesh = fenics.UnitIntervalMesh(10)
    V = fenics.FunctionSpace(mesh, "DG", 0)
    test_input = fenics.interpolate(fenics.Expression("x[0]", degree=1), V)
    test_input_vector = test_input.vector()
    expected = numpy.linspace(0.05, 0.95, num=10)
    assert numpy.allclose(fenics_to_numpy(test_input_vector), expected)

@pytest.mark.parametrize(
    "test_input,expected",
    [
        (numpy.asarray(0.66), fenics.Constant(0.66)),
        (numpy.asarray([0.5, 0.66]), fenics.Constant([0.5, 0.66])),
    ],
)
def test_numpy_to_fenics_constant(test_input, expected):
    fenics_test_input = numpy_to_fenics(test_input, fenics.Constant(0.0))
    assert numpy.allclose(fenics_test_input.values(), expected.values())

def test_numpy_to_fenics_function():
    test_input = numpy.linspace(0.05, 0.95, num=10)
    mesh = fenics.UnitIntervalMesh(10)
    V = fenics.FunctionSpace(mesh, "DG", 0)
    template = fenics.Function(V)
    fenics_test_input = numpy_to_fenics(test_input, template)
    expected = fenics.interpolate(fenics.Expression("x[0]", degree=1), V)
    assert numpy.allclose(
        fenics_test_input.vector().get_local(), expected.vector().get_local()
    )

max_stars_repo_name: IvanYashchuk/numpy-fenics-adjoint

max_issues_repo_name: IvanYashchuk/numpy-fenics-adjoint

max_forks_repo_name: IvanYashchuk/numpy-fenics-adjoint


text:
#!/usr/bin/env python
#-*- coding:utf-8 -*-
import numpy as np

#fname='000AsP.input' #hamiltonian file name
fname='Cu' #hamiltonian file name
mu=9.8               #chemical potential
mass=1.0             #effective mass
sw_inp=2             #input hamiltonian format
"""
sw_inp: switch input hamiltonian's format
0: .input file
1: ham_r.txt, irvec.txt, ndegen.txt
2: {case}_hr.dat file (wannier90 default hopping file)
else: Hopping.dat file (ecalj hopping file)
"""

option=7
"""
option: switch calculation modes
0: band plot
1: write Fermi surface at kz=0
2: write 3D Fermi surface
3: write Fermi velocity with Fermi surface
4: plot spectrum like band plot
5: plot spectrum at E=EF
6: plot 3D Fermi velocity with Fermi surface
7: calc conductivity
8: plot Dos
9: calc carrier num.
"""

sw_calc_mu =True
fill=5.50

alatt=np.array([1.,1.,1.]) #Bravais lattice parameter a,b,c
#alatt=np.array([3.96*np.sqrt(2.),3.96*np.sqrt(2.),13.02*0.5]) #Bravais lattice parameter a,b,c
Arot=np.array([[ .5,-.5, .5],[ .5, .5, .5],[-.5,-.5, .5]]) #rotation matrix for dec. to primitive vector
k_list=[[0., 0., 0.],[.5, 0., 0.],[.5, .5, 0.],[0.,0.,0.]] #coordinate of sym. points
xlabel=['$\Gamma$','X','M','$\Gamma$'] #sym. points name

olist=[1,2,3]        #orbital number with color plot [R,G,B] if you merge some orbitals input orbital list in elements
N=80                #kmesh btween symmetry points
FSmesh=80           #kmesh for option in {1,2,3,5,6}
eta=5.0e-3           #eta for green function
sw_dec_axis=False    #transform Cartesian axis
sw_color=True        #plot band or FS with orbital weight
kz=np.pi*0.
with_spin=False #use only with soc hamiltonian

#----------import modules without scipy-------------
import scipy.linalg as sclin
import scipy.optimize as scopt
import scipy.constants as scconst
import matplotlib.pyplot as plt
import matplotlib.cm as cm
import input_ham
#----------------define functions-------------------
if sw_dec_axis:
    pass
else:
    avec=alatt*Arot
    len_avec=np.sqrt((abs(avec)**2).sum(axis=1))
    Vuc=sclin.det(avec)

def get_ham(k,rvec,ham_r,ndegen,out_phase=False):
    """
    This function generates hamiltonian from hopping parameters.
    arguments:
    k: k-point coordinate
    rvec: real space coodinate for hoppings
    ham_r: values of hoppings
    ndegen: weight for hoppings
    out_phase: output or not phase array (optional, default=False)
    return values:
    ham: wave-number space hamiltonian in k
    expk: phase in k
    """
    phase=(rvec*k).sum(axis=1)
    expk=(np.cos(phase)-1j*np.sin(phase))/ndegen
    no,nr=len(ham_r),len(expk)
    ham=(ham_r.reshape(no*no,nr)*expk).sum(axis=1).reshape(no,no)
    if out_phase:
        return ham, expk,no,nr
    else:
        return ham

def get_mu(fill,rvec,ham_r,ndegen,temp=1.0e-3,mesh=40):
    """
    This function calculates chemical potential.
    arguments:
    fill: band filling (number of particles in band)
    rvec: real space coodinate for hoppings
    ham_r: values of hoppings
    ndegen: weight for hoppings
    temp: temperature (optional, default=1.0e-3)
    mesh: k-points mesh (optional, default=40)
    return value:
    mu: chemical potential
    """
    km=np.linspace(0,2*np.pi,mesh,False)
    x,y,z=np.meshgrid(km,km,km)
    klist=np.array([x.ravel(),y.ravel(),z.ravel()]).T
    ham=np.array([get_ham(k,rvec,ham_r,ndegen) for k in klist])
    eig=np.array([sclin.eigvalsh(h) for h in ham]).T
    f=lambda mu: 2.*fill*mesh**3+(np.tanh(0.5*(eig-mu)/temp)-1.).sum()
    #mu=scopt.brentq(f,eig.min(),eig.max())
    mu=scopt.newton(f,0.5*(eig.min()+eig.max()))
    print('chemical potential = %6.3f'%mu)
    return mu

def get_vec(k,rvec,ham_r,ndegen):
    """
    This function generates velocities from hopping parameters.
    arguments:
    k: k-point coordinate
    rvec: real space coodinate for hoppings
    ham_r: values of hoppings
    ndegen: weight for hoppings
    return values:
    vec: velocity in k for each bands
    """
    ihbar=1./scconst.physical_constants['Planck constant over 2 pi in eV s'][0]*1.0e-10
    #ihbar=1.
    ham,expk,no,nr=get_ham(k,rvec,ham_r,ndegen,out_phase=True)
    uni=sclin.eigh(ham)[1]
    vec0=np.array([-1j*ihbar*(ham_r.reshape(no*no,nr)*(r*expk)).sum(axis=1).reshape(no,no)
                    for r in (len_avec*rvec).T])
    vec=np.array([(uni.conjugate().T.dot(v0).dot(uni)).diagonal() for v0 in vec0]).T
    return vec

def gen_eig(ham,mass,mu,sw):
    """
    This function generates eigenvalue and eigenvectors or max and min energy values of hamiltonian.
    arguments:
    ham: wave-number space hamiltonian in k
    mass: effective mass
    mu: chemical potential
    sw: switch for retrun enegyes or max/min energy
    return values:
    eig: eigenvalues of hamiltonian
    uni: eigenvectors of hamiltonian
    """
    if sw:
        etmp=[sclin.eigh(h) for h in ham]
        eigtmp=np.array([eg[0] for eg in etmp])
        eig=eigtmp.T/mass-mu
        uni=np.array([eg[1] for eg in etmp]).T
        return eig,uni
    else:
        eigtmp=np.array([sclin.eigvalsh(h) for h in ham])
        return (eigtmp.max()/mass-mu),(eigtmp.min()/mass-mu)

def mk_klist(k_list,N):
    """
    This function generates klist of spaghetti.
    arguments:
    k_list: name and coordinate of sym. points
    N: k-mesh between sym. points
    return values:
    klist: klist of spaghetti
    splen: length of hrizontal axis
    xticks: xlabels
    """
    klist=[]
    splen=[]
    maxsplen=0
    xticks=[]
    for ks,ke in zip(k_list,k_list[1:]):
        dkv=np.array(ke)-np.array(ks)
        dkv_length=np.sqrt(((dkv*len_avec)**2).sum())
        tmp=2.*np.pi*np.linspace(ks,ke,N)
        tmp2=np.linspace(0,dkv_length,N)+maxsplen
        maxsplen=tmp2.max()
        xticks=xticks+[tmp2[0]]
        klist=klist+list(tmp[:-1])
        splen=splen+list(tmp2[:-1])
    klist=klist+[2*np.pi*np.array(k_list[-1])]
    splen=splen+[maxsplen+dkv_length/N]
    xticks=xticks+[splen[-1]]
    return np.array(klist),np.array(splen),xticks

def plot_band(eig,spl,xticks,uni,ol):
    """
    This function plot spaghetti.
    arguments:
    eig: energy array
    spl: coordinates of horizontal axis
    xticks: xlabels
    uni: weight of orbitals
    ol: plot orbital list
    """
    def get_col(cl,ol):
        col=(np.abs(cl[ol])**2 if isinstance(ol,int)
             else (np.abs(cl[ol])**2).sum(axis=0)).round(4)
        return col
    for e,cl in zip(eig,uni):
        c1=get_col(cl,ol[0])
        c2=get_col(cl,ol[1])
        c3=get_col(cl,ol[2])
        clist=np.array([c1,c2,c3]).T
        plt.scatter(spl,e,s=5,c=clist)
    for x in xticks[1:-1]:
        plt.axvline(x,ls='-',lw=0.25,color='black')
    plt.xlim(0,spl.max())
    plt.axhline(0.,ls='--',lw=0.25,color='black')
    plt.xticks(xticks,xlabel)
    plt.show()

def plot_spectrum(ham,klen,xticks,mu,eta0=5.e-2,de=100,smesh=200):
    """
    This function plot spaghetti like spectrum.
    arguments:
    ham: hamiltonian array
    klen: coordinates of horizontal axis
    mu: chemical potential
    de: energy mesh (optional, default=100)
    eta: eta for green function (optional, default=5e-2)
    smesh: contor mesh (optional, default=200)
    """
    emax,emin=gen_eig(ham,mass,mu,False)
    w=np.linspace(emin*1.1,emax*1.1,de)
    no=len(ham[0])
    #eta=w*0+eta0
    etamax=4.0e0
    eta=etamax*w*w/min(emax*emax,emin*emin)+eta0
    G=np.array([[-sclin.inv((ww+mu+et*1j)*np.identity(no)-h) for h in ham] for ww,et in zip(w,eta)])
    trG=np.array([[np.trace(gg).imag/(no*no) for gg in g] for g in G])
    sp,w=np.meshgrid(klen,w)
    plt.hot()
    plt.contourf(sp,w,trG,smesh)
    plt.colorbar()
    for x in xticks[1:-1]:
        plt.axvline(x,ls='-',lw=0.25,color='black')
    plt.xlim(0,klen.max())
    plt.axhline(0.,ls='--',lw=0.25,color='black')
    plt.xticks(xticks,xlabel)
    plt.show()

def make_kmesh(mesh,dim,kz=0,sw=False):
    """
    This function generates square k mesh for 2D spectrum plot
    arguments:
    mesh: k-mesh grid size
    kz: kz of plotting FS plane
    """
    km=np.linspace(-np.pi,np.pi,mesh+1,True)
    if dim==2:
        x,y=np.meshgrid(km,km)
        z=y*0.0+kz
    elif dim==3:
        x,y,z=np.meshgrid(km,km,km)
    klist=np.array([x.ravel(),y.ravel(),z.ravel()]).T
    if sw:
        return(klist,x,y)
    else:
        return(klist)

def mk_kf(mesh,sw_bnum,dim,rvec,ham_r,ndegen,mu,kz=0):
    """
    This function generates k-list on Fermi surfaces
    arguments:
    mesh: initial k-mesh grid size
    sw_bnum:switch output format
    dim: output dimension
    kz: kz of plotting FS plane use only dim=2 (optional,default=0)
    return values:
    v2: klist on Fermi surface
    fsband: band number crossing Fermi energy
    """
    import skimage.measure as sk
    from mpl_toolkits.mplot3d import axes3d
    klist=make_kmesh(mesh,dim,kz)
    ham=np.array([get_ham(k,rvec,ham_r,ndegen) for k in klist])
    eig=np.array([sclin.eigvalsh(h) for h in ham]).T/mass-mu
    v2=[]
    if sw_bnum:
        fsband=[]
    for i,e in enumerate(eig):
        if(e.max()*e.min() < 0. ):
            if dim==2:
                cont=sk.find_contours(e.reshape(mesh+1,mesh+1),0)
                ct0=[]
                for c in cont:
                    ct0.extend(c)
                ct=(np.array([[c[0],c[1],+mesh/2] for c in ct0])-mesh/2)*2*np.pi/mesh
                ct[:,2]=kz
                if sw_bnum:
                    fsband.append(i)
                    v2.append(ct)
                else:
                    v2.extend(ct)
            elif dim==3:
                vertices,faces,normals,values=sk.marching_cubes_lewiner(e.reshape(mesh+1,mesh+1,mesh+1),0)
                if sw_bnum:
                    fsband.append(i)
                    v2.append((vertices-mesh/2)*2*np.pi/mesh)
                    #v3.append(faces)
                else:
                    v2.extend((2*np.pi*(vertices-mesh/2)/mesh)[faces])
    if sw_bnum:
        return v2,fsband
    else:
        return np.array(v2)

def gen_3d_fs_plot(mesh,rvec,ham_r,ndegen,mu):
    """
    This function plot 3D Fermi Surface
    argument:
    mesh: k-grid mesh size
    """
    from mpl_toolkits.mplot3d import axes3d
    from mpl_toolkits.mplot3d.art3d import Poly3DCollection
    vert=mk_kf(mesh,False,3,rvec,ham_r,ndegen,mu)
    fig=plt.figure()
    ax=fig.add_subplot(111,projection='3d')
    m = Poly3DCollection(vert)
    ax.add_collection3d(m)
    ax.set_xlim(-np.pi, np.pi)
    ax.set_ylim(-np.pi, np.pi)
    ax.set_zlim(-np.pi, np.pi)
    plt.tight_layout()
    plt.show()

def plot_veloc_FS(vfs,kfs):
    """
    This function plot 3D Fermi velocities
    argument:
    mesh: k-grid mesh size
    """
    from mpl_toolkits.mplot3d import axes3d
    fig=plt.figure()
    ax=fig.add_subplot(111,projection='3d')
    vf,kf=[],[]
    for v,k in zip(vfs,kfs):
        ave_vx=np.abs(np.array(v).T[0]).mean()
        ave_vy=np.abs(np.array(v).T[1]).mean()
        ave_vz=np.abs(np.array(v).T[2]).mean()
        print('%.3e %.3e %.3e'%(ave_vx,ave_vy,ave_vz))
        vf.extend(v)
        kf.extend(k)
    x,y,z=zip(*np.array(kf))
    vf=np.array(vf)
    ave_vx=np.abs(vf.T[0]).mean()
    ave_vy=np.abs(vf.T[1]).mean()
    ave_vz=np.abs(vf.T[2]).mean()
    print('%.3e %.3e %.3e'%(ave_vx,ave_vy,ave_vz))
    absv=np.array([np.abs(v).sum() for v in vf])
    fs=ax.scatter(x,y,z,c=absv,cmap=cm.jet)
    ax.set_xlim(-np.pi, np.pi)
    ax.set_ylim(-np.pi, np.pi)
    ax.set_zlim(-np.pi, np.pi)
    plt.colorbar(fs,format='%.2e')
    plt.show()

def plot_vec2(veloc,klist):
    v=[]
    k=[]
    for vv,kk in zip(veloc,klist):
        v0=np.array([np.sqrt((np.abs(v0)**2).sum()) for v0 in vv])
        v.extend(v0)
        k.extend(kk)
    v=np.array(v)
    k=np.array(k)
    plt.scatter(k[:,0],k[:,1],s=1.0,c=v)
    plt.jet()
    plt.xlim(-np.pi,np.pi)
    plt.ylim(-np.pi,np.pi)
    plt.xticks([-np.pi,0,np.pi],['-$\pi$','0','$\pi$'])
    plt.yticks([-np.pi,0,np.pi],['-$\pi$','0','$\pi$'])
    plt.colorbar(format='%.2e')
    plt.show()

def plot_FS(uni,klist,ol,eig,X,Y,sw_color,ncut=8):
    """
    This function plot 2D Fermi Surface with/without orbital weight
    argument:
    uni: eigenvectors
    klist: klist of Fermi surface
    ol: orbital list using color plot
    eig: eigenvalues
    X: X axis array
    Y: Y axis array
    sw_color: swtich of color plot
    """
    def get_col(cl,ol):
        col=(np.abs(cl[:,ol])**2 if isinstance(ol,int)
             else (np.abs(cl[:,ol])**2).sum(axis=0)).round(4)
        return col
    fig=plt.figure()
    ax=fig.add_subplot(111,aspect='equal')
    if sw_color:
        col=['r','g','b','c','m','y','k','w']
        for kk,cl,cb in zip(klist,uni,col):
            cl=np.array(cl)
            c1=get_col(cl,ol[0])
            c2=get_col(cl,ol[1])
            c3=get_col(cl,ol[2])
            clist=np.array([c1,c2,c3]).T
            if(with_spin):
                vud=cl[:,no//2:]*cl[:,:no//2].conjugate()
                vdu=cl[:,:no//2]*cl[:,no//2:].conjugate()
                v1=(vud+vdu).sum(axis=1).real
                v2=(vud-vdu).sum(axis=1).imag
                #v3=(abs(cl[:,:no//2])**2-abs(cl[:,no//2:])**2).sum(axis=1).real
                v1=v1[::ncut].round(4)
                v2=v2[::ncut].round(4)
                #v3=v3[::ncut].round(4)
                k1=kk[::ncut,0]
                k2=kk[::ncut,1]
                plt.quiver(k1,k2,v1,v2,color=cb,angles='xy',scale_units='xy',scale=3.0)
            plt.scatter(kk[:,0],kk[:,1],s=2.0,c=clist)
    else:
        for en in eig:
            if(en.max()*en.min()<0.0):
                plt.contour(X,Y,en.reshape(FSmesh,FSmesh),levels=[0.],color='black')
    plt.xlim(-np.pi,np.pi)
    plt.ylim(-np.pi,np.pi)
    plt.xticks([-np.pi,0,np.pi],['-$\pi$','0','$\pi$'])
    plt.yticks([-np.pi,0,np.pi],['-$\pi$','0','$\pi$'])
    plt.show()

def plot_vec(veloc,eig,X,Y):
    fig=plt.figure()
    ax=fig.add_subplot(111,aspect='equal')
    for v,en in zip(veloc,eig):
        plt.contourf(X,Y,v.reshape(FSmesh,FSmesh).real,100)
        plt.colorbar()
        if(en.max()*en.min()<0.0):
            plt.contour(X,Y,en.reshape(FSmesh,FSmesh),levels=[0.])
        plt.show()

def plot_FSsp(ham,mu,X,Y,eta=5.0e-2,smesh=50):
    no=len(ham[0])
    G=np.array([-sclin.inv((0.+mu+eta*1j)*np.identity(no)-h) for h in ham])
    trG=np.array([np.trace(gg).imag/(no*no) for gg in G]).reshape(FSmesh+1,FSmesh+1)
    #trG=np.array([(gg[4,4]+gg[9,9]).imag/(no*no) for gg in G]).reshape(FSmesh,FSmesh)

    fig=plt.figure()
    ax=fig.add_subplot(111,aspect='equal')
    ax.set_xticks([-np.pi,0,np.pi])
    ax.set_xticklabels(['-$\pi$','0','$\pi$'])
    ax.set_yticks([-np.pi,0,np.pi])
    ax.set_yticklabels(['-$\pi$','0','$\pi$'])
    cont=ax.contourf(X,Y,trG,smesh,cmap=plt.jet())
    fig.colorbar(cont)
    plt.show()

def get_conductivity(mesh,rvec,ham_r,ndegen,mu,temp=1.0e-3):
    """
    this function calculates conductivity at tau==1 from Boltzmann equation in metal
    """
    kb=scconst.physical_constants['Boltzmann constant in eV/K'][0] #temp=kBT[eV], so it need to convert eV>K
    #kb=1.
    km=np.linspace(0,2*np.pi,mesh,False)
    x,y,z=np.meshgrid(km,km,km)
    klist=np.array([x.ravel(),y.ravel(),z.ravel()]).T
    Nk=len(klist)
    ham=np.array([get_ham(k,rvec,ham_r,ndegen) for k in klist])
    eig=np.array([sclin.eigvalsh(h) for h in ham]).T/mass-mu
    dfermi=0.25*(1.-np.tanh(0.5*eig/temp)**2)/temp
    veloc=np.array([get_vec(k,rvec,ham_r,ndegen).real for k in klist])
    sigma=np.array([[(vk1*vk2*dfermi).sum() for vk2 in veloc.T] for vk1 in veloc.T])/Nk
    l12=kb*np.array([[(vk1*vk2*eig*dfermi).sum() for vk2 in veloc.T] for vk1 in veloc.T])/(temp*Nk))
    kappa=kb*np.array([[(vk1*vk2*eig**2*dfermi).sum() for vk2 in veloc.T] for vk1 in veloc.T])/(temp*Nk)
    Seebeck=l12.dot(sclin.inv(sigma))
    print('sigma matrix')
    print(sigma)
    print('Seebeck matrix')
    print(Seebeck)
    print('kappa matrix')
    print(kappa)
    print('lorenz matrix')
    print(kb*kappa/(sigma*temp))

def get_carrier_num(mesh,rvec,ham_r,ndegen,mu):
    km=np.linspace(0,2*np.pi,mesh,False)
    x,y,z=np.meshgrid(km,km,km)
    klist=np.array([x.ravel(),y.ravel(),z.ravel()]).T
    Nk=klist.size/3
    ham=np.array([get_ham(k,rvec,ham_r,ndegen) for k in klist])
    eig=np.array([sclin.eigvalsh(h) for h in ham]).T/mass-mu
    for i,en in enumerate(eig):
        num_hole=float(np.where(en>0)[0].size)/Nk
        num_particle=float(np.where(en<=0)[0].size)/Nk
        print(i+1,round(num_hole,4),round(num_particle,4))

def plot_dos(mesh,rvec,ham_r,ndegen,mu,no,eta,de=200):
    km=np.linspace(0,2*np.pi,mesh,False)
    x,y,z=np.meshgrid(km,km,km)
    klist=np.array([x.ravel(),y.ravel(),z.ravel()]).T
    ham=np.array([get_ham(k,rvec,ham_r,ndegen) for k in klist])
    eig=np.array([sclin.eigvalsh(h) for h in ham])-mu
    emax=eig.max()
    emin=eig.min()
    w=np.linspace(emin*1.1,emax*1.1,de)
    dos=np.array([(eta/((ww-eig)**2+eta**2)).sum() for ww in w])/len(klist)*eta
    plt.plot(w,dos)
    plt.ylim(0,dos.max()*1.2)
    plt.show()
def main():
    if sw_inp==0: #.input file
        rvec,ndegen,ham_r,no,nr=input_ham.import_out(fname,False)
    elif sw_inp==1: #rvec.txt, ham_r.txt, ndegen.txt files
        rvec,ndegen,ham_r,no,nr=input_ham.import_hop(fname,True,False)
    elif sw_inp==2:
        rvec,ndegen,ham_r,no,nr=input_ham.import_hr(fname,False)
    else: #Hopping.dat file
        rvec,ndegen,ham_r,no,nr,axis=input_ham.import_Hopping(fname,False,True)

    if sw_calc_mu:
        mu=get_mu(fill,rvec,ham_r,ndegen)
    else:
        try:
            mu
        except NameError:
            mu=get_mu(fill,rvec,ham_r,ndegen)

    if sw_dec_axis:
        rvec1=np.array([Arot.T.dot(r) for r in rvec])
        rvec=rvec1

    if option in (0,1,4,5):
        if option in (0,4):
            klist,spa_length,xticks=mk_klist(k_list,N)
        else: #1,5
            klist,X,Y=make_kmesh(FSmesh,2,kz=0,sw=True)
        ham=np.array([get_ham(k,rvec,ham_r,ndegen) for k in klist])
        if option in (0,1):
            eig,uni=gen_eig(ham,mass,mu,True)

    if option==0: #band plot
        plot_band(eig,spa_length,xticks,uni,olist)
    elif option==1: #write Fermi surface at kz=0
        klist1,blist=mk_kf(FSmesh,True,2,rvec,ham_r,ndegen,mu,kz)
        ham1=np.array([[get_ham(k,rvec,ham_r,ndegen) for k in kk] for kk in klist1])
        uni=np.array([[sclin.eigh(h)[1][:,b] for h in hh] for hh,b in zip(ham1,blist)])
        plot_FS(uni,klist1,olist,eig,X,Y,sw_color)
    elif option==2: #write 3D Fermi surface
        gen_3d_fs_plot(FSmesh,rvec,ham_r,ndegen,mu)
    elif option==3: #write Fermi velocity with Fermi surface
        klist,blist=mk_kf(FSmesh,True,2,rvec,ham_r,ndegen,mu,kz)
        veloc=[[get_vec(k,rvec,ham_r,ndegen)[b].real for k in kk] for b,kk in zip(blist,klist)]
        plot_vec2(veloc,klist)
    elif option==4: #plot spectrum like band plot
        plot_spectrum(ham,spa_length,xticks,mu,eta)
    elif option==5: #plot spectrum at E=EF
        plot_FSsp(ham,mu,X,Y,eta)
    elif option==6: #plot 3D Fermi velocity with Fermi surface
        klist,blist=mk_kf(FSmesh,True,3,rvec,ham_r,ndegen,mu)
        veloc=[[get_vec(k,rvec,ham_r,ndegen)[b].real for k in kk] for b,kk in zip(blist,klist)]
        plot_veloc_FS(veloc,klist)
    elif option==7:
        get_conductivity(FSmesh,rvec,ham_r,ndegen,mu,temp=1.0e-3)
    elif option==8:
        plot_dos(FSmesh,rvec,ham_r,ndegen,mu,no,eta)
    elif option==9:
        get_carrier_num(FSmesh,rvec,ham_r,ndegen,mu)
#--------------------------main program-------------------------------
if __name__=="__main__":
    main()
__license__="""Copyright (c) 2018-2019 K. Suzuki
Released under the MIT license
http://opensource.org/licenses/mit-license.php
"""

reason: import numpy,import scipy


text:
# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import datetime
import json
import logging
import os
import pathlib
import typing

import numpy as np
import pandas as pd
import requests
from google.cloud import storage

def main(
    source_url: str,
    year_report: str,
    api_naming_convention: str,
    target_file: pathlib.Path,
    target_gcs_bucket: str,
    target_gcs_path: str,
    headers: typing.List[str],
    rename_mappings: dict,
    pipeline_name: str,
    geography: str,
    report_level: str,
    concat_col: typing.List[str],
) -> None:

    logging.info(
        f"ACS {pipeline_name} process started at "
        + str(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
    )

    logging.info("Creating 'files' folder")
    pathlib.Path("./files").mkdir(parents=True, exist_ok=True)

    json_obj_group_id = open("group_ids.json")
    group_id = json.load(json_obj_group_id)

    json_obj_state_code = open("state_codes.json")
    state_code = json.load(json_obj_state_code)

    logging.info("Extracting the data from API and loading into dataframe...")
    if report_level == "national_level":
        df = extract_data_and_convert_to_df_national_level(
            group_id, year_report, api_naming_convention, source_url
        )
    elif report_level == "state_level":
        df = extract_data_and_convert_to_df_state_level(
            group_id, state_code, year_report, api_naming_convention, source_url
        )

    logging.info("Replacing values...")
    df = df.replace(to_replace={"KPI_Name": group_id})

    logging.info("Renaming headers...")
    rename_headers(df, rename_mappings)

    logging.info("Creating column geo_id...")
    if geography == "censustract" or geography == "blockgroup":
        df["tract"] = df["tract"].apply(pad_zeroes_to_the_left, args=(6,))
        df["state"] = df["state"].apply(pad_zeroes_to_the_left, args=(2,))
        df["county"] = df["county"].apply(pad_zeroes_to_the_left, args=(3,))

    df = create_geo_id(df, concat_col)

    logging.info("Pivoting the dataframe...")
    df = df[["geo_id", "KPI_Name", "KPI_Value"]]
    df = df.pivot_table(
        index="geo_id", columns="KPI_Name", values="KPI_Value", aggfunc=np.sum
    ).reset_index()

    logging.info("Reordering headers...")
    df = df[headers]

    logging.info(f"Saving to output file.. {target_file}")
    try:
        save_to_new_file(df, file_path=str(target_file))
    except Exception as e:
        logging.error(f"Error saving output file: {e}.")

    logging.info(
        f"Uploading output file to.. gs://{target_gcs_bucket}/{target_gcs_path}"
    )
    upload_file_to_gcs(target_file, target_gcs_bucket, target_gcs_path)

    logging.info(
        f"ACS {pipeline_name} process completed at "
        + str(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
    )

def string_replace(source_url, replace: dict) -> str:
    for k, v in replace.items():
        source_url_new = source_url.replace(k, v)
    return source_url_new

def extract_data_and_convert_to_df_national_level(
    group_id: dict, year_report: str, api_naming_convention: str, source_url: str
) -> pd.DataFrame:
    list_temp = []
    for key in group_id:
        logging.info(f"reading data from API for KPI {key}...")
        str1 = source_url.replace("~year_report~", year_report)
        str2 = str1.replace("~group_id~", key[0:-3])
        str3 = str2.replace("~row_position~", key[-3:])
        source_url_new = str3.replace("~api_naming_convention~", api_naming_convention)
        try:
            r = requests.get(source_url_new, stream=True)
            logging.info(f"Source url : {source_url_new}")
            logging.info(f"status code : {r.status_code}")
            if r.status_code == 200:
                text = r.json()
                frame = load_nested_list_into_df_without_headers(text)
                frame["KPI_Name"] = key
                list_temp.append(frame)
        except OSError as e:
            logging.info(f"error : {e}")
            pass
    logging.info("creating the dataframe...")
    df = pd.concat(list_temp)
    return df

def load_nested_list_into_df_without_headers(text: typing.List) -> pd.DataFrame:
    frame = pd.DataFrame(text)
    frame = frame.iloc[1:, :]
    return frame

def extract_data_and_convert_to_df_state_level(
    group_id: dict,
    state_code: dict,
    year_report: str,
    api_naming_convention: str,
    source_url: str,
) -> pd.DataFrame:
    list_temp = []
    for key in group_id:
        for sc in state_code:
            logging.info(f"reading data from API for KPI {key}...")
            logging.info(f"reading data from API for KPI {sc}...")
            str1 = source_url.replace("~year_report~", year_report)
            str2 = str1.replace("~group_id~", key[0:-3])
            str3 = str2.replace("~row_position~", key[-3:])
            str4 = str3.replace("~api_naming_convention~", api_naming_convention)
            source_url_new = str4.replace("~state_code~", sc)
            try:
                r = requests.get(source_url_new, stream=True)
                logging.info(f"Source url : {source_url_new}")
                logging.info(f"status code : {r.status_code}")
                if r.status_code == 200:
                    text = r.json()
                    frame = load_nested_list_into_df_without_headers(text)
                    frame["KPI_Name"] = key
                    list_temp.append(frame)
            except OSError as e:
                logging.info(f"error : {e}")
                pass

    logging.info("creating the dataframe...")
    df = pd.concat(list_temp)
    return df

def create_geo_id(df: pd.DataFrame, concat_col: str) -> pd.DataFrame:
    df["geo_id"] = ""
    for col in concat_col:
        df["geo_id"] = df["geo_id"] + df[col]
    return df

def pad_zeroes_to_the_left(val: str, length: int) -> str:
    if len(str(val)) < length:
        return ("0" * (length - len(str(val)))) + str(val)
    else:
        return str(val)

def rename_headers(df: pd.DataFrame, rename_mappings: dict) -> None:
    rename_mappings = {int(k): str(v) for k, v in rename_mappings.items()}
    df.rename(columns=rename_mappings, inplace=True)

def save_to_new_file(df: pd.DataFrame, file_path: str) -> None:
    df.to_csv(file_path, index=False)

def upload_file_to_gcs(file_path: pathlib.Path, gcs_bucket: str, gcs_path: str) -> None:
    storage_client = storage.Client()
    bucket = storage_client.bucket(gcs_bucket)
    blob = bucket.blob(gcs_path)
    blob.upload_from_filename(file_path)

if __name__ == "__main__":
    logging.getLogger().setLevel(logging.INFO)

    main(
        source_url=os.environ["SOURCE_URL"],
        year_report=os.environ["YEAR_REPORT"],
        api_naming_convention=os.environ["API_NAMING_CONVENTION"],
        target_file=pathlib.Path(os.environ["TARGET_FILE"]).expanduser(),
        target_gcs_bucket=os.environ["TARGET_GCS_BUCKET"],
        target_gcs_path=os.environ["TARGET_GCS_PATH"],
        headers=json.loads(os.environ["CSV_HEADERS"]),
        rename_mappings=json.loads(os.environ["RENAME_MAPPINGS"]),
        pipeline_name=os.environ["PIPELINE_NAME"],
        geography=os.environ["GEOGRAPHY"],
        report_level=os.environ["REPORT_LEVEL"],
        concat_col=json.loads(os.environ["CONCAT_COL"]),
    )

max_stars_repo_name: renovate-bot/public-datasets-pipelines

max_issues_repo_name: renovate-bot/public-datasets-pipelines

max_forks_repo_name: renovate-bot/public-datasets-pipelines


text:
import numbers
import warnings
from contextlib import contextmanager
from functools import wraps
from typing import List, Callable

import numpy as np
import torch
import torch.fft
import torch.nn.functional as torchf

from phi.math import DType
from phi.math.backend import Backend, NUMPY, ComputeDevice
from phi.math.backend._backend import combined_dim, SolveResult

class TorchBackend(Backend):

    def __init__(self):
        cpu = NUMPY.cpu
        self.cpu = ComputeDevice(self, "CPU", 'CPU', cpu.memory, cpu.processor_count, cpu.description, ref='cpu')
        Backend.__init__(self, 'PyTorch', default_device=self.cpu)

    def prefers_channels_last(self) -> bool:
        return False

    def list_devices(self, device_type: str or None = None) -> List[ComputeDevice]:
        devices = []
        if device_type in (None, 'CPU'):
            devices.append(self.cpu)
        if device_type in (None, 'GPU'):
            for index in range(torch.cuda.device_count()):
                properties = torch.cuda.get_device_properties(index)
                devices.append(ComputeDevice(self,
                                             properties.name,
                                             'GPU',
                                             properties.total_memory,
                                             properties.multi_processor_count,
                                             f"compute capability {properties.major}.{properties.minor}",
                                             ref=f'cuda:{index}'))
        return devices

    def is_tensor(self, x, only_native=False):
        if isinstance(x, (torch.Tensor, JITFunction)):
            return True
        if only_native:
            return False
        if isinstance(x, numbers.Number):
            return True
        if isinstance(x, (tuple, list)) and all(isinstance(c, numbers.Number) for c in x):
            return True
        if isinstance(x, np.ndarray) and x.dtype != np.object:
            return True  # this is pretty much required, else we couldn't perform NP+PyTorch operations
        return False

    def as_tensor(self, x, convert_external=True):
        if self.is_tensor(x, only_native=convert_external):
            tensor = x
        elif isinstance(x, np.ndarray):
            try:
                tensor = torch.from_numpy(x)
            except ValueError:  # or TypeError?
                tensor = torch.from_numpy(x.copy())
            tensor = tensor.to(self.get_default_device().ref)
        elif isinstance(x, (tuple, list)):
            try:
                tensor = torch.tensor(x, device=self.get_default_device().ref)
            except ValueError:  # there may be Tensors inside the list
                components = [self.as_tensor(c) for c in x]
                tensor = torch.stack(components, dim=0)
        else:
            tensor = torch.tensor(x, device=self.get_default_device().ref)
        # --- Enforce Precision ---
        if self.is_tensor(tensor, only_native=True):
            dtype = self.dtype(tensor)
            if dtype.kind == float:
                tensor = self.to_float(tensor)
            elif dtype.kind == complex:
                tensor = self.to_complex(tensor)
        return tensor

    def auto_cast(self, *tensors) -> list:
        tensors = [t if isinstance(t, (torch.Tensor, numbers.Number, bool)) else self.as_tensor(t, True) for t in tensors]
        return Backend.auto_cast(self, *tensors)

    def is_available(self, tensor) -> bool:
        return torch._C._get_tracing_state() is None  # TODO can we find out whether this tensor specifically is being traced?

    def numpy(self, tensor):
        if tensor.requires_grad:
            return tensor.detach().cpu().numpy()
        else:
            return tensor.cpu().numpy()

    def to_dlpack(self, tensor):
        from torch.utils import dlpack
        return dlpack.to_dlpack(tensor)

    def from_dlpack(self, capsule):
        from torch.utils import dlpack
        tensor = dlpack.from_dlpack(capsule)
        tensor = tensor.to(self.get_default_device().ref)
        return tensor

    def copy(self, tensor, only_mutable=False):
        return torch.clone(tensor)

    sqrt = torch.sqrt
    exp = torch.exp
    sin = torch.sin
    cos = torch.cos
    tan = torch.tan
    log = torch.log
    log2 = torch.log2
    log10 = torch.log10
    isfinite = torch.isfinite
    abs = torch.abs
    sign = torch.sign
    round = torch.round
    ceil = torch.ceil
    floor = torch.floor
    nonzero = torch.nonzero
    flip = torch.flip
    seed = staticmethod(torch.manual_seed)
    einsum = staticmethod(torch.einsum)

    def jit_compile(self, f: Callable) -> Callable:
        return JITFunction(f)

    def custom_gradient(self, f: Callable, gradient: Callable = None) -> Callable:
        TRACED_F = {}
        TRACED_B = {}

        class TorchFunction(torch.autograd.Function):

            @staticmethod
            def forward(ctx, *args, **kwargs):
                current_jit = CURRENT_JIT_CALLS[-1] if CURRENT_JIT_CALLS else None
                if torch._C._get_tracing_state() is not None:
                    assert current_jit, "Detected PyTorch tracing outside Φ-Flow"
                    f_ = f

                    def trace_later():
                        TRACED_F[current_jit] = torch.jit.trace(f, args)  # nested traces not allowed in PyTorch
                    current_jit.post_trace.append(trace_later)
                elif CURRENT_JIT_CALLS:
                    jit_f = TRACED_F[CURRENT_JIT_CALLS[-1]]
                    f_ = jit_f
                else:
                    f_ = f
                y = f_(*args, **kwargs)
                ctx.save_for_backward(*args, *y)
                ctx.input_count = len(args)
                ctx.jit_f = current_jit
                return y

            @staticmethod
            def backward(ctx, *grad_args):
                x = ctx.saved_tensors[:ctx.input_count]
                y = ctx.saved_tensors[ctx.input_count:]
                if torch._C._get_tracing_state() is not None:
                    assert CURRENT_JIT_CALLS
                    raise NotImplementedError()
                    # g_ = gradient
                    #
                    # def trace_later():
                    #     TRACED_F[current_jit] = torch.jit.trace(f, args)  # nested traces not allowed in PyTorch
                elif ctx.jit_f:
                    jit_f = ctx.jit_f
                    if ctx.jit_f in TRACED_B:
                        g_ = TRACED_B[ctx.jit_f]
                    else:
                        # jit-compile the gradient function
                        # jit-functions cannot return None, so we have to filter out non-required gradients
                        needs_input_grad = ctx.needs_input_grad
                        with ctx.jit_f:
                            def filter_required_grads(*args):
                                grads = gradient(*args)
                                filtered = [g for g, need in zip(grads, needs_input_grad) if need]
                                return filtered

                            needed_g = torch.jit.trace(filter_required_grads, tuple([x, y, grad_args]), check_trace=False)

                            def g_(*args):
                                with jit_f:
                                    needed = self.as_registered.call(needed_g, *args, name=f"run jit-compiled custom backward '{gradient.__name__}'")
                                assert not jit_f.post_trace
                                assert isinstance(needed, (tuple, list))
                                needed = list(needed)
                                result = [(needed.pop(0) if need else None) for need in needs_input_grad]
                                return result

                            TRACED_B[ctx.jit_f] = g_
                else:
                    g_ = gradient
                output = g_(x, y, grad_args)
                result = output[0] if len(output) == 1 else (*output, )
                return result

        return TorchFunction.apply

    def transpose(self, tensor, axes):
        return tensor.permute(axes)

    def equal(self, x, y):
        x, y = self.auto_cast(x, y)
        return x == y

    def random_uniform(self, shape):
        return torch.rand(size=shape, dtype=to_torch_dtype(self.float_type), device=self.get_default_device().ref)

    def random_normal(self, shape):
        return torch.randn(size=shape, dtype=to_torch_dtype(self.float_type), device=self.get_default_device().ref)

    def stack(self, values, axis=0):
        values = [self.as_tensor(v) for v in values]
        return torch.stack(values, dim=axis)

    def concat(self, values, axis):
        values = [self.as_tensor(v) for v in values]
        return torch.cat(values, dim=axis)

    def pad(self, value, pad_width, mode='constant', constant_values=0):
        """
        pad tensor using mode

        Args:
          value(torch.Tensor): values
          pad_width(iterable): left, right, upper, lower
          mode(str, optional, optional): type of padding to be applied, defaults to 'constant'
          constant_values(int, optional, optional): value to pad, defaults to 0

        Returns:
          torch.Tensor: padded tensor
        """
        mode = {'constant': 'constant', 'reflect': 'reflect', 'boundary': 'replicate', 'periodic': 'circular'}.get(mode, None)
        if not mode:
            return NotImplemented
        # transpose for leading zero-pad: [(0, 0), (0, 0), ...]
        ndims = self.ndims(value)
        if ndims > 2 and pad_width[0] == pad_width[1] == (0, 0):
            reordered = value
            pad_width_reordered = pad_width[2:]
            undo_transform = lambda x: x
        elif ndims > 2 and pad_width[0] == (0, 0) and self.ndims(value) < 5:
            reordered = torch.unsqueeze(value, 0)
            pad_width_reordered = pad_width[1:]
            undo_transform = lambda x: torch.squeeze(x, 0)
        elif ndims < 4:
            reordered = torch.unsqueeze(torch.unsqueeze(value, 0), 0)
            pad_width_reordered = pad_width
            undo_transform = lambda x: torch.squeeze(torch.squeeze(x, 0), 0)
        else:
            raise NotImplementedError()  # TODO transpose to get (0, 0) to the front
        pad_width_spatial = [item for sublist in reversed(pad_width_reordered) for item in sublist]  # flatten
        try:
            result = torchf.pad(reordered, pad_width_spatial, mode, value=constant_values)  # supports 3D to 5D (2 + 1D to 3D)
        except RuntimeError as err:
            warnings.warn(f"PyTorch error {err}")
            return NotImplemented
        result = undo_transform(result)
        return result

    def grid_sample(self, grid, coordinates, extrapolation='constant'):
        assert extrapolation in ('undefined', 'zeros', 'boundary', 'periodic', 'symmetric', 'reflect'), extrapolation
        extrapolation = {'undefined': 'zeros', 'zeros': 'zeros', 'boundary': 'border', 'reflect': 'reflection'}.get(extrapolation, None)
        if extrapolation is None:
            return NotImplemented
        grid = channels_first(self.as_tensor(grid))
        coordinates = self.as_tensor(coordinates)
        if coordinates.shape[0] != grid.shape[0]:  # repeating yields wrong result
            return NotImplemented
        if coordinates.ndim != grid.ndim or coordinates.ndim not in (4, 5):
            return NotImplemented  # torchf.grid_sample cannot handle this case
        if coordinates.dtype.is_floating_point and not grid.dtype.is_complex and not grid.dtype.is_floating_point:
            grid = self.to_float(grid)
        resolution = torch.tensor(self.staticshape(grid)[2:], dtype=coordinates.dtype, device=coordinates.device)
        coordinates = 2 * coordinates / (resolution - 1) - 1
        coordinates = torch.flip(coordinates, dims=[-1])
        batch_size = combined_dim(coordinates.shape[0], grid.shape[0])
        coordinates = coordinates.repeat(batch_size, *[1] * (len(coordinates.shape-1))) if coordinates.shape[0] < batch_size else coordinates
        grid = grid.repeat(batch_size, *[1] * (len(grid.shape)-1)) if grid.shape[0] < batch_size else grid
        result = torchf.grid_sample(grid, coordinates, mode='bilinear', padding_mode=extrapolation, align_corners=True)  # can cause segmentation violation if NaN or inf are present
        result = channels_last(result)
        return result

    def reshape(self, value, shape):
        return torch.reshape(self.as_tensor(value), shape)

    def sum(self, value, axis=None, keepdims=False):
        if axis is None:
            axis = tuple(range(len(value.shape)))
        return torch.sum(value, dim=axis, keepdim=keepdims)

    def prod(self, value, axis=None):
        if isinstance(axis, (tuple, list)):
            for dim in reversed(sorted(axis)):
                value = torch.prod(value, dim=dim)
            return value
        return torch.prod(value, dim=axis)

    def any(self, boolean_tensor, axis=None, keepdims=False):
        boolean_tensor = self.as_tensor(boolean_tensor, convert_external=True)
        if axis is None:
            return torch.any(boolean_tensor)
        else:
            axes = axis if isinstance(axis, (tuple, list)) else [axis]
            for axis in reversed(sorted(axes)):
                boolean_tensor = torch.any(boolean_tensor, dim=axis, keepdim=keepdims)
            return boolean_tensor

    def all(self, boolean_tensor, axis=None, keepdims=False):
        boolean_tensor = self.as_tensor(boolean_tensor, convert_external=True)
        if axis is None:
            return torch.all(boolean_tensor)
        else:
            axes = axis if isinstance(axis, (tuple, list)) else [axis]
            for axis in reversed(sorted(axes)):
                boolean_tensor = torch.all(boolean_tensor, dim=axis, keepdim=keepdims)
            return boolean_tensor

    def quantile(self, x, quantiles):
        x = self.to_float(x)
        result = torch.quantile(x, quantiles, dim=-1)
        return result

    def divide_no_nan(self, x, y):
        x, y = self.auto_cast(x, y)
        return divide_no_nan(x, y)

    def where(self, condition, x=None, y=None):
        condition = self.as_tensor(condition).bool()
        x, y = self.auto_cast(x, y)
        return torch.where(condition, x, y)

    def mean(self, value, axis=None, keepdims=False):
        return torch.mean(value, dim=axis, keepdim=keepdims)

    def range(self, start, limit=None, delta=1, dtype: DType = DType(int, 32)):
        if limit is None:
            start, limit = 0, start
        return torch.arange(start, limit, delta, dtype=to_torch_dtype(dtype))

    def zeros(self, shape, dtype=None):
        return torch.zeros(shape, dtype=to_torch_dtype(dtype or self.float_type), device=self.get_default_device().ref)

    def zeros_like(self, tensor):
        return torch.zeros_like(tensor, device=self.get_default_device().ref)

    def ones(self, shape, dtype: DType = None):
        return torch.ones(shape, dtype=to_torch_dtype(dtype or self.float_type), device=self.get_default_device().ref)

    def ones_like(self, tensor):
        return torch.ones_like(tensor, device=self.get_default_device().ref)

    def meshgrid(self, *coordinates):
        coordinates = [self.as_tensor(c) for c in coordinates]
        return torch.meshgrid(coordinates)

    def linspace(self, start, stop, number):
        return torch.linspace(start, stop, number, dtype=to_torch_dtype(self.float_type), device=self.get_default_device().ref)

    def tensordot(self, a, a_axes: tuple or list, b, b_axes: tuple or list):
        return torch.tensordot(a, b, (a_axes, b_axes))

    def matmul(self, A, b):
        if isinstance(A, torch.Tensor) and A.is_sparse:
            result = torch.sparse.mm(A, torch.transpose(b, 0, 1))
            return torch.transpose(result, 0, 1)
        raise NotImplementedError(type(A), type(b))

    def cumsum(self, x, axis: int):
        return torch.cumsum(x, dim=axis)

    def while_loop(self, loop: Callable, values: tuple):
        if torch._C._get_tracing_state() is not None:
            if isinstance(loop, torch.ScriptFunction):
                jit_loop = loop
                while torch.any(values[0]):
                    values = jit_loop(*values)
                return values
            else:
                warnings.warn("Tracing a PyTorch while loop requires an additional tracing pass. You can avoid this by passing a torch.ScriptFunction.")
                raise NotImplementedError()
                # def trace_later():
                #     jit_loop = torch.jit.trace(loop)
                #     @torch.jit.script
                #     def loop_script(values: Tuple[torch.Tensor], loop_script: Callable):
                #         while torch.any(values[0]):
                #             values = loop_script(*values)
                #         return values
                # CURRENT_JIT_CALLS[-1].post_trace.append(trace_later)
        else:
            while torch.any(values[0]):
                values = loop(*values)
            return values

    def max(self, x, axis=None, keepdims=False):
        if axis is None:
            result = torch.max(x)
            if keepdims:
                result = self.expand_dims(result, axis=0, number=self.ndims(x))
            return result
        elif isinstance(axis, (tuple, list)):
            for dim in reversed(sorted(axis)):
                x, _ = torch.max(x, dim=dim, keepdim=keepdims)
            return x
        else:
            return torch.max(x, dim=axis, keepdim=keepdims)[0]

    def min(self, x, axis=None, keepdims=False):
        if axis is None:
            result = torch.min(x)
            if keepdims:
                result = self.expand_dims(result, axis=0, number=self.ndims(x))
            return result
        elif isinstance(axis, (tuple, list)):
            for dim in reversed(sorted(axis)):
                x, _ = torch.min(x, dim=dim, keepdim=keepdims)
            return x
        else:
            return torch.min(x, dim=axis, keepdim=keepdims)[0]

    def maximum(self, a, b):
        a_ = self.as_tensor(a)
        b_ = self.as_tensor(b)
        return torch.max(a_, other=b_)

    def minimum(self, a, b):
        a_ = self.as_tensor(a)
        b_ = self.as_tensor(b)
        return torch.min(a_, other=b_)

    def clip(self, x, minimum, maximum):
        if isinstance(minimum, numbers.Number) and isinstance(maximum, numbers.Number):
            return torch.clamp(self.as_tensor(x), minimum, maximum)
        else:
            return self.maximum(minimum, self.minimum(x, maximum))

    def conv(self, value, kernel, zero_padding=True):
        value = self.as_tensor(value)
        kernel = self.as_tensor(kernel)
        value, kernel = self.auto_cast(value, kernel)
        if zero_padding:
            if all(s % 2 == 1 for s in kernel.shape[3:]):
                padding = [s // 2 for s in kernel.shape[3:]]
            else:
                padding = 0
                value_padding = sum([[s // 2, (s - 1) // 2] for s in kernel.shape[3:]], [])
                value = torchf.pad(value, value_padding)
        else:
            padding = 0
        convf = {3: torchf.conv1d, 4: torchf.conv2d, 5: torchf.conv3d}[len(value.shape)]
        if kernel.shape[0] == 1:
            result = convf(value, kernel[0, ...], padding=padding)
        else:
            result = []
            for b in range(kernel.shape[0]):
                result.append(convf(value[b:b+1, ...], kernel[b, ...], padding=padding))
            result = torch.cat(result, 0)
        return result

    def expand_dims(self, a, axis=0, number=1):
        for _ in range(number):
            a = torch.unsqueeze(a, dim=axis)
        return a

    def shape(self, tensor):
        if self.is_tensor(tensor, only_native=True):
            return tensor.shape
        else:
            return NUMPY.shape(tensor)

    def staticshape(self, tensor):
        if self.is_tensor(tensor, only_native=True):
            return tuple([int(s) for s in tensor.shape])
        else:
            return NUMPY.staticshape(tensor)

    def batched_gather_nd(self, values, indices):
        values = self.as_tensor(values)
        indices = self.as_tensor(indices).long()
        batch_size = combined_dim(values.shape[0], indices.shape[0])
        result = []
        for b in range(batch_size):
            b_indices = self.unstack(indices[min(b, indices.shape[0] - 1)], -1)
            result.append(values[(min(b, values.shape[0] - 1),) + b_indices])
        return self.stack(result, axis=0)

    def unstack(self, tensor, axis=0, keepdims=False):
        unstacked = torch.unbind(tensor, dim=axis)
        if keepdims:
            unstacked = [self.expand_dims(c, axis=axis) for c in unstacked]
        return unstacked

    def std(self, x, axis=None, keepdims=False):
        return torch.std(x, dim=axis, keepdim=keepdims)

    def boolean_mask(self, x, mask, axis=0):
        x = self.as_tensor(x)
        mask = self.as_tensor(mask)
        result = []
        for selected, data in zip(mask, self.unstack(x, axis)):
            if selected:
                result.append(data)
        return self.stack(result, axis)
        # return torch.masked_select(x_, mask_)

    def scatter(self, base_grid, indices, values, mode: str):
        base_grid, values = self.auto_cast(base_grid, values)
        indices = self.as_tensor(indices)
        batch_size = combined_dim(combined_dim(indices.shape[0], values.shape[0]), base_grid.shape[0])
        scatter = torch.scatter_add if mode == 'add' else torch.scatter
        if indices.shape[0] < batch_size:
            indices = indices.repeat([batch_size] + [1] * (len(indices.shape)-1))
        if values.shape[0] < batch_size or values.shape[1] == 1:
            values = values.repeat([batch_size // values.shape[0], indices.shape[1] // indices.shape[1]] + [1] * (len(values.shape)-2))
        if len(base_grid.shape) > 3:
            resolution = base_grid.shape[1:-1]
            ravel = [1]
            for i in range(1, len(resolution)):
                ravel.insert(0, ravel[0] * resolution[-i])
            ravel = self.to_int64(self.as_tensor(ravel, True))
            indices = torch.sum(indices * ravel, dim=-1, keepdim=True)
        base_grid_flat = torch.reshape(base_grid, [base_grid.shape[0], -1, base_grid.shape[-1]])
        indices = indices.long().repeat([1, 1, values.shape[-1]])
        result = scatter(base_grid_flat, dim=1, index=indices, src=values)
        return torch.reshape(result, base_grid.shape)

    def fft(self, x, axes: tuple or list):
        if not x.is_complex():
            x = self.to_complex(x)
        for i in axes:
            x = torch.fft.fft(x, dim=i)
        return x

    def ifft(self, k, axes: tuple or list):
        if not k.is_complex():
            k = self.to_complex(k)
        for i in axes:
            k = torch.fft.ifft(k, dim=i)
        return k

    def imag(self, x):
        dtype = self.dtype(x)
        if dtype.kind == complex:
            return torch.imag(x)
        else:
            return self.zeros(x.shape, DType(float, dtype.precision))

    def real(self, x):
        if self.dtype(x).kind == complex:
            return torch.real(x)
        else:
            return x

    def conj(self, x):
        if self.dtype(x).kind == complex:
            return torch.conj(x)
        else:
            return x

    def cast(self, x, dtype: DType):
        if isinstance(x, (numbers.Number, bool)):
            return x  # Creating a Tensor here would raise warnings during tracing.
        if not self.is_tensor(x, only_native=True):
            x = self.as_tensor(x)
        if self.dtype(x) == dtype:
            return x
        else:
            return x.to(to_torch_dtype(dtype))

    def dtype(self, array) -> DType:
        if self.is_tensor(array, only_native=True):
            return from_torch_dtype(array.dtype)
        else:
            return NUMPY.dtype(array)

    def tile(self, value, multiples):
        if isinstance(multiples, np.ndarray):
            multiples = multiples.tolist()
        return self.as_tensor(value).repeat(multiples)

    def sparse_tensor(self, indices, values, shape):
        indices_ = self.to_int64(indices)
        values_ = self.to_float(values)
        if not self.is_available(values_):
            # the output of torch.sparse_coo_tensor is considered constant
            @torch.jit.script
            def sparse_coo_tensor(values, indices, cols: int, rows: int, dtype: torch.dtype) -> torch.sparse.Tensor:
                size = torch.Size([cols, rows])
                return torch.sparse_coo_tensor(indices, values, size=size, dtype=dtype)
            result = sparse_coo_tensor(values_, indices_, shape[0], shape[1], to_torch_dtype(self.float_type))
        else:
            result = torch.sparse_coo_tensor(indices_, values_, shape, dtype=to_torch_dtype(self.float_type))
        return result

    def coordinates(self, tensor):
        assert isinstance(tensor, torch.Tensor) and tensor.is_sparse
        idx = tensor._indices()
        idx = self.unstack(idx, axis=0)
        return idx, tensor._values()

    def conjugate_gradient(self, lin, y, x0, rtol, atol, max_iter, trj: bool) -> SolveResult or List[SolveResult]:
        if callable(lin) or trj:
            assert self.is_available(y), "Tracing conjugate_gradient with linear operator is not yet supported."
            return Backend.conjugate_gradient(self, lin, y, x0, rtol, atol, max_iter, trj)
        assert isinstance(lin, torch.Tensor) and lin.is_sparse, "Batched matrices are not yet supported"
        y = self.to_float(y)
        x0 = self.copy(self.to_float(x0))
        rtol = self.as_tensor(rtol)
        atol = self.as_tensor(atol)
        max_iter = self.as_tensor(max_iter)
        x, residual, iterations, function_evaluations, converged, diverged = torch_sparse_cg(lin, y, x0, rtol, atol, max_iter)
        return SolveResult(f"Φ-Flow CG ({'PyTorch*' if self.is_available(y) else 'TorchScript'})", x, residual, iterations, function_evaluations, converged, diverged, "")

    def conjugate_gradient_adaptive(self, lin, y, x0, rtol, atol, max_iter, trj: bool) -> SolveResult or List[SolveResult]:
        if callable(lin) or trj:
            assert self.is_available(y), "Tracing conjugate_gradient with linear operator is not yet supported."
            return Backend.conjugate_gradient_adaptive(self, lin, y, x0, rtol, atol, max_iter, trj)
        assert isinstance(lin, torch.Tensor) and lin.is_sparse, "Batched matrices are not yet supported"
        y = self.to_float(y)
        x0 = self.copy(self.to_float(x0))
        rtol = self.as_tensor(rtol)
        atol = self.as_tensor(atol)
        max_iter = self.as_tensor(max_iter)
        x, residual, iterations, function_evaluations, converged, diverged = torch_sparse_cg_adaptive(lin, y, x0, rtol, atol, max_iter)
        return SolveResult(f"Φ-Flow CG ({'PyTorch*' if self.is_available(y) else 'TorchScript'})", x, residual, iterations, function_evaluations, converged, diverged, "")

    def functional_gradient(self, f, wrt: tuple or list, get_output: bool):
        @wraps(f)
        def eval_grad(*args):
            args = [self.as_tensor(arg, True) if i in wrt else arg for i, arg in enumerate(args)]
            for i, arg in enumerate(args):
                if self.is_tensor(arg, True) and arg.requires_grad and not arg.is_leaf:
                    arg = torch.clone(arg).detach()
                    arg.requires_grad = i in wrt
                    args[i] = arg
                elif i in wrt:
                    arg = self.as_tensor(arg, True)
                    arg = arg.detach()  # returns a new tensor in any case
                    arg.requires_grad = True
                    args[i] = arg
            wrt_args = [arg for i, arg in enumerate(args) if i in wrt]
            for t in wrt_args:
                assert t.requires_grad
            output = f(*args)
            loss, aux = (output[0], output[1:]) if isinstance(output, (tuple, list)) else (output, None)
            if loss.ndim > 0:
                loss = loss.sum()
            grads = torch.autograd.grad(loss, wrt_args)  # grad() cannot be called during jit trace
            if get_output:
                loss = loss.detach()
                if aux is not None:
                    aux = [aux_.detach() for aux_ in aux]
                    return (loss, *aux, *grads)
                else:
                    return (loss, *grads)
            else:
                return grads
        return eval_grad

    def jit_compile_grad(self, f, wrt: tuple or list, get_output: bool):
        jit = self.jit_compile(f)
        return self.functional_gradient(jit, wrt, get_output)

    def gradients(self, y, xs: tuple or list, grad_y) -> tuple:
        if self.ndims(y) > 0:
            y = self.sum(y)
        grad = torch.autograd.grad(y, xs, grad_y)
        return grad

    @contextmanager
    def record_gradients(self, xs: tuple or list, persistent=False):
        for x in xs:
            assert self.is_tensor(x, only_native=True), f"Must be a PyTorch tensor but got {x}"
        xs = [x if x.is_leaf else x.detach_() for x in xs]
        assert not any(x.requires_grad for x in xs)
        for x in xs:
            x.requires_grad = True
        try:
            yield None
        finally:
            for x in xs:
                x.requires_grad = False

    def stop_gradient(self, value):
        return value.detach()

def channels_first(x):
    return x.permute(*((0, -1) + tuple(range(1, len(x.shape) - 1))))

def channels_last(x):
    return x.permute((0,) + tuple(range(2, len(x.shape))) + (1,))

class JITFunction:

    def __init__(self, f):
        self.f = f
        self.traced = None
        self.post_trace = []

    def __call__(self, *args, **kwargs):
        if kwargs:
            raise NotImplementedError("kwargs not supported for traced function")
        if self.traced is None:
            with self:
                self.traced = torch.jit.trace(self.f, example_inputs=args, check_trace=False)
        with self:
            from phi.math.backend import choose_backend
            return choose_backend(self).call(self.traced, *args, name=f"run jit-compiled '{self.f.__name__}'")

    def __repr__(self):
        return f"jit-TorchScript[{self.f.__name__}]"

    def __enter__(self):
        CURRENT_JIT_CALLS.append(self)
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        assert CURRENT_JIT_CALLS.pop(-1) == self
        while self.post_trace:
            self.post_trace.pop(0)()

CURRENT_JIT_CALLS: List[JITFunction] = []

def to_torch_dtype(dtype: DType):
    return _TO_TORCH[dtype]

def from_torch_dtype(torch_dtype):
    if torch_dtype in _FROM_TORCH:
        return _FROM_TORCH[torch_dtype]
    else:
        kind = {'i': int, 'b': bool, 'f': float, 'c': complex}[torch_dtype.kind]
        return DType(kind, torch_dtype.itemsize * 8)

_TO_TORCH = {
    DType(float, 16): torch.float16,
    DType(float, 32): torch.float32,
    DType(float, 64): torch.float64,
    DType(complex, 64): torch.complex64,
    DType(complex, 128): torch.complex128,
    DType(int, 8): torch.int8,
    DType(int, 16): torch.int16,
    DType(int, 32): torch.int32,
    DType(int, 64): torch.int64,
    DType(bool): torch.bool,
}
_FROM_TORCH = {np: dtype for dtype, np in _TO_TORCH.items()}

@torch.jit._script_if_tracing
def torch_sparse_cg(lin, y, x0, rtol, atol, max_iter):
    batch_size = y.shape[0]
    tolerance_sq = torch.maximum(rtol ** 2 * torch.sum(y ** 2, -1), atol ** 2)
    x = x0
    dx = residual = y - sparse_matmul(lin, x)
    it_counter = torch.tensor(0, dtype=torch.int32, device=x.device)
    iterations = torch.zeros([batch_size], dtype=torch.int32, device=x.device)
    function_evaluations = torch.ones([batch_size], dtype=torch.int32, device=x.device)
    residual_squared = rsq0 = torch.sum(residual ** 2, -1, keepdim=True)
    diverged = torch.any(~torch.isfinite(x), dim=1)
    converged = torch.all(residual_squared <= tolerance_sq, dim=1)
    finished = converged | diverged | (iterations >= max_iter); not_finished_1 = (~finished).to(torch.int32)
    while ~torch.all(finished):
        it_counter += 1; iterations += not_finished_1
        dy = sparse_matmul(lin, dx); function_evaluations += not_finished_1
        dx_dy = torch.sum(dx * dy, dim=-1, keepdim=True)
        step_size = divide_no_nan(residual_squared, dx_dy)
        step_size *= torch.unsqueeze(not_finished_1.to(y.dtype), -1)  # this is not really necessary but ensures batch-independence
        x += step_size * dx
        if it_counter % 20 == 0:
            residual = y - sparse_matmul(lin, x); function_evaluations += 1
        else:
            residual = residual - step_size * dy  # in-place subtraction affects convergence
        residual_squared_old = residual_squared
        residual_squared = torch.sum(residual ** 2, -1, keepdim=True)
        dx = residual + divide_no_nan(residual_squared, residual_squared_old) * dx
        diverged = torch.any(residual_squared / rsq0 > 100, dim=1) & (iterations >= 8)
        converged = torch.all(residual_squared <= tolerance_sq, dim=1)
        finished = converged | diverged | (iterations >= max_iter); not_finished_1 = (~finished).to(torch.int32)
    return x, residual, iterations, function_evaluations, converged, diverged

@torch.jit._script_if_tracing
def torch_sparse_cg_adaptive(lin, y, x0, rtol, atol, max_iter):
    batch_size = y.shape[0]
    tolerance_sq = torch.maximum(rtol ** 2 * torch.sum(y ** 2, -1), atol ** 2)
    x = x0
    dx = residual = y - sparse_matmul(lin, x)
    it_counter = torch.tensor(0, dtype=torch.int32, device=x.device)
    iterations = torch.zeros([batch_size], dtype=torch.int32, device=x.device)
    function_evaluations = torch.ones([batch_size], dtype=torch.int32, device=x.device)
    residual_squared = rsq0 = torch.sum(residual ** 2, -1, keepdim=True)
    diverged = torch.any(~torch.isfinite(x), dim=1)
    converged = torch.all(residual_squared <= tolerance_sq, dim=1)
    finished = converged | diverged | (iterations >= max_iter); not_finished_1 = (~finished).to(torch.int32)
    while ~torch.all(finished):
        it_counter += 1; iterations += not_finished_1
        dy = sparse_matmul(lin, dx); function_evaluations += not_finished_1
        dx_dy = torch.sum(dx * dy, dim=-1, keepdim=True)
        step_size = divide_no_nan(torch.sum(dx * residual, dim=1, keepdim=True), dx_dy)
        step_size *= torch.unsqueeze(not_finished_1.to(y.dtype), -1)  # this is not really necessary but ensures batch-independence
        x += step_size * dx
        if it_counter % 20 == 0:
            residual = y - sparse_matmul(lin, x); function_evaluations += 1
        else:
            residual = residual - step_size * dy  # in-place subtraction affects convergence
        residual_squared = torch.sum(residual ** 2, -1, keepdim=True)
        dx = residual - divide_no_nan(torch.sum(residual * dy, dim=1, keepdim=True) * dx, dx_dy)
        diverged = torch.any(residual_squared / rsq0 > 100, dim=1) & (iterations >= 8)
        converged = torch.all(residual_squared <= tolerance_sq, dim=1)
        finished = converged | diverged | (iterations >= max_iter); not_finished_1 = (~finished).to(torch.int32)
    return x, residual, iterations, function_evaluations, converged, diverged

def sparse_matmul(matrix: torch.sparse.Tensor, b: torch.Tensor):
    return torch.transpose(torch.sparse.mm(matrix, torch.transpose(b, 0, 1)), 0, 1)

def divide_no_nan(x: torch.Tensor, y: torch.Tensor):
    # --- PyTorch backward pass of where produces nan gradients when inf values are present.
    # Workaround is to avoid zero division by replacing zeros with ones (which then get filtered
    # in the return where). ---
    result = x / torch.where(y == 0, torch.ones_like(y), y)
    result = torch.where(y == 0, torch.zeros_like(result), result)
    return result

max_stars_repo_name: tum-pbs/PhiFlow

max_issues_repo_name: tum-pbs/PhiFlow

max_forks_repo_name: tum-pbs/PhiFlow


text:
from Q50_config import *
import sys, os
from GPSReader import *
from GPSTransforms import *
from VideoReader import *
from LidarTransforms import *
from ColorMap import *
from transformations import euler_matrix
import numpy as np
import cv2
from ArgParser import *
import h5py

WINDOW = 250*2.5

def cloudToPixels(cam, pts_wrt_cam):

    width = 4
    (pix, J)  = cv2.projectPoints(pts_wrt_cam.transpose(), np.array([0.0,0.0,0.0]), np.array([0.0,0.0,0.0]), cam['KK'], cam['distort'])

    pix = pix.transpose()
    pix = np.around(pix[:, 0, :])
    pix = pix.astype(np.int32)
    mask = np.logical_and(True, pix[0,:] > 0 + width/2)
    mask = np.logical_and(mask, pix[1,:] > 0 + width/2)
    mask = np.logical_and(mask, pix[0,:] < 1279 - width/2)
    mask = np.logical_and(mask, pix[1,:] < 959 - width/2)
    mask = np.logical_and(mask, pts_wrt_cam[2,:] > 0)
    dist_sqr = np.sum( pts_wrt_cam[0:3, :] ** 2, axis = 0)
    mask = np.logical_and(mask, dist_sqr > 3)

    return (pix, mask)

def localMapToPixels(map_data, imu_transforms_t, T_from_i_to_l, cam):
    # load nearby map frames
    pts_wrt_imu_0 = array(map_data[:,0:3]).transpose()
    pts_wrt_imu_0 = np.vstack((pts_wrt_imu_0,
        np.ones((1,pts_wrt_imu_0.shape[1]))))
    # transform points from imu_0 to imu_t
    pts_wrt_imu_t = np.dot( np.linalg.inv(imu_transforms_t), pts_wrt_imu_0)
    # transform points from imu_t to lidar_t
    pts_wrt_lidar_t = np.dot(T_from_i_to_l, pts_wrt_imu_t);
    # transform points from lidar_t to camera_t
    pts_wrt_camera_t = pts_wrt_lidar_t.transpose()[:, 0:3] + cam['displacement_from_l_to_c_in_lidar_frame']
    pts_wrt_camera_t = dot(R_to_c_from_l(cam),
            pts_wrt_camera_t.transpose())

    pts_wrt_camera_t = np.vstack((pts_wrt_camera_t,
        np.ones((1,pts_wrt_camera_t.shape[1]))))
    pts_wrt_camera_t = dot(cam['E'], pts_wrt_camera_t)
    pts_wrt_camera_t = pts_wrt_camera_t[0:3,:]

    # reproject camera_t points in camera frame
    (pix, mask) = cloudToPixels(cam, pts_wrt_camera_t)

    return (pix, mask)

def trackbarOnchange(t, prev_t):
    if abs(t - prev_t) > 1:
        video_reader.setFrame(t)

if __name__ == '__main__':
    args = parse_args(sys.argv[1], sys.argv[2])
    cam_num = int(sys.argv[2][-5])
    video_file = args['video']

    params = args['params']
    cam = params['cam'][cam_num-1]
    video_reader = VideoReader(video_file)
    gps_reader = GPSReader(args['gps'])
    GPSData = gps_reader.getNumericData()
    imu_transforms = IMUTransforms(GPSData)

    T_from_i_to_l = np.linalg.inv(params['lidar']['T_from_l_to_i'])

    # Left and right data
    left_file = np.load(sys.argv[3])
    left_data = left_file['data']
    right_file = np.load(sys.argv[4])
    right_data = right_file['data']

    # Label left and right data
    left_data = np.hstack((left_data, np.zeros((left_data.shape[0], 1))))
    right_data = np.hstack((right_data, np.ones((right_data.shape[0], 1))))

    map_data = np.vstack((left_data, right_data))
    print map_data.shape

    #map_data = map_data[map_data[:,3] > 60, :]
    # map points are defined w.r.t the IMU position at time 0
    # each entry in map_data is (x,y,z,intensity,framenum).

    red = [0, 0, 255]
    blue = [255, 0, 0]
    green = [0, 255, 0]

    fps = 30  # PARAM
    fourcc = cv2.cv.CV_FOURCC(*'MJPG')
    video_writer = cv2.VideoWriter()
    video_writer.open('tmp.avi', fourcc, fps, (1280, 960))  # PARAM

    print "Hit 'q' to quit"
    trackbarInit = False
    init = False
    num_skipped = 0

    pts_3d_left = list()
    pts_3d_right = list()
    pix_2d_left = list()
    pix_2d_right = list()
    ts = list()

    while True:
        if not init:
            (success, I) = video_reader.getNextFrame()
            init = True
        else:
            for count in range(10):
                (success, I) = video_reader.getNextFrame()

        if not success:
            break

        t = video_reader.framenum - 1
        print t
        mask_window = (map_data[:,4] < t + WINDOW) & (map_data[:,4] > t );
        map_data_copy = array(map_data[mask_window, :]);

        pts_wrt_imu_0 = np.vstack((map_data_copy[:,0:3].transpose(),
            np.ones((1, map_data_copy.shape[0]))))
        transformed_map_data = np.dot(np.linalg.inv(imu_transforms[t, :, :]), pts_wrt_imu_0)

        far_mask = (np.sqrt(np.sum(np.square(transformed_map_data[0:3,:]), axis=0)) > 165) & (np.sqrt(np.sum(np.square(transformed_map_data[0:3,:]), axis=0)) < 175)
        far_map_data = map_data_copy[far_mask, :]
        close_map_data = map_data_copy[np.logical_not(far_mask), :]

        far_map_data_left = far_map_data[far_map_data[:, 5] == 0, :]
        far_map_data_right = far_map_data[far_map_data[:, 5] == 1, :]
        #print far_map_data_left.shape
        #print far_map_data_right.shape

        # reproject
        #(close_pix, close_mask) = localMapToPixels(close_map_data, imu_transforms[t,:,:], T_from_i_to_l, cam)
        #print far_map_data_right.shape
        if far_map_data_left.shape[0] == 0:
            print 'skipping left', t
            num_skipped += 1
            continue
        if far_map_data_right.shape[0] == 0:
            print 'skipping right', t
            num_skipped += 1
            continue

        far_map_data_left = np.median(far_map_data_left[:, 0:3], axis=0).reshape((1, 3))
        far_map_data_right = np.median(far_map_data_right[:, 0:3], axis=0).reshape((1, 3))

        (far_pix_left, far_mask_left) = localMapToPixels(far_map_data_left, imu_transforms[t,:,:], T_from_i_to_l, cam)
        (far_pix_right, far_mask_right) = localMapToPixels(far_map_data_right, imu_transforms[t,:,:], T_from_i_to_l, cam)

        # Find the closest far pix and get 3d and pixel information

        #closest_left = np.sqrt(np.sum(np.square(far_map_data_left[:, 0:3]), axis=1)).argmin()
        #closest_left = far_map_data_left[:, 4].argmax()
        #closest_right = far_map_data_right[:, 4].argmin()
        #closest_right = np.sum(np.square(far_map_data_right[:, 0:2] - far_map_data_left[closest_left, 0:2])).argmin()

        #pts_3d_left.append(far_map_data_left[closest_left, 0:3])
        #pts_3d_right.append(far_map_data_right[closest_right, 0:3])
        pts_3d_left.append(far_map_data_left.ravel())
        pts_3d_right.append(far_map_data_right.ravel())

        #print far_mask.shape
        #far_mask_left = np.array([far_mask_left[closest_left]])
        #far_mask_right = np.array([far_mask_right[closest_right]])
        #print far_mask.shape

        # draw
        #intensity = map_data_copy[mask, 3]
        #heat_colors = heatColorMapFast(intensity, 0, 100)

        green_colors = np.tile(green, (np.sum(far_mask_left), 1))
        blue_colors = np.tile(blue, (np.sum(far_mask_right), 1))

        #pt_left = tuple(far_pix_left[:, far_mask_left].ravel())
        #pt_right = tuple(far_pix_right[:, far_mask_right].ravel())
        pt_left = tuple(far_pix_left.ravel())
        pt_right = tuple(far_pix_right.ravel())

        pix_2d_left.append(pt_left)
        pix_2d_right.append(pt_right)

        ts.append(t)

        cv2.circle(I, pt_left, 3, green, -1)
        cv2.circle(I, pt_right, 3, blue, -1)
        cv2.line(I, pt_left, pt_right, red, 2)

        #cv2.imshow(video_file, cv2.pyrDown(I))
        #if not trackbarInit:
            #cv2.createTrackbar('trackbar', video_file, 0, int(video_reader.total_frame_count), lambda x: trackbarOnchange(x, t))
            #trackbarInit = True
        #else:
            #cv2.setTrackbarPos('trackbar', video_file, t)

        #keycode = cv2.waitKey(1)
        #if keycode == 113:
            #break

        video_writer.write(I)

    print 'Played %d frames' % t
    print 'skipped', num_skipped
    video_writer.release()

    import IPython; IPython.embed()

    # Save data

    h5f = h5py.File('tmp.h5', 'w')
    h5f['pix_2d_left'] = np.vstack(pix_2d_left)
    h5f['pix_2d_right'] = np.vstack(pix_2d_right)
    h5f['pts_3d_left'] = np.vstack(pts_3d_left)
    h5f['pts_3d_right'] = np.vstack(pts_3d_right)
    h5f['ts'] = np.array(ts)
    h5f.close()

max_stars_repo_name: sameeptandon/sail-car-log

max_issues_repo_name: sameeptandon/sail-car-log

max_forks_repo_name: sameeptandon/sail-car-log


text:
from sklearn.model_selection import train_test_split
import pandas as pd
import numpy as np
from sklearn.linear_model import LinearRegression
from matplotlib import pyplot
from sklearn.metrics import accuracy_score

data=pd.read_csv('C:\\Users\\canara\\Downloads\\crimeML.csv')
#print(data.shape)
#print(data.head)
#print(data.describe)
dataset=data.loc[data['Location']=='A']

dataset.plot()
pyplot.show()
#autocorrelation_plot(data)
z=dataset.pop('Date')
z=dataset.pop('Location')

Y=dataset.pop('Robbery')
X=dataset

xtrain,xtest,ytrain,ytest=train_test_split(X,Y,train_size=0.8)
model=LinearRegression().fit(xtrain,ytrain)
ypred=model.predict(xtest)
print("Predicted number of crimes in Location A :",end="")
print(ypred[0])