import json
import os
from typing import Any, Dict, List, Optional, Union

import torch
from optimum.gptq import GPTQQuantizer
from tqdm import tqdm
from transformers import PreTrainedModel, PreTrainedTokenizer

from .. import enums
from ..core.config import Config
from ..core.dependencies import build_dataset, build_model, build_tokenizer
from ..datasets.registry import datasets_registry
from ..utils.logger import dist_logger
from ..utils.post_training import push_to_hub_bos_add_bos_token

class Quantizer:
    """
    The `Quantizer` class is responsible for the quantization of pretrained language models using the GPTQ approach
    provided by the Optimum library. Quantization is the process of reducing the precision of the model's weights,
    which can lead to decreased model size and potentially faster inference times while maintaining comparable accuracy.

    The class provides tools to set up the quantization environment, execute the quantization, and save the resultant
    quantized model.

    Thanks to Phil Schmid's post: https://www.philschmid.de/gptq-llama

    Key methods:
    - `__init__`: Initializes the Quantizer with a configuration object and optional tokenizer, model, and dataset.
    - `build`: Sets up the tokenizer, model, and dataset if not provided during initialization and prepares the
        internal `GPTQQuantizer` for the quantization process.
    - `build_dataset`: Constructs or retrieves the dataset to calibrate the model for the quantization process.
    - `quantize`: Runs the actual quantization process by fine-tuning quantization scales with the dataset and
        quantizing model weights.
    - `save`: Saves the quantized model to disk and optionally uploads it to the Hugging Face model hub.

    Attributes:
    - `config` (`Config`): Configuration parameters for building the tokenizer, model, creating the dataset, and
        quantization settings.
    - `tokenizer` (`PreTrainedTokenizer`, defaults to `None`): The tokenizer to format the input for the model
        to be quantized.
    - `model` (`PreTrainedModel`, optional): The pretrained language model to be quantized.
    - `dataset` (`Union[str, List[str], None]`, defaults to `None`): Identifier or samples for the dataset used during
        quantization calibration.
    - `low_cpu_mem_usage` (`bool`, defaults to `None`): Whether to use optimized settings to lower CPU memory usage
        during quantization.
    - `quantizer` (`GPTQQuantizer`, defaults to `None`): The Optimum library's quantizer instance to perform
        the quantization.
    - `quantized_model` (`PreTrainedModel`, defaults to `None`): The resultant quantized model post-quantization
      process.

    This class should be used when there is a need to quantize language models for deployment or when optimizing models
    for environments with resource constraints or specific performance targets.
    """

    def __init__(
        self,
        config: Config,
        tokenizer: Optional[PreTrainedTokenizer] = None,
        model: Optional[PreTrainedModel] = None,
        dataset: Union[str, List[str], None] = None,
        low_cpu_mem_usage: Optional[bool] = None,
    ):
        """
        Initializes the Quantizer object which is responsible for the quantization of a pretrained language model.

        The quantization process aims to reduce the precision of the model's weights to a specific bit-width to optimize
        its size and, potentially, inference speed, often with minimal impact on performance.

        Args:
            config (`Config`):
                The configuration object containing parameters for model building, tokenizer, datasets,
                and quantization.
            tokenizer (`PreTrainedTokenizer`, defaults to `None`):
                The tokenizer associated with the model that will be quantized. It is required for the quantization
                process as it formats the input for the model. If not provided, it will be built from the configuration.
            model (`PreTrainedModel`, optional):
                The pretrained model to be quantized. If not provided, it will be built from the configuration.
            dataset (`Union[str, List[str], None]`, defaults to `None`):
                The dataset to be used for calibrating the quantization process. It can be either a dataset identifier
                string or a list of text samples. If not provided, it will be built from the configuration.
            low_cpu_mem_usage (`bool`, defaults to `None`):
                If set to `True`, the model will attempt to use less CPU memory during quantization which can be
                beneficial when working with large models on machines with limited resources.

        The initializer of the Quantizer sets up the necessary components such as the tokenizer, model, and dataset,
        which will be used for quantization. If any of these components are not provided, they will be constructed based
        on the specified configuration. The `low_cpu_mem_usage` parameter is an important consideration for systems with
        restricted resources, and it's recommended to be set to `True` for quantization.

        Quantizer uses the `GPTQQuantizer` class from Optimum library for the quantization process. The attributes
        `quantizer` and `quantized_model` are initialized as `None` and will be created and filled during the build and
        quantize steps, respectively.

        Upon initialization, this class also performs internal checks like the availability of CUDA and whether all
        necessary settings for automatic GPT quantization are provided within the configuration.
        """
        self.config = config

        self.tokenizer = tokenizer
        self.model = model
        self.dataset = dataset

        self.low_cpu_mem_usage = low_cpu_mem_usage

        self.quantizer: Optional[GPTQQuantizer] = None
        self.quantized_model: Optional[PreTrainedModel] = None

    def internal_checks(self) -> None:
        if not torch.cuda.is_available():
            dist_logger.warning("CUDA is not available")

        self.config.check_auto_gptq()

    def build(self) -> None:
        """
        Builds the necessary components for the quantization process by performing internal checks, constructing
        the tokenizer, model and building/verifying the dataset.

        This method prepares the `Quantizer` instance to perform the quantization process on the language model.
        It initializes internal class attributes such as tokenizer, model, and dataset if they are not already provided.

        The method performs the following steps:
        - Validates the availability of CUDA and performs checks based on the configuration settings for quantization.
        - Constructs the tokenizer if it wasn't provided during Quantizer initialization. The tokenizer is essential
          for formatting the input data for the model.
        - Constructs the model if it wasn't provided during Quantizer initialization. A warning is issued if the
          `low_cpu_mem_usage` attribute is not set since quantization can be resource-intensive.
        - Sets up the quantizer instance of `GPTQQuantizer` class, configured with bits, group size, dataset, and
          model sequence length based on the provided configuration.

        Post execution, the Quantizer is set up with a tokenizer and model ready for quantization, as well as a
        `GPTQQuantizer` instance initialized with the appropriate configuration and dataset.

        Does not return any value.

        Raises:
            - ValueError if the dataset class corresponding to the `config.dataset_key` is not found in the registry.
            - ValueError if the quantization dataset cannot be loaded.

        Note: If `low_cpu_mem_usage` is not specified or set to `False`, a warning is given to consider setting it to
        `True` for quantization, especially when dealing with large models and systems with limited resources.
        """
        self.internal_checks()

        if self.tokenizer is None:
            self.tokenizer = build_tokenizer(config=self.config, use_fast=False)
            dist_logger.info(f"Tokenizer {self.config.correct_tokenizer_name_or_path} was built")

        if self.model is None:
            if self.low_cpu_mem_usage is None or not self.low_cpu_mem_usage:
                dist_logger.warning("low_cpu_mem_usage is None. Recommended to set to True for quantization")
            self.model = build_model(
                config=self.config,
                quantization_config=None,
                low_cpu_mem_usage=self.low_cpu_mem_usage,
            )
            dist_logger.info(f"Model {self.config.model_name_or_path} was built")

        dataset = self.build_dataset() if self.dataset is None else self.dataset

        self.quantizer = GPTQQuantizer(
            bits=self.config.gptq_bits,
            group_size=self.config.gptq_group_size,
            dataset=dataset,
            model_seqlen=self.config.max_length,
        )
        dist_logger.info("Quantizer loaded")

        return None

    def build_dataset(self) -> Union[str, List[str]]:
        """
        Constructs or retrieves the dataset to be used for calibrating the quantization process of the model.

        The dataset is a critical component for the quantization process as it is used to fine-tune the quantization
        scales on actual data, ensuring model accuracy is preserved post-quantization.

        The method performs the following steps:
        - If a dataset ID is specified in the configuration for quantization, this ID is used directly.
        - If no dataset ID is provided, but `prepare_dataset` is enabled in the configuration, the dataset class
            associated
          with `config.dataset_key` is used to prepare the data.
        - If neither of the above are provided or applicable, the method attempts to build the dataset by leveraging
          the configuration settings and dataset preparation procedures.

        During the process, it will:
        - Fetch and combine text parts from the raw dataset into sample texts for quantization.
        - Limit the total number of samples to `config.quantization_max_samples` if this value is set.

        Returns:
            `Union[str, List[str]]`: A dataset identifier string if the dataset comes from a repository or a list
            of text samples for calibrating quantization.

        Raises:
            ValueError: If the dataset class for `config.dataset_key` is not found in the datasets registry or
            if the dataset cannot be built or loaded according to the configuration specifications.

        This method ensures that a suitable and correctly formatted dataset is available for quantizing the model.
        """
        dataset_id = None
        samples: List[str] = list()

        if self.config.quantization_dataset_id is not None:
            dataset_id = self.config.quantization_dataset_id
        else:
            if self.config.prepare_dataset:
                dataset_cls = datasets_registry.get(self.config.dataset_key)

                if dataset_cls is None:
                    raise ValueError(f"Dataset with key {self.config.dataset_key} not found")

                dataset_cls.prepare(config=self.config)

            raw_dataset = build_dataset(config=self.config, is_train=True)

            if raw_dataset is None:
                raise ValueError("Quantization dataset can't be loaded")

            samples = list()

            for sample_index in tqdm(
                range(len(raw_dataset)), desc="Loading quantization dataset", total=self.config.quantization_max_samples
            ):
                sample: Dict[str, Any] = raw_dataset[sample_index]
                text_parts = sample[enums.General.text_parts]
                text = "\n".join(text_parts)
                if isinstance(text, str):
                    samples.append(text)

                if 0 < self.config.quantization_max_samples == len(samples):
                    break

        return dataset_id or samples

    def quantize(self) -> None:
        """
        Executes the quantization process on the pre-loaded pretrained language model using the `GPTQQuantizer`.

        This is the core method where the quantization scales are fine-tuned based on the provided dataset and the model
        weights are quantized to the specified bit-width.

        The method performs the following steps:
        - Validates that the tokenizer, model, and quantizer have been set up by calling `build`. If any of these
          components are not initialized, a ValueError is raised.
        - Proceeds with the quantization process, during which the `GPTQQuantizer` adjusts the quantization parameters
          based on the provided dataset and quantizes the model's weights.
        - Stores the resulting quantized model in the `quantized_model` attribute.

        Does not return any value.

        Raises:
            ValueError: If the tokenizer, model, or quantizer is not initialized before calling this method.

        Note:
            The quantized model is stored within the `quantized_model` attribute of the `Quantizer` instance and can be
            accessed or saved after quantization.
        """
        if self.tokenizer is None:
            raise ValueError("tokenizer is None. It is impossible to quantize. Please run build")

        if self.model is None:
            raise ValueError("model is None. It is impossible to quantize. Please run build")

        if self.quantizer is None:
            raise ValueError("quantizer is None. It is impossible to quantize. Please run build")

        dist_logger.info("Start quantization")
        self.quantized_model = self.quantizer.quantize_model(self.model, self.tokenizer)
        dist_logger.info("Quantization complete")

        return None

    def save(self) -> None:
        """
        Saves the quantized model to a specified directory and pushes it to the Hugging Face model hub if desired.

        This method deals with the post-quantization step of persisting the quantized model for future use. It ensures
        that the model is stored in an accessible location and is properly serialized.

        The method performs the following steps:
        - Checks if the quantized model is present. If not, raises a ValueError as there is nothing to save.
        - Saves the quantized model to the path specified in `config.quantized_model_path`.
        - Saves the tokenizer associated with the quantized model to the same path.
        - Adjusts the configuration file for the quantized model to ensure that any `disable_exllama` flags are set
          correctly and in sync with the quantizer's configuration.
        - Optionally, if a `quantized_hub_model_id` is specified in the config, the method pushes the quantized model
          and tokenizer to the Hugging Face model hub using this ID.

        Does not return any value.

        Raises:
            ValueError: If `quantized_model` is not set, indicating that there is no model to save.

        Note:
            The method logs appropriate messages to keep the user informed about the saving process and any issues that
            might occur, such as the absence of a `quantized_hub_model_id` for hub uploads. It's essential to ensure
            that the model hub ID is set if pushing the model to the cloud is intended.
        """
        if self.quantized_model is None:
            raise ValueError("quantized_model is None. Nothing to save")

        dist_logger.info(f"Saving quantized model to {self.config.quantized_model_path}")
        self.quantized_model.save_pretrained(
            save_directory=self.config.quantized_model_path,
            safe_serialization=self.config.save_safetensors,
        )

        fast_tokenizer = build_tokenizer(config=self.config)

        fast_tokenizer.save_pretrained(save_directory=self.config.quantized_model_path)

        path_to_config = os.path.join(self.config.quantized_model_path, "config.json")
        path_to_quantize_config = os.path.join(self.config.quantized_model_path, "quantize_config.json")

        if self.quantizer is not None:
            with open(
                path_to_quantize_config,
                "w",
                encoding="utf-8",
            ) as file_object:
                self.quantizer.disable_exllama = False
                json.dump(self.quantizer.to_dict(), file_object, indent=2)
        else:
            dist_logger.error("quantizer is None. saved quantized model can be broken")

        with open(path_to_config, "r", encoding="utf-8") as file_object:
            model_config = json.load(file_object)
            model_config["quantization_config"]["disable_exllama"] = False

        with open(path_to_config, "w", encoding="utf-8") as file_object:
            json.dump(model_config, file_object, indent=2)

        if self.config.quantized_hub_model_id is not None:
            dist_logger.info(f"Push quantized model to the hub {self.config.quantized_hub_model_id}")
            self.quantized_model.push_to_hub(
                repo_id=self.config.quantized_hub_model_id,
                private=self.config.quantized_hub_private_repo,
                safe_serialization=self.config.save_safetensors,
                max_shard_size=self.config.max_shard_size,
            )
            if fast_tokenizer is not None:
                fast_tokenizer.push_to_hub(
                    repo_id=self.config.quantized_hub_model_id,
                    private=self.config.quantized_hub_private_repo,
                    safe_serialization=self.config.save_safetensors,
                )
                if self.config.push_to_hub_bos_add_bos_token:
                    push_to_hub_bos_add_bos_token(repo_id=self.config.quantized_hub_model_id)
        else:
            dist_logger.warning("quantized_hub_model_id is None. Model will stay locally")


#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Created on Sat Jun 24 15:22:49 2017
@author: rfrazin
"""

import matplotlib.pyplot as plt
import numpy as np
import scipy.optimize as so
import scipy.sparse as sparsemat
import time

# NDim 1 for 1D simulation, 2 for 2D simulation
# basis - basis set for phase reconstruction.  must be
#    supported by a member function
# npup - number of sample points across the pupil
# npad - size of the padded array for the 1st FFT
# pupil_geom - 'None' or 'circle', for square or circular pupil
# lam - wavelength in microns
# d - reduced beam diameter (feeding WFS)
# D - telescope diamter (after pupil mask)
# fl - focal length in units of d
# slope - angle of pyramid in deg
# indref - index of  refraction at lam
# rotate rotation angle of PyWFS (degrees)
# self.PointWiseOrthogonalBasis is True when the basis functions have no
#     overlapping support points
class Pyr():
    def __init__(self, NDim=2, basis='PhasePixel', RegMat='D1', npup=33,
                 npad=1024, pupil_geom='circle', lam=0.85, d=7.2e3, D=7.992e6,
                 fl=40, slope=3.734, indref=1.452, rotate=45.):
        slope = np.tan(slope*np.pi/180.)
        self.NDim = NDim
        self.basis_type = basis
        self.pupil_geom = pupil_geom
        self.PointWiseOrthogonalBasis = False  # property of basis functions
        self.npup = npup
        self.npad = npad
        self.rotate = rotate
        self.x = np.linspace(-1, 1, npup)
        self.RegScale = None # needed for regularization (scale param)
        self.RegHess = None  #  needed for regularized Hessian
        self.AmpConst = None # needed for magnitude constraint
        fcl = fl*d
        if (npup == 129) and (npad == 4096):
            fscale = lam*fcl/d/32.  # scaling for dist in focal plane
            self.il0, self.il1 = 1833, 1961  # range of left pupil image
            self.ir0, self.ir1 = 2135, 2263  #         right
            if NDim == 1:
                self.nd = 2 + self.il1 - self.il0 + self.ir1 - self.ir0
            else:
                raise Exception('Arrays Too Big for 2D.')
        elif (npup == 33) and (npad == 1024):
            fscale = lam*fcl/d/31.
            if NDim == 2:
                if self.npup == 33:
                    nn = 62
                self.i0 = npad/2 - nn
                self.i1 = npad/2 + nn
                self.nd = (2*nn + 1)*(2*nn + 1)
                self.npix = 1 + 2*nn
            else:
                self.nd = 2*npup
        else:
            assert False
        xf = fscale*np.linspace(-npad/2, npad/2-1, npad)  # focal plane coord.
        if NDim == 1:
            zf = - np.abs(xf)*slope # height of prism surface
            self.pramp = np.exp(-1j*zf*(indref - 1)*2*np.pi/lam)  # phase ramp
        else:

            # index functions for pupil plane
            self.UIp = lambda i: np.unravel_index(i, (npup, npup))  # 1D -> 2D
            self.IUp = lambda i1, i2: np.ravel_multi_index((i1, i2), (npup, npup))
            # index functions for detector plane
            self.UId = lambda i: np.unravel_index(i, (1+2*nn, 1 + 2*nn))
            self.IUd = lambda i1, i2: np.ravel_multi_index((i1, i2), (self.npix, self.npix))
            self.pupil_to_detector = None  #1-to-4 mapping from entrance pupil to detector
            (xf, yf) = np.meshgrid(xf, xf)
            if rotate != 0.:
                rotmat = np.zeros((2,2))
                rotmat[0,0] = np.cos(rotate*np.pi/180)
                rotmat[0,1] = np.sin(rotate*np.pi/180)
                rotmat[1,1] = rotmat[0,0]
                rotmat[1,0] = - rotmat[0,1]
                for k1 in range(npad):
                    for k2 in range(npad):
                        xfyf = rotmat.dot(np.array((xf[k1,k2], yf[k1,k2])))
                        xf[k1,k2] = xfyf[0]
                        yf[k1,k2] = xfyf[1]
            zf = np.zeros((npad, npad))  # height of prism surface
            for k1 in range(npad):
                for k2 in range(npad):
                    angle = np.arctan2(yf[k1,k2], xf[k1,k2])
                    if (angle >= -np.pi/4) and (angle < np.pi/4):
                        zf[k1,k2] = - np.abs(xf[k1,k2])*slope
                    elif (angle >= np.pi/4) and (angle < 3*np.pi/4):
                        zf[k1,k2] = - np.abs(yf[k1,k2])*slope
                    elif (angle >= 3*np.pi/4) or (angle < -3*np.pi/4):
                        zf[k1,k2] = - np.abs(xf[k1,k2])*slope
                    else:
                        zf[k1,k2] = - np.abs(yf[k1,k2])*slope
            self.pramp = np.exp(-1j*zf*(indref - 1)*2*np.pi/lam)  # phase ramp
        # set up a basis set
        if basis == 'ReImPixel':
            self.ReImPixelBasis(RegMat=RegMat)
        elif basis == 'PhasePixel':
            self.PhasePixelBasis(RegMat=RegMat)
        else:
            raise Exception('unknown basis')
        if (pupil_geom == 'circle') and (self.NDim == 2):
            self.SquarePupilToCircle()
        return

    # pyramid operator.  returns the detector field given pupil field g
    # g - pupil plane field
    # FullImage - return entire field on detector plane
    # Block3Faces - allow light only through one face of the pyramid
    # facedex - (0, 1, 2 or 3) index of transmissive face (only matters if Block3Faces)
    def FieldToField(self, g, FullImage=False, Block3Faces=False, facedex=0):
        u = myfft(self.pramp*myfft(myzp(g, self.npad)))
        if Block3Faces:  #only allow light through a single pyramid face
            assert facedex in [0, 1, 2, 3], "facedex must be 0, 1, 2 or 3"
            xf = np.linspace(-self.npad/2, self.npad/2-1, self.npad)  # focal plane coord.
            (xf, yf) = np.meshgrid(xf, xf)
            zf = np.ones(xf.shape)
            if self.rotate != 0.:
                rotmat = np.zeros((2,2))
                rotmat[0,0] = np.cos(self.rotate*np.pi/180)
                rotmat[0,1] = np.sin(self.rotate*np.pi/180)
                rotmat[1,1] = rotmat[0,0]
                rotmat[1,0] = - rotmat[0,1]
                for k1 in range(self.npad):
                    for k2 in range(self.npad):
                        xfyf = rotmat.dot(np.array((xf[k1,k2], yf[k1,k2])))
                        xf[k1,k2] = xfyf[0]
                        yf[k1,k2] = xfyf[1]
            for k1 in range(self.npad):
                for k2 in range(self.npad):
                    angle = np.arctan2(yf[k1,k2], xf[k1,k2])
                    if facedex == 0 and (angle >= -np.pi/4) and (angle < np.pi/4):
                        zf[k1,k2] = 1.
                    elif facedex == 1 and (angle >= np.pi/4) and (angle < 3*np.pi/4):
                        zf[k1,k2] = 1.
                    elif facedex == 2 and ((angle >= 3*np.pi/4) or (angle < - 3*np.pi/4)):
                        zf[k1,k2] = 1.
                    elif facedex == 3 and (angle >= -3*np.pi/4) and (angle < -np.pi/4):
                        zf[k1,k2] = 1.
                    else:
                        zf[k1,k2] = 0.
            u = myfft(self.pramp*zf*myfft(myzp(g, self.npad)))
        if not FullImage:
            u = self.ExtractPupilImage(u)
            u = u.reshape((self.nd,))
        return(u)

    # this extracts the "important" pixels from
    #   the image on the detector
    def ExtractPupilImage(self, h):
        if self.NDim == 1:
            assert len(h) == self.npad
            return(np.hstack((h[self.il0:self.il1+1],
                              h[self.ir0:self.ir1+1])))
        else:
            assert h.shape[0] == self.npad and h.shape[1] == self.npad
            return(h[self.i0:self.i1+1, self.i0:self.i1+1])

    # least-squares solution, response at c0
    # y - data vector
    # wt - weight, can be None
    # c0 - coefficients for gradient evaluation
    # RegParam - regularization parameter, can be None (unregularized solution)
    # PupilImage - take only inensity values from the geometrical images of the pupils
    # slopes - use only 'slopes' instead of raw intesnity values (PupilImage only)
    # returns estimate of coeffienct vector and estimator covariance
    def LeastSqSol(self, y, wt, c0, RegParam=None, ZeroPhaseMean=True,
                   PupilImage=False, slopes=True):
        assert len(c0) == self.nc
        if wt is not None:
            assert len(wt) == len(y)
        if RegParam is not None:  # Cost sets value of self.RegScale
            self.set_RegScale(wt, PupilImage, slopes)
            R = RegParam*self.RegScale*self.RegMat.T.dot(self.RegMat)
        else:
            R = 0.
        if PupilImage:
            if slopes:
                (I0, dI0, I0tot) = self.PupilImageIntensity(c0, g=None, slopes=True)
            else:
                (I0, dI0) = self.PupilImageIntensity(c0, g=None, slopes=False)
        else:  # take pixels from "whole" detector
            (I0, dI0) = self.Intensity(c0)
        y0 = y - I0
        if ZeroPhaseMean:
            y0 = np.hstack((y0, 0))
            if wt is not None:
                wt0 = np.hstack((wt, np.max(wt)))
            newrow = np.zeros(self.nc)
            if self.basis_type == 'ReImPixel':
                rowrange = np.arange(self.nc/2) + self.nc/2
            elif self.basis_type == 'PhasePixel':
                rowrange = np.arange(self.nc)
            else:
                assert False
            maxval = np.max(np.abs(dI0))
            for k in rowrange:
                newrow[k] = maxval
            dI0 = np.vstack((dI0, newrow))
        if wt is None:
            cov = np.linalg.inv(dI0.T.dot(dI0) + R)
            c = cov.dot(dI0.T.dot(y0))
        else:
            W = sparsemat.dia_matrix((wt0, 0), shape=(len(wt0), len(wt0)))
            cov = np.linalg.inv(dI0.T.dot(W.dot(dI0)) + R)
            c = cov.dot(dI0.T.dot(W*y0))
        if len(c.shape) == 2:  # this sucks
            cnew = np.zeros(len(c0))
            if c.shape[0] == 1:
                for k in range(len(c0)):
                    cnew[k] = c[0, k]
            else:
                for k in range(len(c0)):
                    cnew[k] = c[k, 0]
            c = cnew
        c += c0
        return(c, cov)

    # find the coefficient vector that minimizes the weighted cost
    # c0 - initial guess, can be None if LsqStart is set to True
    # y - intensity measurements
    # wt - weights
    # RegParam - regularization parameter
    # method - 'Newton-CG' or 'BFGS'
    # MaxIt - maximum number of iterations
    # LsqStart - start with Linear Least Squares estimate about c0
    # AmpCon - amplitude constraint employed.  Don't do this.
    def FindMinCost(self, c0, y, wt=None, RegParam=None,  method='Newton-CG', MaxIt=10,
                    LsqStart=False, AmpCon=False, ):
        assert method in ['Newton-CG', 'BFGS'], "Invalid method"
        if LsqStart:
            cg, gcg = self.LeastSqSol(y, wt, c0, RegParam, ZeroPhaseMean=True)
        else:
            assert len(c0) == self.nc, "nc parameter set incorrectly"
            cg = c0
        fargs = (y, wt, RegParam, True, AmpCon)
        if method == 'Newton-CG':
            opt = {'disp': True, 'maxiter': MaxIt, 'xtol': 1.e-6, 'return_all': True}
        elif method == 'BFGS':
            opt = {'disp': True, 'maxiter': MaxIt, 'gtol': 1.e-7, 'return_all': True}
        res = so.minimize(self.Cost, cg, args=fargs, method=method,
                          options=opt, jac=True, hess=self.CostHessian)
        sol = res.x
        return(sol, res)

    # calculates the cost function for ReIm basis
    # c - coefficient vector
    # y - measured intensities
    # wt - weight corresponding to y, can be None
    # if ZeroPhaseMean - include ZeroPhaseMean penalty in cost
    # RegParam - bigger is more regularized, None turns it off
    # PupilImage - Consider only pixels in geometrical image of pupil
    #  slopes not implemented, only relevant for PupilImage
    # AmpCon - amplitude inequality penalty for ReImPixel basis,
    #       (assumes max allowed value is 1). This function is
    #       almost convex.   set to 'None' for no constraint
    # returns cost and gradient
    def Cost(self, c, y, wt=None, RegParam=None, ZeroPhaseMean=True,
             PupilImage=False, slopes=False, AmpCon=False):
        assert not PupilImage, "Doesn't work for PupilImage."
        if wt is None:
            wt0 = np.ones(y.shape)
        else:
            assert len(wt) == len(y)
            wt0 = wt
        if (RegParam is not None):
            self.set_RegScale(wt, PupilImage, slopes)
            self.RegHess = self.RegScale*RegParam*self.RegMat.T.dot(self.RegMat)
        else:  # make sparse matrix of zeros
            self.RegHess = sparsemat.dia_matrix((self.RegMat.shape[1], self.RegMat.shape[1]))
        (I, dI) = self.Intensity(c)
        wtIy = wt0*(I - y)
        cost = np.sum(np.power(wtIy, 2))/2.
        dcost = np.zeros(self.nc)
        for k in range(self.nc):
            dcost[k] = np.sum(wtIy*dI[:, k])
        if ZeroPhaseMean:
            if self.basis_type == 'ReImPixel':
                cost += np.max(np.abs(y))*np.max(wt0)*np.power(np.mean(c[self.nc/2:]), 2)/2.
                for k in np.arange(self.nc/2) + self.nc/2:
                    dcost[k] += np.max(np.abs(y))*np.max(wt0)*c[k]/(self.nc/2.)
            elif self.basis_type == 'PhasePixel':
                cost += np.max(np.abs(y))*np.max(wt0)*np.power(np.mean(c), 2)/2.
                for k in np.arange(self.nc):
                    dcost[k] += np.max(np.abs(y))*np.max(wt0)*c[k]/(self.nc)
            else:
                assert False
        if RegParam is not None:
            assert RegParam > 0.
            v = self.RegMat.dot(c)
            cost += RegParam*self.RegScale*v.dot(v)
            dcost += 2*RegParam*self.RegScale*self.RegMat.T.dot(v)
        if not AmpCon:
            return(cost, dcost)
        # implement maximum magnitude inequality penalty - only for ReImPixel basis
        assert self.basis_type == 'ReImPixel'
        MaxSqMag = 1.  # max squared magnitude
        scale = np.power(np.max(wt0)*np.max(y), 2)
        for k in range(self.nc/2):
            l = k + self.nc/2
            smag = c[k]*c[k] + c[l]*c[l]
            if smag > MaxSqMag:  # turn on constraint
                cost += .5*scale*(smag - MaxSqMag)
                dcost[k] += scale*c[k]
                dcost[l] += scale*c[l]
#        if AmpVal is not None:  # this is a non-convex cost -- not useful
#            assert AmpVal > 0.
#            const = self.RegScale*AmpFac/AmpVal/AmpVal
#            self.AmpConst = const
#            for k1 in range(self.nc/2):
#                k2 = k1 + self.nc/2
#                cost += const*np.power(c[k1]*c[k1] + c[k2]*c[k2] - AmpVal, 2)
#                dcost[k1] += 4.*const*c[k1]*(c[k1]*c[k1] + c[k2]*c[k2] - AmpVal)
#                dcost[k1] += 4.*const*c[k2]*(c[k1]*c[k1] + c[k2]*c[k2] - AmpVal)
        return(cost, dcost)

    # sets the value of self.RegScale
    # slopes only has meaning if PupilImage == True
    def set_RegScale(self, wt=None, PupilImage=False, slopes=False):
        if self.basis_type == 'ReImPixel':
            c0 = np.hstack((np.ones(self.nc/2), np.zeros(self.nc/2)))
        elif self.basis_type == 'PhasePixel':
            c0 = np.zeros(self.nc)
        else:
            assert False, "Cost: not set up for this basis."
        if PupilImage and slopes:
            I, dI, Itot = self.PupilImageIntensity(c0, g=None, slopes=True)
            if wt is not None:
                wt0 = wt
            else:
                wt0 = 1.
            self.RegScale = np.sum(np.power(wt0*Itot, 2))/self.nc
            return
        if PupilImage and not slopes:
            I, dI = self.PupilImageIntensity(c0, g=None, slopes=False)
        else:  # not PupilImage
            I, dI = self.Intensity(c0, g=None)
            if wt is None:
                wt0 = 1.
            else:
                wt0 = wt
                assert wt.shape == I.shape
            self.RegScale = np.sum(np.power(wt0*I, 2))/self.nc
            return

    def CostHessian(self, c, y, wt=None, RegParam=None, ZeroPhaseMean=True,
                    AmpCon=False, verbose=True):
        assert len(y) == self.nd
        if wt is None:
            wt0 = np.ones(y.shape)
        else:
            assert len(wt) == self.nd
            wt0 = wt
        if verbose:
            tt0 = time.time()
        (I, dI) = self.Intensity(c, g=None)
        wtIy = wt0*(I - y)

        hes = np.zeros((self.nc, self.nc))
        for k1 in range(self.nc):
            for k2 in range(self.nc):
                if k2 > k1:
                    continue
                hes[k1, k2] = np.sum(wt0*dI[:, k1]*dI[:, k2])
                d2I = self.IntensityHessian([k1, k2], c, g=None)
                hes[k1, k2] += np.sum(wtIy*d2I)
        for k1 in range(self.nc):
            for k2 in range(self.nc):
                if k2 > k1:
                    hes[k1, k2] = hes[k2, k1]

        if ZeroPhaseMean:
            if self.basis_type == 'ReImPixel':
                for k in np.arange(self.nc/2) + self.nc/2:
                    hes[k, k] += np.max(np.abs(y))*np.max(wt0)/(self.nc/2.)
            elif self.basis_type == 'PhasePixel':
                for k in np.arange(self.nc):
                    hes[k, k] += np.max(np.abs(y))*np.max(wt0)/(self.nc)
            else:
                assert False
        if RegParam is not None:
            if self.RegHess is None:  # set it up
                self.Cost(c, y, wt, RegParam=RegParam, ZeroPhaseMean=ZeroPhaseMean, AmpCon=AmpCon)
            hes += self.RegHess
        if verbose:
            print("Hessian calculation time: ", (time.time() - tt0)/60, " minutes.")
        if not AmpCon:
            return(hes)
        # implement maximum magnitude inequality penalty for ReImPixel
        assert self.basis_type == 'ReImPixel'
        MaxSqMag = 1.  # max squared magnitude
        scale = np.power(np.max(wt0)*np.max(y), 2)
        for k in range(self.nc/2):
            l = k + self.nc/2
            smag = c[k]*c[k] + c[l]*c[l]
            if smag > MaxSqMag:  # turn on constraint
                hes[k, k] += scale
                hes[l, l] += scale
#      # non-convex penalty for ReImPixel, so not useful
#            if self.AmpConst is None:  # set it up
#                self.ReImCost(c, y, wt, RegParam=RegParam, AmpVal=1., AmpFac=.1,
#                              ZeroPhaseMean=ZeroPhaseMean)
#            for k1 in range(self.nc/2):
#                k2 = k1 + self.nc/2
#                hes[k1, k1] += 4.*self.AmpConst*(3.*c[k1]*c[k1] + c[k2]*c[k2] - AmpVal)
#                hes[k2, k2] += 4.*self.AmpConst*(c[k1]*c[k1] + 3.*c[k2]*c[k2] - AmpVal)
#                hes[k1, k2] += 8.*self.AmpConst*c[k1]*c[k2]
#                hes[k2, k1] += hes[k1, k2]
        return(hes)

    def CostHessianComponent(self, ind, return_value=False):
        k = self.dct['indlist'][ind]
        k1, k2 = np.unravel_index(k, (self.nc, self.nc))
        print(k, k1, k2)
        d2I = self.IntensityHessian([k1, k2], self.dct['c'], g=None)
        th = np.sum(self.dct['wt0']*self.dct['dI'][:, k1]*self.dct['dI'][:, k2])
        th += np.sum(self.dct['wtIy']*d2I)
        self.temphes[k1, k2] = th
        if return_value:
            return(th)
        else:
            return

    # c is the coefficient vector in the ReImPixels basis.
    # no FullImage capability
    # returns detector itensity and gradient w.t.t. c
    # g argument needed for consistency
    def ReImIntensity(self, c, g=None):
        assert len(c) == self.nc
        ud = np.zeros(self.nd).astype('complex')  # detector field
        for k in range(self.nc):
            ud += c[k]*self.field_grad[:, k]
        I = np.real(ud*np.conj(ud))
        dI = np.zeros((self.nd, self.nc))
        for k in range(self.nc):
            dI[:, k] = 2.*np.real(np.conj(ud)*self.field_grad[:, k])
        return(I, dI)

    # ind - 2-element coefficient index
    #   returns the component of the Intensity Hessian
    #   for all pixels in extraction region
    # c,g arguments needed for consistency
    def ReImIntensityHessian(self, ind, c=None, g=None):
        assert g is None, "Must modify to support field multiplier g."
        assert len(ind) == 2
        hes = self.field_grad[:, ind[0]]*np.conj(self.field_grad[:, ind[1]])
        hes = 2*np.real(hes)
        return(hes)

    # pix - index of desired pixel (can be 1 or 2 numbers for 2D case)
    #   returns the Hessian matrix corresponding to pixel on detector
    # c,g arguments needed for consistency
    def ReImIntensityHessianAtPixel(self, pix, c=None, g=None):
        assert g is None, "Must modify to support field multiplier g."
        if self.NDim == 1:
            assert np.ndim(pix) == 0
            pixel = pix
        else:
            if len(pix) == 2:
                pixel = np.ravel_multi_index((pix[0],pix[1]),(self.npix,self.npix))
        pixhess = np.zeros((self.nc, self.nc))
        for k1 in range(self.nc):
            for k2 in range(self.nc):
                if k2 > k1:
                    continue
                pixhess[k1,k2] = self.ReImIntensityHessian([k1,k2])[pixel]
        for k1 in range(self.nc):
            for k2 in range(self.nc):
                if k2 > k1:
                    pixhess[k1,k2] = pixhess[k2,k1]
        return(pixhess)

    # compute detector field and its gradient given c,
    #  where c is the vector of coefficients in the PhasePixel basis.
    # c - coefficient vector
    # g - field multiplier, can be None
    # In order to avoid recalcuation of gradients, it checks to see if
    #    it has already been calculated for the current value of c
    def PhasePixelToField(self, c, g=None):
        if g is not None:
            gg = g.reshape((self.nc,))
            if np.array_equal(self.c_current, c) and np.array_equal(gg, self.g_current):
                return(self.field_current, self.field_grad_current)
        else:
            if np.array_equal(self.c_current, c) and self.g_current is None:
                return(self.field_current, self.field_grad_current)
        f = np.zeros(self.nd).astype('complex')  # detector field
        gf = np.zeros((self.nd, self.nc)).astype('complex')  # its grad
        for k in range(self.nc):
            s = np.exp(1j*c[k])*self.field_grad_factor[:, k]
            if g is not None:
                s *= gg[k]
            f += s
            gf[:, k] = 1j*s
        self.c_current = 1.*c  # store deep copies
        self.field_current = 1.*f
        self.field_grad_current = 1.*gf
        if g is not None:
            self.g_current = 1.*gg
        return(f, gf)

    # Intensity and its grad for phase-only basis functions
    # c is coefficient vector
    # g is an optional field multiplier
    def PhasePixelIntensity(self, c, g=None):
        field, gfield = self.PhasePixelToField(c, g=g)
        if self.NDim == 2:
            field = field.reshape((self.nd,))
        I = np.real(field*np.conj(field))
        gI = np.zeros(gfield.shape)
        for k in range(self.nc):
            gI[:,k] = 2*np.real(gfield[:,k]*np.conj(field))
        return(I, gI)

    # ind - 2-element coefficient index
    #   returns d^2I/(dc[ind[0]] dc[ind[1]]) for all pixels in extraction region
    def PhasePixelIntensityHessian(self, ind, c, g=None):
        assert g is None, "Must modify to support field multiplier g."
        f, df = self.PhasePixelToField(c, g=g)
        assert len(ind) == 2
        hes = df[:, ind[0]]*np.conj(df[:, ind[1]])
        if ind[0] == ind[1]:
            hes -= np.conj(f)*np.exp(1j*c[ind[0]])*self.field_grad_factor[:,ind[0]]
        hes = 2*np.real(hes)
        return(hes)

    #Returns the intensity Hessian matrix at detector pixel specified by pixind
    #In 2D, pixind can be a 1D or 2D index.
    #c - coef vector
    #g - optional field multiplier
    def PhasePixelIntensityHessianAtPixel(self, pixind, c, g=None):
        assert g is None, "Must modify to support field multiplier g."
        assert self.PointWiseOrthogonalBasis  # otherwise this is MUCH worse
        pix = np.array(pixind).astype('int')  # deep copy
        if self.NDim == 2:
            if len(pix) == 2:
                pix = self.IUd(pix[0],pix[1])
        hesmat = np.zeros((self.nc, self.nc)).astype('complex')
        f, gf = self.PhasePixelToField(c, g)
        for k1 in range(self.nc):
            for k2 in range(self.nc):
                if k2 > k1:
                    continue
                hesmat[k1,k2] = np.conj(gf[pix,k1])*gf[pix,k2]
                if k1 == k2:
                    hesmat[k1,k1] -= np.conj(f[pix])*np.exp(1j*c[k1])*self.field_grad_factor[pix,k1]
        hesmat = 2*np.real(hesmat)
        for k1 in range(self.nc):
            for k2 in range(self.nc):
                if k2 > k1:
                    hesmat[k1,k2] = hesmat[k2,k1]
        return(hesmat)

    # This cuts the pupil grid from a square to a circle
    def SquarePupilToCircle(self):
        assert self.basis_type == 'PhasePixel', "SquareToCircle: Bad basis choice."
        self.pupil_geom = 'circle'
        x = np.linspace(-1, 1, self.npup)
        xg, yg = np.meshgrid(x, x)
        rg = np.sqrt(xg*xg + yg*yg)
        inside = []
        outside = []
        self.pixel_map = dict()  # pixel locations for coefficients
        self.inv_pixel_map = dict()
        count = -1
        for k1 in range(self.npup):
            for k2 in range(self.npup):
                m = self.IUp(k1, k2)
                if rg[k1, k2] > 1.:
                    outside.append(m)
                else:
                    inside.append(m)
                    count += 1
                    self.pixel_map[count] = (k1, k2)
                    self.inv_pixel_map[(k1, k2)] = count
        inside = np.array(inside)
        outside = np.array(outside)
        nin = len(inside)
        basis = np.zeros((self.npup, self.npup, nin))
        field_grad_factor = np.zeros((self.nd, nin)).astype('complex')
        for m in range(len(inside)):
            basis[:, :, m] = self.basis[:, :, inside[m]]
            field_grad_factor[:, m] = self.field_grad_factor[:, inside[m]]

        # splice regularization matrix
        nrow, ncol = self.RegMat.shape
        assert ncol == self.nc
        assert np.mod(nrow, self.nc) == 0
        lmat = nrow/self.nc
        newreg = np.zeros((lmat*nin, nin))  # new RegMat
        rcount = -1
        for k1 in inside:
            rcount += 1
            ccount = -1
            for k2 in inside:
                ccount += 1
                for nm in range(lmat):
                    newreg[rcount + nm*nin, ccount] = RetrieveMatrixElement((k1 + self.nc*nm, k2), self.RegMat)
        del self.RegMat, self.basis, self.field_grad_factor
        self.RegMat = sparsemat.coo_matrix(newreg)
        self.nc = nin
        self.basis = basis
        self.field_grad_factor = field_grad_factor
        if self.rotate == 45.:  # set up pupil-to-detector mapping
            self.pupil_to_detector = dict()
            ph = np.zeros(self.nc)
            for k1 in range(self.nc):
                amp = 0.*ph
                amp[k1] = 1.
                Id, dI = self.PhasePixelIntensity(ph, g=amp)
                Id = Id.reshape(self.npix, self.npix)
                mask = np.zeros((self.npix, self.npix))
                mask[0:self.npix/2, 0:self.npix/2] = 1.
                nw = self.UId(np.argmax(mask*Id))
                mask = 0*mask
                mask[self.npix/2:self.npix, 0:self.npix/2] = 1.
                sw = self.UId(np.argmax(mask*Id))
                mask = 0*mask
                mask[0:self.npix/2, self.npix/2:self.npix] = 1.
                ne = self.UId(np.argmax(mask*Id))
                mask = 0*mask
                mask[self.npix/2:self.npix, self.npix/2:self.npix] = 1.
                se= self.UId(np.argmax(mask*Id))
                self.pupil_to_detector[k1] = [nw, sw, ne, se]
        return

    # return intensities only in the pixel corresponding to the pupil images
    # if slopes, it returns (unnormalized) "slope" values from the intensity
    # normalized only applies to slope measurements, otherwise it does not matter
    def PupilImageIntensity(self, paramvec, g=None, slopes=True, normalized=False):
        assert g==None, "Not implemented for nonzero multiplier g."
        assert self.pupil_to_detector is not None, "must set self.pupil_to_detector"
        sz = len(self.pupil_to_detector)  # = self.nc for PhasePixel
        I, dI = self.Intensity(paramvec, g=g)
        I = I.reshape(self.npix, self.npix)
        dI = dI.reshape(self.npix, self.npix, self.nc)
        if slopes:
            S = np.zeros((2*sz,))
            Itot = np.zeros((2*sz,))  #for calculating shot noise
            dS = np.zeros((2*sz, self.nc))
        else:
            S = np.zeros((4*sz,))
            dS = np.zeros((4*sz, self.nc))
        for k1 in range(sz):
            p2d = self.pupil_to_detector[k1]
            nw = I[p2d[0]]
            sw = I[p2d[1]]
            ne = I[p2d[2]]
            se = I[p2d[3]]
            if slopes:
                S[2*k1 + 0] = nw + sw - ne - se
                S[2*k1 + 1] = nw + ne - sw - se
                Itot[2*k1 + 0] = nw + sw + ne + se
                Itot[2*k1 + 1] = nw + sw + ne + se
                if normalized:
                    S[2*k1 + 0] /= Itot[2*k1 + 0]
                    S[2*k1 + 1] /= Itot[2*k1 + 1]
            else:
                S[4*k1 + 0] = nw
                S[4*k1 + 1] = sw
                S[4*k1 + 2] = ne
                S[4*k1 + 3] = se
            nw = dI[p2d[0][0], p2d[0][1], :].reshape((1, self.nc))
            sw = dI[p2d[1][0], p2d[1][1], :].reshape((1, self.nc))
            ne = dI[p2d[2][0], p2d[2][1], :].reshape((1, self.nc))
            se = dI[p2d[3][0], p2d[3][1], :].reshape((1, self.nc))
            if slopes:
                if normalized:
                    normgrad =  nw + sw + ne + se
                    dS[2*k1 + 0, :] = (nw + sw - ne - se)/Itot[2*k1 + 0] - normgrad*(S[2*k1 + 0]/Itot[2*k1 + 0]**2)
                    dS[2*k1 + 1, :] = (nw + ne - sw - se)/Itot[2*k1 + 1] - normgrad*(S[2*k1 + 1]/Itot[2*k1 + 1]**2)
                else:
                    dS[2*k1 + 0, :] = nw + sw - ne - se
                    dS[2*k1 + 1, :] = nw + ne - sw - se
            else:
                dS[4*k1 + 0, :] = nw
                dS[4*k1 + 1, :] = sw
                dS[4*k1 + 2, :] = ne
                dS[4*k1 + 3, :] = se
        if slopes:
            return(S, dS, Itot)
        else:
            return(S, dS)

    # This function gives the intensity that results from allowing the light
    #  into only of the pyramid faces (at a time) in order to evaluate the
    #  importance of interference.  Assumes cicurlar pupi.
    # phase - the vector of pupil plane phases
    def IntensityBlock3Faces(self, phase, show_indiv=False):
        assert len(phase) == self.nc
        assert self.pupil_geom == 'circle'
        g = np.zeros((self.npup, self.npup)).astype('complex')
        for k in range(len(phase)):
            ki, kj = self.pixel_map[k]
            g[ki, kj] = np.exp(1j*phase[k])
        intensity = 0.*np.real(self.FieldToField(g, FullImage=False, Block3Faces=False, facedex=-1))
        for face in [0, 1, 2, 3]:
            field = self.FieldToField(g, FullImage=False, Block3Faces=True, facedex=face)
            intensity += np.real(field*np.conj(field))
            if show_indiv:
                plt.figure(10+face)
                f = field.reshape(self.npix, self.npix)
                plt.imshow(np.sqrt(np.real(f*np.conj(f))));
                plt.colorbar()
        return(intensity)

    def PhasePixelBasis(self, RegMat='D1'):
        self.Intensity = self.PhasePixelIntensity
        self.IntensityHessian = self.PhasePixelIntensityHessian
        self.IntensityHessianAtPixel = self.PhasePixelIntensityHessianAtPixel
        self.PointWiseOrthogonalBasis = True
        self.c_current = None  # these current values avoid repeat calculation
        self.g_current = None
        self.field_current = None
        self.field_grad_current = None
        if self.NDim == 1:
            self.nc = self.npup
            self.basis = np.zeros((self.npup, self.nc))
            for k in range(self.npup):
                self.basis[k, k] = 1.
            if RegMat == 'ID':
                self.RegMat = sparsemat.dia_matrix((np.ones(self.nc), 0), shape=(self.nc, self.nc))
            elif RegMat == 'D1':
                self.RegMat = sparsemat.dia_matrix((np.ones(self.nc),0), shape=(self.nc, self.nc))
                self.RegMat += sparsemat.dia_matrix((-np.ones(self.nc),1), shape=(self.nc, self.nc))
            elif RegMat == 'D2':
                self.RegMat = sparsemat.dia_matrix((2*np.ones(self.nc),0), shape=(self.nc, self.nc))
                self.RegMat += sparsemat.dia_matrix((-np.ones(self.nc),1), shape=(self.nc, self.nc))
                self.RegMat += sparsemat.dia_matrix((-np.ones(self.nc),-1), shape=(self.nc, self.nc))
            else:
                raise Exception("unknown regularization option.")
        else:  # self.NDim = 2
            self.nc = self.npup*self.npup
            self.basis = np.zeros((self.npup, self.npup, self.nc))
            for k in range(self.nc):
                (i1, i2) = self.UIp(k)
                self.basis[i1, i2, k] = 1
            if RegMat == 'ID':
                self.RegMat = sparsemat.dia_matrix((np.ones(self.nc), 0), shape=(self.nc, self.nc))
            elif RegMat == 'D1':
                Dv = np.zeros((self.nc, self.nc)) # vertical deriv
                for k1 in range(self.npup):  # horizontal index
                    for k2 in np.arange(self.npup-1):  # vertical index
                        ks = self.IUp(k2, k1)  # self index
                        kn = self.IUp(k2+1, k1) # neighbor index
                        Dv[ks, ks] = 1.
                        Dv[ks, kn] = -1.
                Dv = sparsemat.dia_matrix(Dv)
                Dh = np.zeros((self.nc, self.nc)) # horizontal deriv
                for k1 in range(self.npup-1):  # horizontal index
                    for k2 in np.arange(self.npup):  # vertical index
                        ks = self.IUp(k2, k1)  # self index
                        kn = self.IUp(k2, k1+1) # neighbor index
                        Dh[ks, ks] = 1.
                        Dh[ks, kn] = -1.
                Dh = sparsemat.dia_matrix(Dh)
                self.RegMat = sparsemat.vstack((Dv, Dh))
            elif RegMat == 'DelSq':  # del squared regularizer
                assert self.NDim == 2
                R = np.zeros((self.nc, self.nc))
                for k1 in 1 + np.arange(self.npup - 2):
                    for k2 in 1 + np.arange(self.npup - 2):
                        ks = self.IUp(k1, k2)  # self pixel
                        ki = self.IUp(k1-1, k2)  # above neighbor
                        km = self.IUp(k1+1, k2)  # below neighbor
                        kj = self.IUp(k1, k2-1)  # left neighbor
                        kl = self.IUp(k1, k2+1)  # right neighbor
                        R[ks,ks] = 4.
                        R[ks,ki] = -1.
                        R[ks,km] = -1.
                        R[ks,kj] = -1.
                        R[ks,kl] = -1.
                self.RegMat = sparsemat.dia_matrix(R)
            else:
                raise Exception('Unknown Regularization Matrix Type')
        # calculate field gradient factors using PointWiseOrthogonality
        #    actual field gradient is 1j*c[k]*self.field_grad_factor[:,k]
        self.field_grad_factor = np.zeros((self.nd, self.nc)).astype('complex')
        for k in range(self.nc):
            if self.NDim == 2:
                self.field_grad_factor[:,k] = self.FieldToField(self.basis[:,:,k], FullImage=False)
            else:
                self.field_grad_factor[:,k] = self.FieldToField(self.basis[:,k], FullImage=False)
        return

    # pixel basis functions for Re and Im parts of the pupil field.
    # builds self.basis containing basis functions
    # RegMat specifies the type of regularization matrix (self.RegMat)
    def ReImPixelBasis(self, RegMat='ID'):
        self.Intensity = self.ReImIntensity
        self.IntensityHessian = self.ReImIntensityHessian
        self.IntensityHessianAtPixel = self.ReImIntensityHessianAtPixel
        self.PointWiseOrthogonalBasis = True
        if self.NDim == 1:
            self.nc = 2*self.npup
            self.basis = np.zeros((self.npup, self.nc)).astype('complex')
            for k in range(self.npup):
                self.basis[k, k] = 1
                self.basis[k, k + self.npup] = 1j
            self.field_grad = np.zeros((self.nd, self.nc)).astype('complex')
            for k in range(self.nc):  # these are constant!
                self.field_grad[:, k] = self.FieldToField(self.basis[:, k], FullImage=False)
            if RegMat == 'ID':  # regularization matrix
                self.RegMat = sparsemat.dia_matrix((np.ones(self.nc), 0),
                                                   shape=(self.nc, self.nc))
            elif RegMat == 'D1':
                zer = sparsemat.dia_matrix((np.zeros(self.nc/2),0), shape=(self.nc/2, self.nc/2))
                dma = sparsemat.dia_matrix((np.ones(self.nc/2),0), shape=(self.nc/2, self.nc/2))
                dma += sparsemat.dia_matrix((-np.ones(self.nc/2),1), shape=(self.nc/2, self.nc/2))
                dma[self.nc/2 - 1, self.nc/2 - 1] = 0
                top = sparsemat.hstack((dma, zer))
                bot = sparsemat.hstack((zer, dma))
                self.RegMat = sparsemat.vstack((top, bot))
            return()
        # 2D case
        self.nc = 2*self.npup*self.npup
        self.basis = np.zeros((self.npup, self.npup, self.nc)).astype('complex')
        for k in range(self.nc/2):
            (i1, i2) = self.UIp(k)
            self.basis[i1, i2, k] = 1
            self.basis[i1, i2, k + self.nc/2] = 1j
        self.field_grad = np.zeros((self.nd, self.nc)).astype('complex')
        for k in range(self.nc):
            fgrad = self.FieldToField(self.basis[:, :, k], FullImage=False)
            self.field_grad[:, k] = fgrad
        if RegMat == 'ID':  # regularization matrix
            self.RegMat = sparsemat.dia_matrix((np.ones(self.nc), 0),
                                                   shape=(self.nc, self.nc))
        elif RegMat == 'D1':
            assert self.NDim == 2
            Dv = np.zeros((self.nc, self.nc)) # vertical deriv
            for k1 in range(self.npup):  # horizontal index
                for k2 in np.arange(self.npup-1):  # vertical index
                    ks = self.IUp(k2, k1)  # self index
                    kn = self.IUp(k2+1, k1) # neighbor index
                    Dv[ks, ks] = 1.
                    Dv[ks, kn] = -1.
                    ks += self.nc/2
                    kn += self.nc/2
                    Dv[ks, ks] = 1.
                    Dv[ks, kn] = -1.
            Dv = sparsemat.dia_matrix(Dv)
            Dh = np.zeros((self.nc, self.nc)) # horizontal deriv
            for k1 in range(self.npup-1):  # horizontal index
                for k2 in np.arange(self.npup):  # vertical index
                    ks = self.IUp(k2, k1)  # self index
                    kn = self.IUp(k2, k1+1) # neighbor index
                    Dh[ks, ks] = 1.
                    Dh[ks, kn] = -1.
                    ks += self.nc/2
                    kn += self.nc/2
                    Dh[ks, ks] = 1.
                    Dh[ks, kn] = -1.
            Dh = sparsemat.dia_matrix(Dh)
            self.RegMat = sparsemat.vstack((Dv, Dh))

        elif RegMat == 'DelSq':  # del squared regularizer
            assert self.NDim == 2
            R = np.zeros((self.nc, self.nc))
            for k1 in 1 + np.arange(self.npup - 2):
                for k2 in 1 + np.arange(self.npup - 2):
                    ks = self.IUp(k1, k2)  # self pixel
                    ki = self.IUp(k1-1, k2)  # above neighbor
                    km = self.IUp(k1+1, k2)  # below neighbor
                    kj = self.IUp(k1, k2-1)  # left neighbor
                    kl = self.IUp(k1, k2+1)  # right neighbor
                    R[ks,ks] = 4.
                    R[ks,ki] = -1.
                    R[ks,km] = -1.
                    R[ks,kj] = -1.
                    R[ks,kl] = -1.
                    ks += self.nc/2
                    ki += self.nc/2
                    km += self.nc/2
                    kj += self.nc/2
                    kl += self.nc/2
                    R[ks,ks] = 4.
                    R[ks,ki] = -1.
                    R[ks,km] = -1.
                    R[ks,kj] = -1.
                    R[ks,kl] = -1.
            self.RegMat = sparsemat.dia_matrix(R)
        else:
            raise Exception('Unknown Regularization Matrix Type')
        return

# SciPy.sparse matrices don't allow element access with the
#   usual bracket notation, so I had to write this function.
# index_tuple - index pair tuple
# matrix - duh
def RetrieveMatrixElement(index_tuple, matrix):
    assert matrix.ndim == 2
    assert len(index_tuple) == 2
    s0, s1 = matrix.shape
    r = index_tuple[0]
    c = index_tuple[1]
    assert (r >= 0) and (r < s0) and (c >= 0) and (c < s1)
    v = np.zeros(s1)
    v[c] = 1
    col = matrix.dot(v)
    return(col[r])

def myfft(g):  # for centered arrays, custom normalization
    if g.ndim == 1:
        return np.fft.fftshift(np.fft.fft(np.fft.fftshift(g)))/np.sqrt(len(g))
    else:
        return(np.fft.fftshift(np.fft.fft2(np.fft.fftshift(g)))/np.sqrt(g.shape[0]*g.shape[1]))

#  this zero pad function gives rise to purely real myfft with symm. input
def myzp(f, npad):  # zero-pad function for pupil fields
    if f.ndim == 1:
        nfield = len(f) + 1
        if np.mod(nfield, 2) != 0:
            raise Exception("len(f) must be odd.")
        if npad <= len(f):
            raise Exception("npad must be greater than len(f)")
        if np.mod(npad, 2) == 1:
            raise Exception("npad must be even.")
        g = np.zeros(npad).astype('complex')
        g[npad/2 - nfield/2 + 1:npad/2 + nfield/2] = f
        return(g)
    else:
        if f.shape[0] != f.shape[1]:
            raise Exception("f must be square (if 2D).")
        nfield = f.shape[0] + 1
        if np.mod(nfield, 2) != 0:
            raise Exception("len(f) must be odd.")
        if npad <= f.shape[0]:
            raise Exception("npad must be greater than len(f)")
        if np.mod(npad, 2) == 1:
            raise Exception("npad must be even.")
        g = np.zeros((npad, npad)).astype('complex')
        g[npad/2 - nfield/2 + 1:npad/2 + nfield/2,
          npad/2 - nfield/2 + 1:npad/2 + nfield/2] = f
        return(g)

def SigmaToStrehl(sigma):
    sig = np.array(sigma)
    strehl = np.exp(-sig*sig)
    return(strehl)

def StrehlToSigma(strehl):
    st = np.array(strehl)
    sigma = np.sqrt(- np.log(st))
    return(sigma)


from fpdf import FPDF
import cv2
import glob
import os
pdf = FPDF()

print(os.path.dirname(os.path.abspath(__file__)))
filenames = glob.glob(os.path.dirname(os.path.abspath(__file__))+'/Stldakash/*')
print(filenames)
i=1
for f in sorted(filenames):
    gray = cv2.imread(f,0)
    retval, thresh_gray = cv2.threshold(gray, thresh=120, maxval=255, type=cv2.THRESH_BINARY)
    cv2.imwrite('{}.png'.format(i),thresh_gray)
    pdf.add_page("L")
    pdf.image('{}.png'.format(i), x=0, y=0, h=210, w=297, type = '', link = '')
    i+=1
for i in glob.glob("*.png"):
    os.remove(i)
pdf.output('stldakash.pdf', 'F')


#! /usr/bin/python3

import os

import numpy as np
import tensorflow as tf
from sklearn.metrics import precision_recall_fscore_support
from tqdm import tqdm

def weightedLoss(originalLossFunc, weightsList):
    def lossFunc(true, pred):

        # axis = -1  # if channels last
        # axis = 1  # if channels first

        # argmax returns the index of the element with the greatest value
        # done in the class axis, it returns the class index
        # classSelectors = tf.argmax(true, axis=axis)
        # if your loss is sparse, use only true as classSelectors

        # considering weights are ordered by class, for each class
        # true(1) if the class index is equal to the weight index
        # convert i to int64
        # classSelectors = [tf.equal(tf.cast(i, tf.int64), classSelectors)
        #                   for i in range(len(weightsList))]

        classSelectors = [False, True]

        # casting boolean to float for calculations
        # each tensor in the list contains 1 where ground true class is equal to its index
        # if you sum all these, you will get a tensor full of ones.
        classSelectors = [tf.cast(x, tf.float32)
                          for x in classSelectors]

        # for each of the selections above, multiply their respective weight
        weights = [sel * w for sel, w in zip(classSelectors, weightsList)]

        # sums all the selections
        # result is a tensor with the respective weight for each element in predictions
        weightMultiplier = weights[0]
        for i in range(1, len(weights)):
            weightMultiplier = weightMultiplier + weights[i]

        # make sure your originalLossFunc only collapses the class axis
        # you need the other axes intact to multiply the weights tensor
        loss = originalLossFunc(true, pred)
        loss = loss * weightMultiplier

        return loss
    return lossFunc

class Metrics(tf.keras.callbacks.Callback):
    """Calculate metrics for a dev-/testset and add them to the logs."""

    def __init__(self, clf, data, steps, interval, prefix=''):
        self.clf = clf
        self.data = data
        self.steps = steps
        self.interval = interval
        self.prefix = prefix

    def on_epoch_end(self, epoch, logs):
        if (epoch + 1) % self.interval == 0:
            y_true, y_pred = self.clf.eval(
                self.data, self.steps, desc=self.prefix)
            p, r, f, s = precision_recall_fscore_support(y_true, y_pred)
        else:
            p, r, f, s = np.nan, np.nan, np.nan, np.nan
        logs_new = {'{}_precision'.format(self.prefix): p,
                    '{}_recall'.format(self.prefix): r,
                    '{}_f1'.format(self.prefix): f,
                    '{}_support'.format(self.prefix): s}
        logs.update(logs_new)

class Saver(tf.keras.callbacks.Callback):
    """Save the model."""

    def __init__(self, path, interval):
        self.path = path
        self.interval = interval

    def on_epoch_end(self, epoch, logs):
        if (epoch + 1) % self.interval == 0:
            file_name = os.path.join(
                self.path, 'model.{:03d}.h5'.format(epoch))
            self.model.save(file_name)

# pylint: disable=E1101
class LeafClassifier(object):
    """This classifier assigns labels to sequences based on words and HTML tags."""

    def __init__(self, input_size, num_layers, hidden_size, dropout, dense_size, class_weights):
        """Construct the network."""
        self.input_size = input_size
        self.num_layers = num_layers
        self.hidden_size = hidden_size
        self.dropout = dropout
        self.dense_size = dense_size
        self.class_weights = class_weights
        self.model = self._get_model()

    def _get_model(self):
        """Return a keras model."""
        model = tf.keras.Sequential()
        model.add(tf.keras.layers.InputLayer(
            input_shape=(None, self.input_size)))
        model.add(tf.keras.layers.Dense(self.dense_size, activation='relu'))
        model.add(tf.keras.layers.Masking(mask_value=0))
        for _ in range(self.num_layers):
            lstm = tf.keras.layers.LSTM(
                self.hidden_size, return_sequences=True)
            model.add(tf.keras.layers.Bidirectional(lstm))
        model.add(tf.keras.layers.Dropout(self.dropout))
        model.add(tf.keras.layers.Dense(1, activation='sigmoid'))

        model.compile(loss=weightedLoss(
            tf.keras.losses.binary_crossentropy, self.class_weights), optimizer='adam')
        return model

    def train(self, train_dataset, train_steps, epochs, log_file, ckpt,
              dev_dataset=None, dev_steps=None, test_dataset=None, test_steps=None, interval=1):
        """Train a number of input sequences."""
        callbacks = [Saver(ckpt, interval)]
        if dev_dataset is not None:
            callbacks.append(
                Metrics(self, dev_dataset, dev_steps, interval, 'dev'))
        if test_dataset is not None:
            callbacks.append(Metrics(self, test_dataset,
                             test_steps, interval, 'test'))
        callbacks.append(tf.keras.callbacks.CSVLogger(log_file))

        self.model.fit(train_dataset, steps_per_epoch=train_steps, epochs=epochs,
                       callbacks=callbacks)

    def eval(self, dataset, steps, desc=None):
        """Evaluate the model on the test data and return the metrics."""
        y_true, y_pred = [], []
        for b_x, b_y in tqdm(dataset, total=steps, desc=desc):
            # somehow this cast is necessary
            b_x = tf.dtypes.cast(b_x, 'float32')

            y_true.extend(b_y.numpy().flatten())
            y_pred.extend(
                np.around(self.model.predict_on_batch(b_x)).flatten())
        return y_true, y_pred


import xlwt
from django.core.paginator import PageNotAnInteger, EmptyPage, Paginator
from django.http import HttpResponse
from django.shortcuts import render, redirect, get_object_or_404

# Create your views here.
from django.views.generic import ListView

from business.forms import BusinessForm
from business.models import Business
from customer.models import Customer
from users.models import User

class BusinessView(ListView):
    """商机列表"""
    model = Business
    template_name = 'business.html'
    paginate_by = 10
    context_object_name = 'businesses'

    def get_queryset(self):
        user = self.request.session.get('user_id')
        if 'name' in self.request.GET and self.request.GET['name']:
            name = self.request.GET['name']
            return Business.objects.filter(name__icontains=name, user=user, is_valid=True)
        else:
            return Business.objects.filter(user=user, is_valid=True)

def business_add(request):
    """商机添加"""
    user_id = request.session.get('user_id')
    customers = Customer.objects.filter(user=user_id)
    if request.method == 'POST':
        form = BusinessForm(data=request.POST)
        if form.is_valid():
            form.save()
            return redirect('business')
        else:
            print(form.errors.as_json)
    else:
        form = BusinessForm()
    return render(request, 'business_add.html', {
        'form': form,
        'customers': customers
    })

def business_detail(request, pk):
    """商机详情"""
    user = request.session.get('user_id')
    business = get_object_or_404(Business, pk=pk, user=user, is_valid=True)
    if request.method == 'POST':
        form = BusinessForm(data=request.POST, instance=business)
        if form.is_valid():
            form.save()
            return redirect('business_detail', pk)
        else:
            print(form.errors.as_json)
    else:
        form = BusinessForm(instance=business)
    return render(request, 'business_detail.html', {
        'form': form,
        'pk': pk
    })

def business_edit(request, pk):
    """商机修改"""
    user = request.session.get('user_id')
    business = get_object_or_404(Business, pk=pk, user=user, is_valid=True)
    if request.method == 'POST':
        form = BusinessForm(data=request.POST, instance=business)
        if form.is_valid():
            form.save()
            return redirect('business')
        else:
            print(form.errors.as_json)
    else:
        form = BusinessForm(instance=business)
    return render(request, 'business_edit.html', {
        'form': form,
        'pk': pk
    })

def business_delete(request, pk):
    """商机删除"""
    user = request.session.get('user_id')
    business = get_object_or_404(Business, pk=pk, user=user, is_valid=True)
    business.delete()
    return redirect('business')

def business_group(request):
    """团队商机"""
    up_name = request.session.get('user_id')
    users = User.objects.filter(up_name=up_name)
    if 'name' in request.GET and request.GET['name']:
        name = request.GET['name']
        businesses = Business.objects.filter(name__icontains=name, user__up_name=up_name).exclude(is_valid=False)
    elif 'user_id' in request.GET and request.GET['user_id']:
        user_id = request.GET['user_id']
        businesses = Business.objects.filter(user=user_id, user__up_name=up_name).exclude(is_valid=False)
    else:
        businesses = Business.objects.filter(user__up_name=up_name).exclude(is_valid=False)
    paginator = Paginator(businesses, 10)
    page = request.GET.get('page')
    try:
        businesses = paginator.page(page)
    except PageNotAnInteger:
        businesses = paginator.page(1)
    except EmptyPage:
        businesses = paginator.page(paginator.num_pages)
    return render(request, 'business_all.html', {
        'businesses': businesses,
        'users': users
    })

def business_all(request):
    # 所有商机
    users = User.objects.all().exclude(role=3).exclude(role=5)
    if 'name' in request.GET and request.GET['name']:
        name = request.GET['name']
        businesses = Business.objects.filter(name__icontains=name).exclude(is_valid=False)
    elif 'user_id' in request.GET and request.GET['user_id']:
        user_id = request.GET['user_id']
        businesses = Business.objects.filter(user=user_id).exclude(is_valid=False)
    else:
        businesses = Business.objects.exclude(is_valid=False)
    paginator = Paginator(businesses, 10)
    page = request.GET.get('page')
    try:
        businesses = paginator.page(page)
    except PageNotAnInteger:
        businesses = paginator.page(1)
    except EmptyPage:
        businesses = paginator.page(paginator.num_pages)
    return render(request, 'business_all.html', {
        'businesses': businesses,
        'users': users
    })

def business_all_detail(request, pk):
    # 商机详情
    business = get_object_or_404(Business, pk=pk, is_valid=True)
    form = BusinessForm(instance=business)
    return render(request, 'business_all_detail.html', {
        'form': form,
    })

def export_business(request):
    """导出所有商机信息"""
    response = HttpResponse(content_type='application/ms-excel')
    response['Content-Disposition'] = 'attachment; filename="business.xls"'
    # 创建一个workbook，设置编码
    wb = xlwt.Workbook(encoding='utf-8')
    # 创建一个worksheet，对应excel中的sheet，表名称
    ws = wb.add_sheet('business')
    row_num = 0
    # 初始化样式
    font_style = xlwt.XFStyle()
    # 黑体
    font_style.font.bold = True
    # 写表头
    columns = ['商机名称', '客户名称', '赢单率', '预估金额', '创建人', '备注信息', '创建时间']
    for col_num in range(len(columns)):
        # 参数解读：（行，列，值，样式），row行 col列
        ws.write(row_num, col_num, columns[col_num], font_style)
    # 写表格内容
    font_style = xlwt.XFStyle()
    font_style.font.bold = False
    rows = Business.objects.filter(is_valid=True).values_list('name', 'customer__name', 'winning_rate', 'money', 'user__name', 'remarks', 'created_at')
    for row in rows:
        row_num += 1
        for col_num in range(len(row)):
            ws.write(row_num, col_num, row[col_num], font_style)
    # 保存
    wb.save(response)
    return response


#!/usr/bin/env python3

from pathlib import Path
from typing import Callable, NoReturn, Optional, Union
import os
import platform
import readline
import shutil
import sys

#

class Action:
    def __init__(self, name: str, f: Callable[[], Union[None, NoReturn]]) -> None:
        self.name = name
        self._f = f

    def __call__(self) -> None:
        self._f()

#

def colored(text: str, color: str) -> str:
    END = "\33[0m"
    return color + text + END

# Colors
RED = "\33[31m"
GREEN = "\33[32m"
BLUE = "\33[34m"

# Symbols
CROSS = colored("✗", RED)
TICK = colored("✓", GREEN)

#

def yn(question: str) -> bool:
    answer = input(f"{question} (y/N) ").strip()
    return answer.lower() in ["yes", "y"]

def really(func: Callable[[], None], question: str) -> None:
    if yn(question):
        func()
        print(colored(f"\tDone {TICK}", GREEN))
    else:
        print(colored(f"\tAborted {CROSS}", RED))

#

def rel(path: Path) -> str:
    try:
        return f"~/{path.relative_to(Path.home())}"
    except ValueError:
        return str(path)

def print_files(files: list[Path]) -> None:
    for file in files:
        symbol = TICK if file.exists() else CROSS
        text = colored(rel(file), BLUE) if file.is_dir() else rel(file)

        print(f"\t{text} {symbol}")

def delete_files(files: list[Path]) -> None:
    for file in files:
        if not file.exists():
            continue

        try:
            if file.is_dir():
                shutil.rmtree(file)
            else:
                os.remove(file)
        except OSError as e:
            print(colored(f"\tError: {rel(file)} - {e.strerror}", RED))

def search(
    root: Path,
    patterns: Optional[list[str]] = None,
    ignore_case: bool = False
) -> list[Path]:
    if not patterns:
        return list(root.rglob("*"))

    if ignore_case:
        f: Callable[[str], str] \
            = lambda c: f"[{c.lower()}{c.upper()}]" if c.isalpha() else c
        patterns = ["".join(map(f, p)) for p in patterns]

    files: list[Path] = []
    for p in patterns:
        files += root.rglob(p)

    return files

#

def print_and_delete(files: list[Path]) -> None:
    print_files(files)
    really(
        lambda: delete_files(files),
        "\n\tDo you really want to delete these files?"
    )

def search_and_delete(
    root: Optional[Path] = None,
    patterns: Optional[list[str]] = None,
    ignore_case: bool = False
) -> None:
    if root is None:
        while True:
            try:
                answer = input("\tRoot [~]: ").strip() or "~"
                root = Path(os.path.expandvars(answer)) \
                    .expanduser() \
                    .resolve(strict=True)
                break
            except (ValueError, FileNotFoundError):
                print(colored("\tPath not valid. Try again.\n", RED))

    if patterns is None:
        patterns = []
        while True:
            p = input(f"\tPattern #{len(patterns) + 1}: ").strip()
            if p:
                patterns.append(p)
            else:
                break

        if patterns:
            ignore_case = yn("\tIgnore case?")

    files = search(root, patterns, ignore_case)

    print()

    if files:
        print_and_delete(files)
    else:
        print(colored(f"\tNo files found {CROSS}", RED))

#

apple: list[str] = [
    "Assistant",
    "Caches",
    "Calendars",
    "Cookies",
    "Developer",
    "HTTPStorages",
    "LaunchAgents",
    "Logs",
    "Mail",
    "Maps",
    "Messages",
    "Metadata",
    "News",
    "Reminders",
    "Safari",
    "Saved Application State",
    "Spelling",
    "Suggestions",
    "SyncedPreferences",
    "WebKit",

    "Application Support/Caches",
    "Application Support/CloudDocs",
    "Application Support/com.apple.sharedfilelist",
    "Application Support/com.apple.spotlight",
    "Application Support/CrashReporter",
    "Application Support/Dock",
    "Application Support/FileProvider",

    "Preferences/.GlobalPreferences_m.plist",
    "Preferences/.GlobalPreferences.plist",
    "Preferences/ByHost",
    "Preferences/com.apple.dock.plist",
    "Preferences/com.apple.dt.Xcode.plist",
    "Preferences/com.apple.EmojiCache.plist",
    "Preferences/com.apple.EmojiPreferences.plist",
    "Preferences/com.apple.finder.plist",
    "Preferences/com.apple.mediaanalysisd.plist",
    "Preferences/com.apple.SFSymbols.plist",
    "Preferences/com.apple.Spotlight.plist",
    "Preferences/com.apple.Terminal.plist",
    "Preferences/com.apple.universalaccessAuthWarning.plist"
]

app: list[str] = [
    # Adobe
    "Application Support/Adobe",
    "Preferences/Adobe",
    "Preferences/com.adobe.AdobeRdrCEFHelper.plist",
    "Preferences/com.adobe.crashreporter.plist",
    "Preferences/com.adobe.Reader.plist",

    # Alacritty
    "Preferences/io.alacritty.plist",

    # GitKraken
    "Application Support/GitKraken",
    "Preferences/com.axosoft.gitkraken.plist",

    # Google
    "Application Support/Google",
    "Google",
    "Preferences/com.google.Chrome.plist",
    "Preferences/com.google.Keystone.Agent.plist",

    # ImageOptim
    "Application Scripts/net.pornel.ImageOptimizeExtension",
    "Containers/net.pornel.ImageOptimizeExtension",
    "Group Containers/59KZTZA4XR.net.pornel.ImageOptim",
    "Preferences/net.pornel.ImageOptim.plist",

    # JetBrains
    "Application Support/JetBrains",
    "Preferences/com.jetbrains.toolbox.renderer.plist",

    # Mozilla
    "Application Support/Firefox",
    "Application Support/Mozilla",
    "Preferences/org.mozilla.firefoxdeveloperedition.plist",

    # Notion
    "Application Support/Notion",
    "Preferences/notion.id.plist",

    # OBS
    "Application Support/obs-studio",
    "Preferences/com.obsproject.obs-studio.plist",

    # Postman
    "Application Support/Postman",
    "Preferences/com.postmanlabs.mac.plist",

    # ProtonVPN
    "Preferences/ch.protonvpn.mac.plist",

    # SEB
    "Preferences/org.safeexambrowser.SafeExamBrowser.plist",

    # Spotify
    "Application Support/Spotify",
    "Preferences/com.spotify.client.plist",

    # Visual Studio Code
    "Application Support/Code",
    "Preferences/com.microsoft.VSCode.plist",

    # Zoom
    "Application Support/zoom.us",
    "Preferences/us.zoom.xos.plist",
    "Preferences/ZoomChat.plist",

    # Other
    "Preferences/com.qtproject.plist"
]

#

if __name__ == "__main__":
    print(f"OS: {platform.system()}")
    print(f"HOME: {Path.home()}\n")

    # Setup
    home = Path.home()
    library = home / "Library"

    apple_files = [library / file for file in apple]
    app_files = [library / file for file in app]

    # Actions
    actions: list[Action] = [
        Action("Quit\n", sys.exit),

        Action("Delete Apple files", lambda: print_and_delete(apple_files)),
        Action("Delete App files", lambda: print_and_delete(app_files)),
        Action(
            "Delete 'Cache', 'Caches', 'Logs', 'Saved Application State' files\n",
            lambda: search_and_delete(
                library, ["Cache", "Caches", "Logs", "Saved Application State"]
            )
        ),

        Action(
            "Delete '.DS_Store' files\n",
            lambda: search_and_delete(home, [".DS_Store"])
        ),

        Action("Search", lambda: search_and_delete())
    ]

    print("Actions:")
    for i, action in enumerate(actions):
        print(f"\t{i} - {action.name}")

    # Loop
    while True:
        try:
            answer = int(input("\n> "))
            if answer in range(0, len(actions)):
                try:
                    actions[answer]()
                except EOFError:
                    print(colored(f"\n\tAborted {CROSS}", RED))
            else:
                print(colored("\tOut of range. Try again.", RED))
        except ValueError:
            print(colored("\tEnter a valid number!", RED))
        except (EOFError, KeyboardInterrupt):
            sys.exit()


import warnings
warnings.filterwarnings("ignore")

import sys
import os
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.model_selection import LeaveOneOut
from sklearn.model_selection import cross_val_score
from sklearn.metrics import make_scorer
from sklearn.metrics import recall_score

from sklearn.metrics import f1_score
from sklearn.metrics import accuracy_score
from sklearn.metrics import recall_score
from sklearn.metrics import precision_score
from sklearn.metrics import roc_auc_score

sys.path.insert(0, "utils/")

""" Import all python file model_config"""
import models_config
""" Import all python files in utils folder """
import utils

""" List of Directories """
Input_DIR = 'output/preprocess'
Output_DIR = 'output/prediction'
Evaluation_DIR = 'output/evaluation'

if not os.path.exists(Output_DIR):
    os.makedirs(Output_DIR)

if not os.path.exists(Evaluation_DIR):
    os.makedirs(Evaluation_DIR)

def training ():
    """
    Saves CSV file of LOOCV cross validations
    and CSV files of preditions with proper naming
    for re-evaluation.

    """
    # dictionary for storage
    all_prediction = {}
    all_actual = {}
    all_metricCV = {}
    all_metricCV_Final = {}
    id_list = []

    for filename in os.listdir(Input_DIR):  # Goes through all the files in input directory
        if filename.endswith(".csv") : # checks the CSV files
            all_prediction[filename[:-4]], all_actual[filename[:-4]],all_metricCV[filename[:-4]], id_list = trainIndividually(filename,id_list)  # Individually train per file name: output is collected per file name
            print("---------------")
            print("---------------")
            print(filename + " predicted")

    for i in all_prediction:
        for j in all_prediction[i]:
            output_name ="[" + i + "]_["+ j + "]"  # name of CSV file
            all_metricCV_Final[output_name] = all_metricCV[i][j]
            print(output_name)
            # results
            # combine to a temporary diction for the actual and prediction
            tempt_prediction = all_prediction[i][j].copy()
            tempt_actual = all_actual[i][j].copy()
            tempt_dict = {"Actual":tempt_actual,"ID":id_list}
            tempt_dict['Prediction'] = tempt_prediction
            df = pd.DataFrame (tempt_dict)
            utils.saveData(Output_DIR, output_name, df)
            print(output_name + " SAVED")

    df = pd.DataFrame(all_metricCV_Final).T  # save cross validation into csv file
    df = df.reset_index()
    utils.saveData(Evaluation_DIR, "LOOCV Evaluation", df)

    return

def trainIndividually (filename, id_list ):
    """
    Splits data into train-test and then fit_predict and does cross validation
    Input:
        filename: str
            filename of the
        id_list: list
            number of runs
    Output:
        prediction: dictionary
            dictionary of predictions
        actual: dictionary
            dictionary of actual labels (y-test)
        metricCV: dictionary
            dictionary of LOOCV results
        id_list : list
            list of patient ID
    """
    file_description = models_config.fileDescription(filename)  # fileDescription: gets the preprocessing description from the file name

    print(filename)

    file_location = os.path.join(Input_DIR, filename)
    dataframe = pd.read_csv(file_location)

    prediction = {}
    actual = {}
    metricCV = {}

    model_description = models_config.modelDescription()

    df_copy = dataframe.copy()
    if len(id_list) == 0:
        id_list = df_copy['Patient Identification Number'].values.copy()
        print( "File ID  Copied from " + filename)
    else:
        tempt_list_1 = df_copy['Patient Identification Number'].values.copy()
        tempt_list_2 = id_list.copy()
        if len(tempt_list_1)== len(tempt_list_2) and len(tempt_list_1) == sum([1 for i, j in zip(tempt_list_1, tempt_list_2) if i == j]):
            print ("The ID lists of "+filename+ "are identical to previous")
            print("_______________________________________________________")
        else:
            print("Error !!!  Recalibrate all ID Files")

    df_copy = df_copy.drop("Patient Identification Number", axis=1)
    X, Y, features = utils.labelSeparator(df_copy,file_description['label'])  # separates labels and features from dataframe

    for i in model_description:  # goes through all the models # for Hyp 1: only regression models
        print("-------------------")
        print("-------------------")
        print(i['model_name'], " prediction !!!")

        metricCV[i['model_name']], prediction[i['model_name']], actual[i['model_name']] = trainCV(X, Y, i['model'])  # cross validation of training
        print(i['model_name'], " prediction DONE!!!")

    return prediction, actual, metricCV, id_list

def trainCV(X,Y, model):
    """
    Cross validation using LOOCV
    Input:
        X: matrix
            x-train
        Y: list
            y-train
        model: model function
            model
    Output:
        metricCV: dictionary
            cross validation results
        y_pred: list
            list of prediction using the specific and model imported file
        y_true:list
            list of actual value using the specific model and imported file

    """

    metricCV = {}

    # create loocv procedure
    cv = LeaveOneOut()
    # enumerate splits
    y_true, y_pred = list(), list()
    for train_ix, test_ix in cv.split(X):
        # split data
        X_train, X_test = X[train_ix, :], X[test_ix, :]
        y_train, y_test = Y[train_ix], Y[test_ix]
        model.fit(X_train, y_train)
        # evaluate model
        yhat = model.predict(X_test)
        # store
        y_true.append(y_test[0])
        y_pred.append(yhat[0])

    metricCV["accuracy"]= accuracy_score(y_true,y_pred)
    metricCV["sensitivity"] = recall_score(y_true,y_pred)
    metricCV["specificity"] = recall_score(y_true,y_pred, pos_label = 0 )
    metricCV["auc"] = roc_auc_score(y_true,y_pred)

    return metricCV, y_pred, y_true

if __name__ == "__main__":
  print("Starting")

  training()

  print("Done")


"""
Zip - Unindo iteráveis
Zip - longest - Itertools
"""
from itertools import zip_longest, count #so o zip não iria apresentar monte belo

indice = count()
cidades=['São Paulo','Belo Horizonte','Salvador','Monte Belo']
estados=['SP','MG','BA']

cidades_estados=zip(
    indice,
    estados,
    cidades,
)

for valor in cidades_estados:
    print(valor)
   # print(indice, estados, cidades)

aula 075 - Zip e Zip_longest - Unindo iteráveis/aula 75 - Zip e Zip_longest - Unindo iteráveis.py


import asyncio
import os
from shutil import rmtree
import datetime
import wikipedia
import requests
from justwatch import JustWatch
from bs4 import BeautifulSoup
from search_engine_parser import GoogleSearch
from search_engine_parser.core.exceptions import NoResultsOrTrafficError as GoglError
from geopy.geocoders import Nominatim
from telethon.tl import types

from . import *

def progress(current, total):
    logger.info(
        "Downloaded {} of {}\nCompleted {}".format(
            current, total, (current / total) * 100
        )
    )

@bot.on(deadly_cmd(pattern="wikipedia (.*)"))
@bot.on(sudo_cmd(pattern="wikipedia (.*)", allow_sudo=True))
async def _(event):
    if event.fwd_from:
        return
    await edit_or_reply(event, "Processing ...")
    input_str = event.pattern_match.group(1)
    result = ""
    results = wikipedia.search(input_str)
    for s in results:
        page = wikipedia.page(s)
        url = page.url
        result += f"> [{s}]({url}) \n"
    await edit_or_reply(event, "WikiPedia **Search**: {} \n\n **Result**: \n\n{}".format(input_str, result)
    )

@bot.on(deadly_cmd(pattern="watch (.*)"))
@bot.on(sudo_cmd(pattern="watch (.*)", allow_sudo=True))
async def _(event):
    if event.fwd_from:
        return
    query = event.pattern_match.group(1)
    deadly = await eor(event, "Finding Sites...")
    streams = get_stream_data(query)
    title = streams["title"]
    thumb_link = streams["movie_thumb"]
    release_year = streams["release_year"]
    release_date = streams["release_date"]
    scores = streams["score"]
    try:
        imdb_score = scores["imdb"]
    except KeyError:
        imdb_score = None

    try:
        tmdb_score = scores["tmdb"]
    except KeyError:
        tmdb_score = None

    stream_providers = streams["providers"]
    if release_date is None:
        release_date = release_year

    output_ = f"**Movie:**\n`{title}`\n**Release Date:**\n`{release_date}`"
    if imdb_score:
        output_ = output_ + f"\n**IMDB: **{imdb_score}"
    if tmdb_score:
        output_ = output_ + f"\n**TMDB: **{tmdb_score}"

    output_ = output_ + "\n\n**Available on:**\n"
    for provider, link in stream_providers.items():
        if "sonyliv" in link:
            link = link.replace(" ", "%20")
        output_ += f"[{pretty(provider)}]({link})\n"

    await bot.send_file(
        event.chat_id,
        caption=output_,
        file=thumb_link,
        force_document=False,
        allow_cache=False,
        silent=True,
    )
    await event.delete()

@bot.on(deadly_cmd(pattern="google (.*)", outgoing=True))
@bot.on(sudo_cmd(pattern="google (.*)", allow_sudo=True))
async def google(event):
    input_str = event.pattern_match.group(1)
    if not input_str:
        return await eod(event, "`Give something to search..`")
    deadly = await eor(event, "Searching...")
    gos = GoogleSearch()
    try:
        got = await gos.async_search(f"{input_str}", cache=False)
    except GoglError as e:
        return await eod(event, str(e), 10)
    output = ""
    for i in range(len(got["links"])):
        text = got["titles"][i]
        url = got["links"][i]
        des = got["descriptions"][i]
        output += f" 👉🏻  [{text}]({url})\n`{des}`\n\n"
    res = f"**Google Search Query:**\n`{input_str}`\n\n**Results:**\n{output}"
    see = []
    for i in range(0, len(res), 4095):
        see.append(res[i : i + 4095])
    for j in see:
        await bot.send_message(event.chat_id, j, link_preview=False)
    await deadly.delete()
    see.clear()

@bot.on(deadly_cmd(pattern="img (.*)", outgoing=True))
@bot.on(sudo_cmd(pattern="img (.*)", allow_sudo=True))
async def img(event):
    sim = event.pattern_match.group(1)
    if not sim:
        return await eod(event, "`Give something to search...`")
    deadly = await eor(event, f"Searching for  `{sim}`...")
    if "-" in sim:
        try:
            lim = int(sim.split(";")[1])
            sim = sim.split(";")[0]
        except BaseExceptaion:
            lim = 5
    else:
        lim = 5
    imgs = googleimagesdownload()
    args = {
        "keywords": sim,
        "limit": lim,
        "format": "jpg",
        "output_directory": "./DOWNLOADS/",
    }
    letsgo = imgs.download(args)
    gotit = letsgo[0][sim]
    await event.client.send_file(event.chat_id, gotit, caption=sim, album=True)
    rmtree(f"./DOWNLOADS/{sim}/")
    await deadly.delete()

@bot.on(deadly_cmd(pattern="reverse"))
@bot.on(sudo_cmd(pattern="reverse", allow_sudo=True))
async def _(event):
    if event.fwd_from:
        return
    start = datetime.datetime.now()
    BASE_URL = "http://www.google.com"
    OUTPUT_STR = "Reply to an image to do Google Reverse Search"
    if event.reply_to_msg_id:
        deadly = await eor(event, "Pre Processing Media")
        previous_message = await event.get_reply_message()
        previous_message_text = previous_message.message
        if previous_message.media:
            downloaded_file_name = await bot.download_media(
                previous_message, Config.TMP_DOWNLOAD_DIRECTORY
            )
            SEARCH_URL = "{}/searchbyimage/upload".format(BASE_URL)
            multipart = {
                "encoded_image": (
                    downloaded_file_name,
                    open(downloaded_file_name, "rb"),
                ),
                "image_content": "",
            }
            # https://stackoverflow.com/a/28792943/4723940
            google_rs_response = requests.post(
                SEARCH_URL, files=multipart, allow_redirects=False
            )
            the_location = google_rs_response.headers.get("Location")
            os.remove(downloaded_file_name)
        else:
            previous_message_text = previous_message.message
            SEARCH_URL = "{}/searchbyimage?image_url={}"
            request_url = SEARCH_URL.format(BASE_URL, previous_message_text)
            google_rs_response = requests.get(request_url, allow_redirects=False)
            the_location = google_rs_response.headers.get("Location")
        await deadly.edit("Found Google Result. Processing results...")
        headers = {
            "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:58.0) Gecko/20100101 Firefox/58.0"
        }
        response = requests.get(the_location, headers=headers)
        soup = BeautifulSoup(response.text, "html.parser")
        # document.getElementsByClassName("r5a77d"): PRS
        prs_div = soup.find_all("div", {"class": "r5a77d"})[0]
        prs_anchor_element = prs_div.find("a")
        prs_url = BASE_URL + prs_anchor_element.get("href")
        prs_text = prs_anchor_element.text
        # document.getElementById("jHnbRc")
        img_size_div = soup.find(id="jHnbRc")
        img_size = img_size_div.find_all("div")
        end = datetime.datetime.now()
        ms = (end - start).seconds
        OUTPUT_STR = """Possible Related Search : {prs_text}

More Info: Open this Link in {ms} seconds""".format(
            **locals()
        )
    await deadly.edit(OUTPUT_STR, parse_mode="HTML", link_preview=False)

@bot.on(deadly_cmd(pattern="gps ?(.*)"))
@bot.on(sudo_cmd(pattern="gps ?(.*)", allow_sudo=True))
async def gps(event):
    if event.fwd_from:
        return
    reply_to_id = event.message
    if event.reply_to_msg_id:
        reply_to_id = await event.get_reply_message()
    input_str = event.pattern_match.group(1)
    if not input_str:
        return await eod(event, "What should i find? Give me location.🤨")

    await edit_or_reply(event, "Finding😁")

    geolocator = Nominatim(user_agent="DeadlyBot")
    geoloc = geolocator.geocode(input_str)

    if geoloc:
        lon = geoloc.longitude
        lat = geoloc.latitude
        await reply_to_id.reply(
            input_str, file=types.InputMediaGeoPoint(types.InputGeoPoint(lat, lon))
        )
        await event.delete()
    else:
        await eod(event, "I coudn't find it😫")

CmdHelp("google").add_command(
  "google", "<query>", "Does a google search for the query provided"
).add_command(
  "img", "<query>", "Does a image search for the query provided"
).add_command(
  "reverse", "<reply to a sticker/pic>", "Does a reverse image search on google and provides the similar images"
).add_command(
  "gps", "<place>", "Gives the location of the given place/city/state."
).add_command(
  "wikipedia", "<query>", "Searches for the query on Wikipedia."
).add_command(
  "watch", "<query>", "Searches for all the available sites for watching that movie or series."
).add_info(
  "Google Search."
).add_warning(
  "✅ Harmless Module."
).add()


import os
import logging
import psycopg2
import psycopg2.extras as psycop_extras
from nose.tools import nottest
from application.services import config_service
from application.services import logging_service
from application.services import debug_service
from application.services import conversion_service
from application.services import database_service
from application.models.classes import dycast_parameters

debug_service.enable_debugger()

current_dir = os.path.basename(os.path.dirname(os.path.realpath(__file__)))

@nottest
def init_test_environment():
    config_path = os.path.join(current_dir, '..', 'dycast.config')

    config_service.init_config(config_path)
    logging_service.init_logging()

@nottest
def get_dycast_parameters():
    dycast = dycast_parameters.DycastParameters()

    dycast.srid_of_cases = '3857'
    dycast.files_to_import = get_test_cases_import_files_latlong()

    dycast.spatial_domain = 600
    dycast.temporal_domain = 28
    dycast.close_in_space = 100
    dycast.close_in_time = 4
    dycast.case_threshold = 10

    dycast.startdate = conversion_service.get_date_object_from_string('2016-03-30')
    dycast.enddate = conversion_service.get_date_object_from_string('2016-03-31')
    dycast.extent_min_x = 1830400
    dycast.extent_min_y = 2120600
    dycast.extent_max_x = 1830700
    dycast.extent_max_y = 2120300
    dycast.srid_of_extent = 3857

    return dycast

@nottest
def get_test_data_directory():
    return os.path.join(current_dir, 'test_data')

@nottest
def get_test_data_import_directory():
    return os.path.join(get_test_data_directory(), 'import')

@nottest
def get_test_data_export_directory():
    return os.path.join(get_test_data_directory(), 'export')

@nottest
def get_test_cases_import_files_latlong():
    file_1 = os.path.join(get_test_data_import_directory(), 'input_cases_latlong1.tsv')
    file_2 = os.path.join(get_test_data_import_directory(), 'input_cases_latlong2.tsv')
    return [file_1, file_2]

@nottest
def get_test_cases_import_file_geometry():
    return os.path.join(get_test_data_import_directory(), 'input_cases_geometry.tsv')

@nottest
def get_count_from_table(table_name):
    cur, conn = database_service.init_db()
    querystring = "SELECT count(*) from " + table_name
    try:
        cur.execute(querystring)
    except Exception:
        conn.rollback()
        logging.exception("Can't select count from table, exiting...")
        raise
    new_row = cur.fetchone()
    return new_row[0]

@nottest
def insert_test_risk():
    cur, conn = database_service.init_db()
    querystring = "INSERT INTO risk VALUES %s"
    data_tuple = [('2016-03-30', 1830400, 2120400, 10, 1, 6, 7, 0.3946), ('2016-03-31', 1830400, 2120400, 10, 1, 6, 7, 0.3946)]
    try:
        psycop_extras.execute_values(cur, querystring, data_tuple)
    except psycopg2.IntegrityError:
        conn.rollback()
        logging.warning("Couldn't insert duplicate key in tuple: %s...", data_tuple)
    except Exception:
        conn.rollback()
        logging.exception("Couldn't insert tuple")
        raise
    conn.commit()