from __future__ import annotations

import json
import yaml
from string import Formatter
from abc import ABC, abstractmethod
from pathlib import Path
from typing import Any, Callable, Dict, List, Mapping, Optional, Set, Union

from pydantic import BaseModel, Extra, Field, root_validator

from pilot.out_parser.base import BaseOutputParser
from pilot.prompts.base import PromptValue
from pilot.scene.base_message import HumanMessage, BaseMessage
from pilot.common.formatting import formatter

def jinja2_formatter(template: str, **kwargs: Any) -> str:
    """Format a template using jinja2."""
    try:
        from jinja2 import Template
    except ImportError:
        raise ImportError(
            "jinja2 not installed, which is needed to use the jinja2_formatter. "
            "Please install it with `pip install jinja2`."
        )

    return Template(template).render(**kwargs)

def validate_jinja2(template: str, input_variables: List[str]) -> None:
    input_variables_set = set(input_variables)
    valid_variables = _get_jinja2_variables_from_template(template)
    missing_variables = valid_variables - input_variables_set
    extra_variables = input_variables_set - valid_variables

    error_message = ""
    if missing_variables:
        error_message += f"Missing variables: {missing_variables} "

    if extra_variables:
        error_message += f"Extra variables: {extra_variables}"

    if error_message:
        raise KeyError(error_message.strip())

def _get_jinja2_variables_from_template(template: str) -> Set[str]:
    try:
        from jinja2 import Environment, meta
    except ImportError:
        raise ImportError(
            "jinja2 not installed, which is needed to use the jinja2_formatter. "
            "Please install it with `pip install jinja2`."
        )
    env = Environment()
    ast = env.parse(template)
    variables = meta.find_undeclared_variables(ast)
    return variables

DEFAULT_FORMATTER_MAPPING: Dict[str, Callable] = {
    "f-string": formatter.format,
    "jinja2": jinja2_formatter,
}

DEFAULT_VALIDATOR_MAPPING: Dict[str, Callable] = {
    "f-string": formatter.validate_input_variables,
    "jinja2": validate_jinja2,
}

def check_valid_template(
    template: str, template_format: str, input_variables: List[str]
) -> None:
    """Check that template string is valid."""
    if template_format not in DEFAULT_FORMATTER_MAPPING:
        valid_formats = list(DEFAULT_FORMATTER_MAPPING)
        raise ValueError(
            f"Invalid template format. Got `{template_format}`;"
            f" should be one of {valid_formats}"
        )
    try:
        validator_func = DEFAULT_VALIDATOR_MAPPING[template_format]
        validator_func(template, input_variables)
    except KeyError as e:
        raise ValueError(
            "Invalid prompt schema; check for mismatched or missing input parameters. "
            + str(e)
        )

class BasePromptTemplate(BaseModel, ABC):
    """Base class for all prompt templates, returning a prompt."""

    input_variables: List[str]
    """A list of the names of the variables the prompt template expects."""
    output_parser: Optional[BaseOutputParser] = None
    """How to parse the output of calling an LLM on this formatted prompt."""
    partial_variables: Mapping[str, Union[str, Callable[[], str]]] = Field(
        default_factory=dict
    )

    @abstractmethod
    def format_prompt(self, **kwargs: Any) -> PromptValue:
        """Create Chat Messages."""

    @root_validator()
    def validate_variable_names(cls, values: Dict) -> Dict:
        """Validate variable names do not include restricted names."""
        if "stop" in values["input_variables"]:
            raise ValueError(
                "Cannot have an input variable named 'stop', as it is used internally,"
                " please rename."
            )
        if "stop" in values["partial_variables"]:
            raise ValueError(
                "Cannot have an partial variable named 'stop', as it is used "
                "internally, please rename."
            )

        overall = set(values["input_variables"]).intersection(
            values["partial_variables"]
        )
        if overall:
            raise ValueError(
                f"Found overlapping input and partial variables: {overall}"
            )
        return values

    def partial(self, **kwargs: Union[str, Callable[[], str]]) -> BasePromptTemplate:
        """Return a partial of the prompt template."""
        prompt_dict = self.__dict__.copy()
        prompt_dict["input_variables"] = list(
            set(self.input_variables).difference(kwargs)
        )
        prompt_dict["partial_variables"] = {**self.partial_variables, **kwargs}
        return type(self)(**prompt_dict)

    def _merge_partial_and_user_variables(self, **kwargs: Any) -> Dict[str, Any]:
        # Get partial params:
        partial_kwargs = {
            k: v if isinstance(v, str) else v()
            for k, v in self.partial_variables.items()
        }
        return {**partial_kwargs, **kwargs}

    @abstractmethod
    def format(self, **kwargs: Any) -> str:
        """Format the prompt with the inputs.

        Args:
            kwargs: Any arguments to be passed to the prompt template.

        Returns:
            A formatted string.

        Example:

        .. code-block:: python

            prompt.format(variable1="foo")
        """

    @property
    def _prompt_type(self) -> str:
        """Return the prompt type key."""
        raise NotImplementedError

    def dict(self, **kwargs: Any) -> Dict:
        """Return dictionary representation of prompt."""
        prompt_dict = super().dict(**kwargs)
        prompt_dict["_type"] = self._prompt_type
        return prompt_dict

    def save(self, file_path: Union[Path, str]) -> None:
        """Save the prompt.

        Args:
            file_path: Path to directory to save prompt to.

        Example:
        .. code-block:: python

            prompt.save(file_path="path/prompt.api_v1")
        """
        if self.partial_variables:
            raise ValueError("Cannot save prompt with partial variables.")
        # Convert file to Path object.
        if isinstance(file_path, str):
            save_path = Path(file_path)
        else:
            save_path = file_path

        directory_path = save_path.parent
        directory_path.mkdir(parents=True, exist_ok=True)

        # Fetch dictionary to save
        prompt_dict = self.dict()

        if save_path.suffix == ".json":
            with open(file_path, "w") as f:
                json.dump(prompt_dict, f, indent=4)
        elif save_path.suffix == ".api_v1":
            with open(file_path, "w") as f:
                yaml.dump(prompt_dict, f, default_flow_style=False)
        else:
            raise ValueError(f"{save_path} must be json or api_v1")

class StringPromptValue(PromptValue):
    text: str

    def to_string(self) -> str:
        """Return prompt as string."""
        return self.text

    def to_messages(self) -> List[BaseMessage]:
        """Return prompt as messages."""
        return [HumanMessage(content=self.text)]

class StringPromptTemplate(BasePromptTemplate, ABC):
    """String prompt should expose the format method, returning a prompt."""

    def format_prompt(self, **kwargs: Any) -> PromptValue:
        """Create Chat Messages."""
        return StringPromptValue(text=self.format(**kwargs))

class PromptTemplate(StringPromptTemplate):
    """Schema to represent a prompt for an LLM.

    Example:
        .. code-block:: python

            from langchain import PromptTemplate
            prompt = PromptTemplate(input_variables=["foo"], template="Say {foo}")
    """

    input_variables: List[str]
    """A list of the names of the variables the prompt template expects."""

    template: str
    """The prompt template."""

    template_format: str = "f-string"
    """The format of the prompt template. Options are: 'f-string', 'jinja2'."""

    validate_template: bool = True
    """Whether or not to try validating the template."""

    @property
    def _prompt_type(self) -> str:
        """Return the prompt type key."""
        return "prompt"

    class Config:
        """Configuration for this pydantic object."""

        extra = Extra.forbid

    def format(self, **kwargs: Any) -> str:
        """Format the prompt with the inputs.

        Args:
            kwargs: Any arguments to be passed to the prompt template.

        Returns:
            A formatted string.

        Example:

        .. code-block:: python

            prompt.format(variable1="foo")
        """
        kwargs = self._merge_partial_and_user_variables(**kwargs)
        return DEFAULT_FORMATTER_MAPPING[self.template_format](self.template, **kwargs)

    @root_validator()
    def template_is_valid(cls, values: Dict) -> Dict:
        """Check that template and input variables are consistent."""
        if values["validate_template"]:
            all_inputs = values["input_variables"] + list(values["partial_variables"])
            check_valid_template(
                values["template"], values["template_format"], all_inputs
            )
        return values

    @classmethod
    def from_examples(
        cls,
        examples: List[str],
        suffix: str,
        input_variables: List[str],
        example_separator: str = "\n\n",
        prefix: str = "",
        **kwargs: Any,
    ) -> PromptTemplate:
        """Take examples in list format with prefix and suffix to create a prompt.

        Intended to be used as a way to dynamically create a prompt from examples.

        Args:
            examples: List of examples to use in the prompt.
            suffix: String to go after the list of examples. Should generally
                set up the user's input.
            input_variables: A list of variable names the final prompt template
                will expect.
            example_separator: The separator to use in between examples. Defaults
                to two new line characters.
            prefix: String that should go before any examples. Generally includes
                examples. Default to an empty string.

        Returns:
            The final prompt generated.
        """
        template = example_separator.join([prefix, *examples, suffix])
        return cls(input_variables=input_variables, template=template, **kwargs)

    @classmethod
    def from_file(
        cls, template_file: Union[str, Path], input_variables: List[str], **kwargs: Any
    ) -> PromptTemplate:
        """Load a prompt from a file.

        Args:
            template_file: The path to the file containing the prompt template.
            input_variables: A list of variable names the final prompt template
                will expect.
        Returns:
            The prompt loaded from the file.
        """
        with open(str(template_file), "r") as f:
            template = f.read()
        return cls(input_variables=input_variables, template=template, **kwargs)

    @classmethod
    def from_template(cls, template: str, **kwargs: Any) -> PromptTemplate:
        """Load a prompt template from a template."""
        if "template_format" in kwargs and kwargs["template_format"] == "jinja2":
            # Get the variables for the template
            input_variables = _get_jinja2_variables_from_template(template)

        else:
            input_variables = {
                v for _, v, _, _ in Formatter().parse(template) if v is not None
            }

        if "partial_variables" in kwargs:
            partial_variables = kwargs["partial_variables"]
            input_variables = {
                var for var in input_variables if var not in partial_variables
            }

        return cls(
            input_variables=list(sorted(input_variables)), template=template, **kwargs
        )

# For backwards compatibility.
Prompt = PromptTemplate


import numpy as np
import time
from tqdm import tqdm
import matplotlib.pyplot as plt

from pydrake.all import InitializeAutoDiff, ExtractGradient
from alpha_gradient.objective_function import ObjectiveFunction
from alpha_gradient.statistical_analysis import compute_mean, compute_variance_norm
from heaviside_objective import HeavisideAllPositive

dmax = 50
n_gradient_samples = 10000
n_samples = 10000
sigma = 0.2
coordinate_range = np.linspace(-1.5, 1.5, dmax)
alpha = 0.5

fom_storage = np.zeros(dmax)
zom_storage = np.zeros(dmax)
aom_storage = np.zeros(dmax)
fom_zom_diff = np.zeros(dmax)
#aom_zom_diff = np.zeros(dmax)

fov_storage = np.zeros(dmax)
zov_storage = np.zeros(dmax)
#aov_storage = np.zeros(dmax)

f_storage = np.zeros(dmax)
F_storage = np.zeros(dmax)

for i in tqdm(range(len(coordinate_range))):
    d = 1
    lp_norm = HeavisideAllPositive(d)

    xval = coordinate_range[i]
    x = xval * np.ones(d)

    fobg_storage = np.zeros((n_gradient_samples, d))
    zobg_storage = np.zeros((n_gradient_samples, d))
    #aobg_storage = np.zeros((n_gradient_samples, d))

    """
    for k in range(n_gradient_samples):
        fobg_storage[k,:] = lp_norm.first_order_batch_gradient(
            x, n_samples, sigma/d)[0]
        zobg_storage[k,:] = lp_norm.zero_order_batch_gradient(
            x, n_samples, sigma/d)[0]
    """

    F_storage[i] = lp_norm.bundled_objective(
        x, n_samples, sigma/d)[0]
    f_storage[i] = lp_norm.evaluate(x, np.zeros(d))

    fom_storage[i] = lp_norm.first_order_batch_gradient(
        x, n_samples, sigma/d)[0]
    zom_storage[i] = lp_norm.zero_order_batch_gradient(
        x, n_samples, sigma/d)[0]

    """
    fom = compute_mean(fobg_storage)
    zom = compute_mean(zobg_storage)
    #aom = compute_mean(aobg_storage)

    fom_zom_diff[i] = np.linalg.norm(fom - zom)
    #aom_zom_diff[i] = np.linalg.norm(aom - zom)
    fom_storage[i] = np.linalg.norm(fom)
    zom_storage[i] = np.linalg.norm(zom)
    #aom_storage[i] = np.linalg.norm(aom)

    fov_storage[i] = compute_variance_norm(fobg_storage, 'fro')
    zov_storage[i] = compute_variance_norm(zobg_storage, 'fro')
    #aov_storage[i] = compute_variance_norm(aobg_storage, 'fro')
    """

"""
plt.figure(figsize=(8,4))
plt.subplot(2,1,1)
plt.plot(coordinate_range, fov_storage, 'r-', label=('FOBG Bias'))
plt.plot(coordinate_range, zov_storage, color='royalblue', label=('ZOBG Bias'))
#plt.plot(coordinate_range, aov_storage, color='springgreen', label=('AOBG Bias'))
plt.xlabel('Coordinate along diagonal line')
plt.legend()

plt.subplot(2,1,2)
plt.plot(coordinate_range, fom_zom_diff, 'r-', label=('FOBG Variance'))
plt.plot(coordinate_range, np.zeros(dmax), color='royalblue', label=('ZOBG Variance'))
#plt.plot(coordinate_range, aom_zom_diff, color='springgreen', label=('AOBG'))
plt.xlabel('Coordinate along diagonal line')
plt.legend()
#plt.savefig("results_zero.png")
plt.show()
"""

import matplotlib
matplotlib.rcParams['text.usetex'] = True

plt.figure(figsize=(16,3))
plt.subplot(1,4,1)
plt.plot(coordinate_range, f_storage, 'k-')
plt.title(r'$f(\mathbf{\theta})$', fontsize=26)
plt.xticks([]),plt.yticks([])

plt.subplot(1,4,2)
plt.plot(coordinate_range, F_storage, 'k-')
plt.title(r'$F(\mathbf{\theta})$', fontsize=26)
plt.xticks([]),plt.yticks([])

plt.subplot(1,4,3)
plt.plot(coordinate_range, fom_storage, 'k-')
plt.title(r'$\hat{\nabla}^1_{\mathbf{\theta}} F(\mathbf{\theta})$', fontsize=26)
plt.xticks([]),plt.yticks([])

plt.subplot(1,4,4)
plt.plot(coordinate_range, zom_storage, 'k-')
plt.title(r'$\hat{\nabla}^0_{\mathbf{\theta}} F(\mathbf{\theta})$', fontsize=26)
plt.xticks([]),plt.yticks([])
plt.show()


from collections import deque
from functools import partial
from tabulate import tabulate

counter = 0
x = "12341234"
y = "12345123"
data = []

def basicOps():
    global counter
    return counter

# Divide and Conquer
def all_alignments(x, y):
    """Return an iterable of all alignments of two
    sequences.

    x, y -- Sequences.
    """

    def F(x, y):
        """A helper function that recursively builds the
        alignments.

        x, y -- Sequence indices for the original x and y.
        """
        if len(x) == 0 and len(y) == 0:
            yield deque()

        scenarios = []
        if len(x) > 0 and len(y) > 0:
            scenarios.append((x[0], x[1:], y[0], y[1:]))
        if len(x) > 0:
            scenarios.append((x[0], x[1:], None, y))
        if len(y) > 0:
            scenarios.append((None, x, y[0], y[1:]))

        # NOTE: "xh" and "xt" stand for "x-head" and "x-tail",
        # with "head" being the front of the sequence, and
        # "tail" being the rest of the sequence. Similarly for
        # "yh" and "yt".
        for xh, xt, yh, yt in scenarios:
            for alignment in F(xt, yt):
                alignment.appendleft((xh, yh))
                yield alignment

    alignments = F(range(len(x)), range(len(y)))
    return map(list, alignments)

def print_alignment(x, y, alignment):
    print("".join(
        "-" if i is None else x[i] for i, _ in alignment
    ))
    print("".join(
        "-" if j is None else y[j] for _, j in alignment
    ))

def alignment_score(x, y, alignment):
    """Score an alignment.

    x, y -- sequences.
    alignment -- an alignment of x and y.
    """
    global counter
    score_gap = 2
    score_same = 0
    score_different = 1

    score = 0
    for i, j in alignment:
        counter += 1
        if (i is None) or (j is None):
            score += score_gap
        elif x[i] == y[j]:
            score += score_same
        elif x[i] != y[j]:
            score += score_different

    return score

def align_bf(x, y):
    """Align two sequences, minimizing the
    alignment score, using brute force.

    x, y -- sequences.
    """
    return min(
        all_alignments(x, y),
        key=partial(alignment_score, x, y),
    )

print("Alignment of Divide and Conquer:")
print_alignment(x, y, align_bf(x, y))
print("Total Cost/Penality For Divide and Conquer: %s" %alignment_score(x, y, alignment=align_bf(x,y)))
print("")
print("Divide and Conquer Table:")
data.append([basicOps()])
print(tabulate(data, headers=["# of times the basic operation"]))


#!/usr/bin/python3.6
# coding: utf-8
'''
calibrate MISR dataset with ATM measurement data and returns the training dataset

input: dir path where is home to masked_toa_refl & ATM data
output: single datasets.csv for each ATM.csv; and then will merge all output single datasets together in the end and will return a final merged dataset for filtering

to-do:
- do not check empty lists
- check all code again
'''
'''
notes:
	-999.0 represents no MISR toa file
	-998 is the first line
'''

import MisrToolkit as mtk
import pandas as pd
import numpy as np
import sys, glob, os, csv
from platform import python_version
import datetime as dt

# !pip uninstall jedi --yes

# check python version be 3.6 cuz MTK works with 3.6

# py_version = sys.version
# py_ver = py_version[0:3]
# if (py_ver != str(3.6)):

# another way to get verion. note: this f() returns py version as str dtype, we change it to float
py_ver = float(python_version()[0:3])
print(py_ver)
if (py_ver != 3.6):
	print('python version is NOT 3.6- MTK does not work')
	raise SystemExit()
else:
	print('python version: %s' %py_ver)

##- set up paths
#################### input dir should include 9 dir
masked_toa_home = "/media/ehsan/Gdrive_18TB/all_masked_toa_refl_2010_2014_2019"
# masked_toa_home = "/media/ehsan/6T_part1/2016/april_2016/14528_apr2016/project_april_2016_9cam3bands/masked_toa_refl_april_2016_9cam4bands_day1_30_p1_233_b1_46"

atm_dir = "/home/ehsan/misr_lab/ATM_2010_2014_2019"
# atm_dir = "/media/ehsan/6T_part1/2016/april_2016/14528_apr2016/project_april_2016_3cam/ATM_apr2016_5days"

#################### output
trainingDS_dir = "/media/ehsan/Gdrive_18TB/training_DS_2010_2014_2019"
# trainingDS_dir = "/media/ehsan/6T_part1/2016/april_2016/14528_apr2016/project_april_2016_3cam/training_data/april_2016"

#################### setup pattern- do not change
atm_file_pattern = 'ILATM2*'+'.csv'
single_csv_ds_label = "9cam3bands"

# atm_file_sample = 'ILATM2_20160420_175457_smooth_nadir3seg_50pt.csv'
# atm_file_fp = os.path.join(atm_dir, atm_file_sample)
# check ATM file exists
# print(os.path.isfile(atm_file_fp))
# print(atm_file_fp)

# check runtime
t1 = dt.datetime.now()

#### create datadet table
# cam_labels = ['Da_r','Ca_r','Ba_r','Aa_r','An_r','An_g','An_b','An_nir','Af_r','Bf_r','Cf_r','Df_r','mean_ATM_roughness']

# add all found fields to pandas.DataFrame and make a dataset with column labels and filter for similar pixels
# column_names = ['path','orbit','block','line','sample','lat','lon', 'points_in_pixel', cam_labels[0],cam_labels[1],cam_labels[2],cam_labels[3],cam_labels[4],cam_labels[5],cam_labels[6],cam_labels[7],cam_labels[8],cam_labels[9],cam_labels[10],cam_labels[11],'mean_ATM_roughness']

ds_row_index = 0

column_names = ['path','orbit','block','line','sample','lat','lon',\
				'Da_r','Ca_r','Ba_r','Aa_r','An_r','An_g','An_b','An_nir','Af_r','Bf_r','Cf_r','Df_r',\
				'mean_ATM_roughness']#,'ATM_start_time','ATM_end_time']

final_ds_list = []

## get a list of ATM csv files
atm_list = glob.glob(os.path.join(atm_dir, atm_file_pattern))
total_atm_files_found = len(atm_list)
print('total ATM.csv files: %d' %total_atm_files_found)

## create pandas DF and set index to DF based on pre-estimate: 100000 per each ATM file
# final_df = pd.DataFrame(columns=column_names) #, index=range(total_atm_files_found*1000))  # if nut set the index number correctly, returns segmentation fault- Q- how create a DF with variable size?

## this does not work
# final_ds.set_index(total_atm_files_found*10000)

# open each ATM file and read each row and will eventually write out a train dataset for each ATM file
for atm_cntr, ATMfile in enumerate(atm_list):

	print('\n********** processing ATM file: (%d/%d)' %(atm_cntr+1, len(atm_list)))
	print("try to use ATM time and locations to find MISR pixel...\n")
	print(ATMfile)

	atm_filelabel = ATMfile.split('/')[-1]
	atm_filelabel_date_list = atm_filelabel.split('_')[0:3]
	atm_filelabel_date = atm_filelabel_date_list[0]+"_"+atm_filelabel_date_list[1]+"_"+atm_filelabel_date_list[2]
	output_ds_label = single_csv_ds_label+'_'+atm_filelabel_date+'.csv'
	print("output csv: %s" %output_ds_label)

	output_file_fp = os.path.join(trainingDS_dir, output_ds_label)
	if (os.path.isfile(output_file_fp)):
		print("CSV file exists on disk, continue to next ATM file")
		continue

	# get date-time from ATM file label
	atm_label = ATMfile.split('/')[-1]
	yrmonday = atm_label.split('_')[1]
	atm_yr = yrmonday[0:4]
	atm_mon = yrmonday[4:6]
	atm_day = yrmonday[6:8]
	# print(atm_yr)
	# print(atm_mon)
	# print(atm_day)

	hrminsec = atm_label.split('_')[2]
	atm_hr = hrminsec[0:2]
	atm_min = hrminsec[2:4]
	atm_sec = hrminsec[4:6]

	ATM_start_time = atm_yr+'-'+atm_mon+'-'+atm_day+"T00:00:00Z" # YYYY-MM-DDThh:mm:ssZ
	ATM_end_time = atm_yr+'-'+atm_mon+'-'+atm_day+"T23:59:59Z" # YYYY-MM-DDThh:mm:ssZ

	ATM_start_time_hrminsec = atm_yr+'-'+atm_mon+'-'+atm_day+"T"+atm_hr+":"+atm_min+":"+atm_sec+"Z" # YYYY-MM-DDThh:mm:ssZ
	ATM_end_time_hrminsec = atm_yr+'-'+atm_mon+'-'+atm_day+"T"+atm_hr+":"+atm_min+":"+atm_sec+"Z" # YYYY-MM-DDThh:mm:ssZ

	print("ATM start: %s" %ATM_start_time)
	print("ATM end:   %s" %ATM_end_time)

	# get a list of orbits for date-time
	orbit_list_today = mtk.time_range_to_orbit_list(ATM_start_time, ATM_end_time)
	# print('orbits today: %s' %len(orbit_list_today))
	# print(orbit_list_today)

	# for each orbit we find path number and use that to find MISR pixel
	for misr_orbit_today in orbit_list_today:
		# print('\nprocessing orbit: %d' %misr_orbit_today)

		# orbit to path
		misr_pathNumber_today = mtk.orbit_to_path(misr_orbit_today)
		# print('path: %s' %misr_pathNumber_today)

		# open and read ATM file
		# atm_file_columnt = []

		input_atm_file = pd.read_csv(ATMfile, comment='#', sep=',') #skiprows=15, header=0)

		# input_atm_file.head(5)

		# find total atm_rows to make DF with Nan
		csv_total_rows = input_atm_file.shape[0]
		# print(csv_total_rows)

		# read each row in ATM-file and extract latLong
		for atm_row_num in range(csv_total_rows):
			# print(type(atm_row_num))
			atm_lat = input_atm_file.iloc[atm_row_num, 1]
			atm_lon = input_atm_file.iloc[atm_row_num, 2]
			atm_roughness = input_atm_file.iloc[atm_row_num, 6]

			# print('atm lat %s' %atm_lat)
			# print('atm lon %s' %atm_lon)
			# print('roughness %s' %atm_roughness)

			# print('finidng list of paths for ATM location...')
			try:  # to ignore Exception for some path numbers
				paths_cover_ATM_location = mtk.latlon_to_path_list(atm_lat, atm_lon)
				# print('MISR paths cover ATM latlon:')
				# print(paths_cover_ATM_location)
			except Exception:
				print('Exception was raised- continue to next ATM row')
				continue

			if misr_pathNumber_today not in paths_cover_ATM_location:
				# print('MISR path %d in day %s not match with path that ATM lat-lon falls in, continue to next ATM latLon row!' %(misr_pathNumber_today, atm_day))
				continue    # to next location == latLon

			else:
				# print('MISR path %d covers ATM location- we will find Block-Line-Sample!' %misr_pathNumber_today)

				# find MISR pixel from that
				resolution_meters = 275

				misr_block, misr_pixel_x, misr_pixel_y = mtk.latlon_to_bls(misr_pathNumber_today, resolution_meters, atm_lat, atm_lon) # q- why crash? these paths cover that that ATM location

				# print('block: %s' %misr_block)
				# print('pixel x: %s' %misr_pixel_x)
				# print('pixel y: %s' %misr_pixel_y)

				path_num_str = str(misr_pathNumber_today).zfill(3)
				misr_orbit_str = str(misr_orbit_today).zfill(6)
				misr_block_str = str(misr_block).zfill(3)

				# print('path-str: %s' %path_num_str)
				# print('orbit-str: %s' %misr_orbit_str)
				# print('block-str: %s' %misr_block_str)

				#### create fulll path to 9 cameras and 4 spectral bands

				# find MISR files associated with P-O-B: masked_toa_P_O_B
				# aft/back cameras
				aa_red_fp = os.path.join(masked_toa_home,'Aa', ('masked_toa_refl_P'+path_num_str+'_'+'O'+misr_orbit_str+'_'+'B'+misr_block_str+'_aa_red.dat'))
				ba_red_fp = os.path.join(masked_toa_home,'Ba', ('masked_toa_refl_P'+path_num_str+'_'+'O'+misr_orbit_str+'_'+'B'+misr_block_str+'_ba_red.dat'))
				ca_red_fp = os.path.join(masked_toa_home,'Ca', ('masked_toa_refl_P'+path_num_str+'_'+'O'+misr_orbit_str+'_'+'B'+misr_block_str+'_ca_red.dat'))
				da_red_fp = os.path.join(masked_toa_home,'Da', ('masked_toa_refl_P'+path_num_str+'_'+'O'+misr_orbit_str+'_'+'B'+misr_block_str+'_da_red.dat'))

				# front cameras
				af_red_fp = os.path.join(masked_toa_home,'Af', ('masked_toa_refl_P'+path_num_str+'_'+'O'+misr_orbit_str+'_'+'B'+misr_block_str+'_af_red.dat'))
				bf_red_fp = os.path.join(masked_toa_home,'Bf', ('masked_toa_refl_P'+path_num_str+'_'+'O'+misr_orbit_str+'_'+'B'+misr_block_str+'_bf_red.dat'))
				cf_red_fp = os.path.join(masked_toa_home,'Cf', ('masked_toa_refl_P'+path_num_str+'_'+'O'+misr_orbit_str+'_'+'B'+misr_block_str+'_cf_red.dat'))
				df_red_fp = os.path.join(masked_toa_home,'Df', ('masked_toa_refl_P'+path_num_str+'_'+'O'+misr_orbit_str+'_'+'B'+misr_block_str+'_df_red.dat'))

				# nadir
				an_red_fp = os.path.join(masked_toa_home,'An', ('masked_toa_refl_P'+path_num_str+'_'+'O'+misr_orbit_str+'_'+'B'+misr_block_str+'_an_red.dat'))
				an_green_fp = os.path.join(masked_toa_home,'An', ('masked_toa_refl_P'+path_num_str+'_'+'O'+misr_orbit_str+'_'+'B'+misr_block_str+'_an_green.dat'))
				an_blue_fp = os.path.join(masked_toa_home,'An', ('masked_toa_refl_P'+path_num_str+'_'+'O'+misr_orbit_str+'_'+'B'+misr_block_str+'_an_blue.dat'))
				an_nir_fp = os.path.join(masked_toa_home,'An', ('masked_toa_refl_P'+path_num_str+'_'+'O'+misr_orbit_str+'_'+'B'+misr_block_str+'_an_NIR.dat')) # note here: should match the file names on drive ("an_NIR" is ok, not "an_nir")

				#### check if camera files availabel

				# create a list of camreas based on order from Da->Df- order if important
				masked_toaFile_orderedList = [da_red_fp, ca_red_fp, ba_red_fp, aa_red_fp,
											  an_red_fp, an_green_fp, an_blue_fp, an_nir_fp,
											  af_red_fp, bf_red_fp, cf_red_fp, df_red_fp]

				available_file_status = []
				# check file exists
				for masked_toa_file in masked_toaFile_orderedList:
					# print(masked_toa_file)
					available_status = os.path.isfile(masked_toa_file)
					# print(available_status)
					available_file_status.append(available_status)

				##-- check if all 12 files/cameras are available

				# if (all(available_file_status)==False): # check if any element is false==file not found
				#     # print('***some masked-files missing- continue*** \n')
				#     continue # to the next ATM row

				if (not any(available_file_status)):
					# print('did not find even a single camera value for ATM location/row- continue')
					continue

				# print("num. of masked toa files found:")
				# print(available_file_status)

				# open masked-toa-refl and extract pixel value
				pixel_values = []

				for maskedTOA_rawfile in masked_toaFile_orderedList:
					# print(maskedTOA_rawfile)
					is_file = os.path.isfile(maskedTOA_rawfile)
					# print(is_file)

					#-- if one camera is missing, assign fill value for that camera that was not found
					if (is_file==False):
						pixel_values.append(-999.0)
					else:
						rough_2d_arr = np.fromfile(maskedTOA_rawfile, dtype=np.double)[0:1048576].reshape((512,2048))   # 'double==float64'     # is this roughness in cm?
						# print(rough_2d_arr.shape)
						# print(type(rough_2d_arr))
						##- maybe make a image from it?
						pixel_value = rough_2d_arr[int(misr_pixel_x), int(misr_pixel_y)]
						pixel_values.append(round(pixel_value,5))

				# print('path-str: %s' %path_num_str)
				# print('orbit-str: %s' %misr_orbit_str)
				# print('block-str: %s' %misr_block_str)

				# print("found these pixel values for an ATM location:")
				# print(pixel_values)

				########################################################
				# new way of making a list and then open it in dataframe

				final_ds_values = [misr_pathNumber_today, misr_orbit_today, misr_block,\
				 int(misr_pixel_x), int(misr_pixel_y), round(atm_lat,7), round(atm_lon,7), \
				 pixel_values[0],pixel_values[1],pixel_values[2],pixel_values[3],pixel_values[4],\
				 pixel_values[5],pixel_values[6],pixel_values[7],pixel_values[8],pixel_values[9],\
				 pixel_values[10],pixel_values[11],atm_roughness]#,\
				 # ATM_start_time_hrminsec,ATM_end_time_hrminsec]

				zipped = zip(column_names, final_ds_values)
				a_dictionary = dict(zipped)
				# print(a_dictionary)
				final_ds_list.append(a_dictionary)

				ds_row_index +=1

	##################################################################
	# write out the train dataset for each ATM file

	if (len(final_ds_list) == 0):
		print(final_ds_list)
		print("ATM list is empty, we write -998 flag & continue to next ATM file")
		print("note: -998 means we did not find any MISR pixel that covers ATM locations in that file.")

		final_ds_list = [-998,-998,-998,-998,-998,-998,-998,-998,-998,-998,-998,-998,-998,-998,-998,-998,-998,-998,-998,-998]#,-998,-998]

		with open(output_file_fp, 'w', encoding='UTF8', newline='') as fileObj:
			writer = csv.writer(fileObj)  # create a new instance of the DictWriter class by passing the file object
			writer.writerow(column_names)
			writer.writerow(final_ds_list)

		final_ds_list.clear()
		# continue
		print(output_file_fp)

	else:
		print('writing out single CSV file at a time...')

		with open(output_file_fp, 'w', encoding='UTF8', newline='') as fileObj:
			writer = csv.DictWriter(fileObj, fieldnames=column_names)  # create a new instance of the DictWriter class by passing the file object
			writer.writeheader()
			writer.writerows(final_ds_list)

		print(output_file_fp)
		print('successfully finished writing a single CSV file!')

		## empty the list
		print("before empty the list...")
		print(len(final_ds_list))
		final_ds_list.clear()
		print("after empty the list...")
		print(len(final_ds_list))

# check runtime
t2 = dt.datetime.now()
runtime = t2-t1
print("\nruntime: %s" %runtime)
print("\nFINISHED SUCCESSFULLY: we processed %d csv files. In next step we should use another code to merge all single csv files..." %atm_cntr)

#########################################################################################################################################################
# ## now merge all csv files into a single dataframe and write the output
# print("\n")
# print("now merging all CSV files...")
# csv_file_pattern = "april_2016_9cam_4bands_ILATM2*"+".csv"
# csv_files_list = glob.glob(os.path.join(trainingDS_dir, csv_file_pattern))
# print(len(csv_files_list))
# ## join files
# final_df = pd.concat(map(pd.read_csv, csv_files_list), ignore_index=False, axis=0)  # true: assigns a continuous index numbers for the merged DF; axis=0 along the rows
# out_DS  = os.path.join(trainingDS_dir, output_final_ds_label)
# final_df.to_csv(out_DS, index=False) # do not include index column in output csv
# print(out_DS)
# print('successfully finished writing final DataFrame!')

#########################################################################################################################################################

######## old code
		## dataframe method
		# final_df = pd.DataFrame(columns=column_names)
		# final_df = final_df.append(final_ds_list, ignore_index=True) # add list to existing dataframe; True: preserve the DataFrame indices;

		# out_DS  = os.path.join(trainingDS_dir, output_ds_label)
		# final_df.to_csv(out_DS)
		# print(out_DS)
		# final_df=None
		# final_df.drop(columns=column_names, axis=1, inplace=True)

		## csv writer method
		# ## if each row is a list
		# with open (output_file_fp, "w") as fileObj:
		# 	writer = csv.writer(fileObj)
		# 	writer.writerow(column_names)
		# 	writer.writerows(final_ds_list)

		## if each row is a dictionary


from django.core.validators import MinValueValidator, MaxValueValidator
from django.db import models
from django.contrib.auth.models import User, Group
from phonenumber_field.modelfields import PhoneNumberField

limit = 61     # Separation of Students in A and B

Departments = [
        ("1", "Aerospace"),
        ("2", "Civil"),
        ("3", "Computer Science"),
        ("4", "Electronics & Communication"),
        ("5", "Electrical"),
        ("6", "Mechanical"),
        ("7", "Materials & Metallurgical"),
        ("8", "Production & Industrial"),
    ]

Designations = [
        ("1", "Professor"),
        ("2", "Associate Professor"),
        ("3", "Assistant Professor"),
        ("4", "PhD Scholar")
    ]

CourseList = [
        ('CSN101', 'INTRODUCTION TO COMPUTER SCIENCE AND ENGINEERING', '2'),
        ('CSN103', 'DIGITAL ELECTRONIC AND LOGIC DESIGN', '4'),
        ('CSN201', 'DISCRETE STRUCTURES FOR COMPUTER SCIENCE', '4'),
        ('CSN202', 'COMPUTER ARCHITECTURE AND ORGANIZATION', '4'),
        ('CSN203', 'OBJECT ORIENTED PROGRAMMING', '4 '),
        ('CSN204', 'ANALYSIS AND DESIGN OF ALGORITHMS', '4'),
        ('CSN206', 'ENGINEERING ANALYSIS ', '4'),
        ('CSN207', 'MICROPROCESSOR AND ITS APPLICATIONS', '4'),
        ('CSN208', 'DATA BASE MANAGEMENT SYSTEM', '4'),
        ('CSN210', 'COMPUTER ', '4'),
        ('CSN301', 'THEORY OF COMPUTATION', '4'),
        ('CSN302', 'SOFTWARE ENGINEERING', '4'),
        ('CSN303', 'WEB TECHNOLOGIES', '4  '),
        ('CSN304', 'COMPUTER GRAPHICS', '4'),
        ('CSN305', 'ARTIFICIAL INTELLIGENCE', '4'),
        ('CSN401', 'COMPILER DESIGN', '4'),
        ('CSN402', 'SOFTWARE TESTING ', '4'),
        ('CSN403', 'SOFT COMPUTING', '4'),
        ('CSN404', 'DIGITAL IMAGE PROCESSING', '4'),
        ('CSN405', 'CLOUD COMPUTING', '4'),
        ('CSN406', 'AGILE SOFTWARE DEVELOPMENT ', '4'),
        ('CSN407', 'Natural Language Processing', '4'),
        ('CSN408', 'SOFTWARE PROJECT MAN', '4'),
        ('CSN409', 'BIG DATA ANALYTICS', '4'),
        ('CSN410', 'BIOINFORMATICS', '4'),
        ('CSN411', 'NETWORK SECURITY', '4'),
        ('CSN412', 'APPLIED CRYPTOGRAPHY', '4'),
        ('CSN413', 'COMPUTER CRIME INVESTIGATION AND FORENSICS', '4'),
        ('CSN414', 'BIOMETRIC SECURITY', '4'),
        ('CSN415', 'Advanced', '4'),
        ('CSN416', 'ADVANCED WIRELESS AND MOBILE NETWORKS', '4'),
        ('CSN417', 'WIRELESS SENSOR NETWORKS', '4'),
        ('CSN 418', 'MOBILE COMPUTING', '4'),
        ('CSN461', 'OBJECT ORIENTED PROGRAMMING', '4'),
        ('CSN462', 'OPERATING SYSTEMS', '4'),
        ('CSN463', 'WEB ', '4  '),
        ('CSN421', 'MACHINE LEARNING', '4'),
        ('CSN422', 'ADVANCED DATABASE SYSTEMS', '4'),
        ('CSN423', 'ADVANCED SOFTWARE EN', '4'),
        ('CSN424', 'ADVANCED ALGORITHM DESIGN AND ANALYSIS', '4'),
        ('CSN425', 'ADVANCED COMPUTER ARCHITECTURE', '4'),
        ('CSN431', 'DATA STRUCTURES ', '4 '),
        ('CSN432', 'COMPUTER NETWORKS', '4'),
        ('CSN433', 'COMPUTER CRIME INVESTIGATION AND FORENSICS', '4 '),
        ('CSN434', 'DATA BASE SYSTEMS', '4'),
        ('CSN435', 'SOFTWARE ENGINEERING', '4'),
        ('GSC101', 'ENVIRONMENTAL SCIENCES', '3'),
        ('MAN 101', 'MATHEMATICS I', '4'),
        ('MAN 103', 'PROBABILITY AND STATISTICS', '4'),
        ('MAN105', 'VECTOR CALCULUS, FOURIER SERIES AND LAPLACE ', '4'),
        ('MAN 106', 'PARTIAL DIFFERENTIAL EQUATIONS AND SPECIAL ', '4'),
        ('PYN101', 'OSCILLATIONS AND OPTICS', '4'),
        ('PYN102', 'CONDENSED MATTER PHYSICS', '4'),
        ('PYN', 'MECHANICS', '4'),
        ('CHN101', 'APPLIED CHEMISTRY', '4'),
        ('CHN', 'PHYSICAL CHEMISTRY', '4'),
        ('CHN', 'INORGANIC ', '4'),
        ('CHN', 'PHYSICAL CHEMISTRY', '4'),
        ('HSS 101', 'ETHICS AND SELF AWARENESS     ', '2'),
        ('HSS 103', 'COMMUNICATION SKILLS (ADVANCED)', '2'),
        ('HSS 201', 'ECONOMICS', '3'),
        ('HSS 202', 'PSYCHOLOGY', '3'),
        ('HSS 203', 'SOCIOLOGY', '3'),
        ('HSS 204', 'FRENCH   ', '3'),
        ('HSM 401', 'PRINCIPLES OF MANAGEMENT', '3'),
        ('HSM 402', 'BUSINESS ENVIRONMENT AND BUSINESS LAWS     ', '3'),
        ('HSM 403', 'ENTREPRENEURSHIP AND PROJECT MANAGEMENT', '3'),
        ('HSM 405', 'MARKETING MANAGEMENT     ', '3'),
        ('HSM 406', 'HUMAN RESOURCE MANAGEMENT     ', '3'),
        ('CSN104', 'COMPUTER PROGRAMMING (BASIC)', '4'),
        ('CSN105', 'COMPUTER PROGRAMMING (ADVANCED)', '4'),
        ('ESC 101', 'ENGINEERING DRAWING ', '4'),
        ('ESC102', 'FLUID MECHANICS', '4'),
        ('ESC 103', 'INTRODUCTION TO MANUFACTURING', '4'),
        ('ESC 201', 'THERMODYNAMICS', '4'),
        ('ESC202', 'ESSENTIALS OF INFORMATION TECHNOLOGY', '4'),
        ('ESC 203', 'MATERIALS SCIENCE', '04'),
        ('ESC 204', 'SOLID MECHANICS', '4'),
        ('ESC 205', 'INTRODUCTION TO ELECTRONICS', '4'),
        ('ESC 206', 'BASIC ELECTRICAL SCIENCES', '04'),
        ('ESC 207', 'MECHATRONICS', '4'),
        ('ESC', 'MECHANICAL ENGINEERING DRAWING', '4'),
        ('CSN205', 'TECHNICAL COMMUNICATION', '2'),
        ('MAN 401', 'OPERATIONS  RESEARCH', '4'),
        ('PYN', 'ADVANCED PHYSICS ', '4'),
        ('PYN', 'CRYSTAL PHYSICS', '4'),
        ('CHN 401', 'MODERN INSTRUMENTAL METHODS OF CHEMICAL ANALYSIS', '4'),
        ('HSM 401', 'PRINCIPLES OF MANAGEMENT', '4'),
        ('HSM 402', 'BUSINESS ENVIRONMENT AND BUSINESS LAWS     ', '4'),
        ('HSM 404', 'FINANCIAL MANAGEMENT', '4'),
        ('HSM 405', 'MARKETING MANAGEMENT', '4'),
        ('HSM 406', 'HUMAN RESOURCE MANAGEMENT', '4'),
        ('HSM 431', 'MANAGING INNOVATION AND CHANGE', '4'),
        ('HSM 432', 'BUSINESS RESEARCH ', '4'),
        ('MAN 431', 'ALGEBRA ', '4'),
        ('MAN 432', 'NUMBER THEORY ', '4'),
        ('MAN 433', 'FOURIER SERIES AND INTEGRAL TRANSFORMS', '4'),
        ('MAN 434', 'CALCULUS OF VARIATIONS', '4'),
        ('PYN', 'QUANTUM MECHANICS', '4'),
        ('PYN', 'STATISTICAL PHYSICS', '4'),
        ('PYN', 'NUCLEAR PHYSICS', '4'),
        ('PYN', 'EXPERIMENTAL NUCLEAR PHYSICS', '4'),
        ('PYN', 'X', '4'),
        ('CHN', 'ANALYTICAL CHEMISTRY', '4'),
        ('CHN', 'ENVIRONMENTAL CHEMISTRY', '4'),
        ('CHN', 'RECENT ADVANCES IN CHEMICAL SCIENCES', '4'),

]

class Course(models.Model):

    CourseId = models.CharField(max_length=8, blank=False, unique=False)
    CourseName = models.CharField(max_length=250)
    Credit = models.IntegerField( validators=[
            MaxValueValidator(4),
            MinValueValidator(1)
        ])

    def __str__(self):
        return self.CourseId + " - " + self.CourseName

class Student(models.Model):

    User = models.OneToOneField(User, on_delete=models.CASCADE, default=None)
    StudentID = models.CharField(max_length=8, blank=False, unique=True)
    Name = models.CharField(max_length=50)
    Branch = models.IntegerField()
    YearOfStudy = models.IntegerField()
    ContactNumber = PhoneNumberField(blank=False)
    Email = models.EmailField(blank=False)
    Courses = models.ManyToManyField(Course, through='StudentCourse')

    def add_into_group(self):

        if self.StudentID[:5] == "16103":

            maingroup = Group.objects.get(name="CSE")
            self.User.groups.add(maingroup)

            if int(self.StudentID[5:]) < limit:
                self.User.groups.add(Group.objects.get(name="CSEA"))
            else:
                self.User.groups.add(Group.objects.get(name="CSEB"))

    def __str__(self):
        return self.StudentID

    def get_branch_display(self):

        return Departments[int(self.Branch)][1]

class Faculty(models.Model):

    User = models.OneToOneField(User, on_delete=models.CASCADE, default=None)
    FacultyID = models.CharField(max_length=8, blank=False, unique=True)
    Name = models.CharField(max_length=50)
    Department = models.IntegerField()
    Designation = models.IntegerField()
    ContactNumber = PhoneNumberField(blank=False)
    Email = models.EmailField(blank=False)
    Courses = models.ManyToManyField(Course, through='FacultyCourse')

    def add_into_group(self):

        maingroup = Group.objects.get(name="FAC")
        self.User.groups.add(maingroup)

    def get_department_display(self):

        return Departments[int(self.Department)][1]

    def get_designation_display(self):

        return Designations[int(self.Designation)][1]

    def __str__(self):
        return self.FacultyID

class StudentCourse(models.Model):

    Course = models.ForeignKey(Course, on_delete=models.CASCADE)
    User = models.ForeignKey(Student, on_delete=models.CASCADE)
    Semester = models.IntegerField(validators=[MinValueValidator(1), MaxValueValidator(8)])

class FacultyCourse(models.Model):

    Course = models.ForeignKey(Course, on_delete=models.CASCADE)
    User = models.ForeignKey(Faculty, on_delete=models.CASCADE)
    Semester = models.IntegerField(validators=[MinValueValidator(1), MaxValueValidator(8)])


from __future__ import print_function
import argparse
import cv2
import numpy as np
from keras.preprocessing import image as image_utils
from PIL import Image

ap = argparse.ArgumentParser()
ap.add_argument("-i", "--image", required = True,
	help = "path to image")
args = vars(ap.parse_args())
image = cv2.imread(args["image"])
print("width:{} pixels".format(image.shape[1]))
print("height: {} pixels".format(image.shape[0]))
#cv2.imshow("Image", image)
cv2_im = cv2.cvtColor(image,cv2.COLOR_BGR2RGB)
#pil_im = Image.open(args["image"])
pil_im = Image.fromarray(cv2_im)
pil_im.show()
cv2.waitKey(0)
cv2.imwrite("copyImage.jpg", image)
image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
image = cv2.GaussianBlur(image,(5,5),0)
#cv2.imshow("Blurred",image)
canny = cv2.Canny(image,30,150)
#cv2.imshow("Canny", canny)
#cv2.waitKey(0)
(_,cnts, _) = cv2.findContours(canny.copy(), cv2.RETR_EXTERNAL,
	cv2.CHAIN_APPROX_SIMPLE)
print("number of countours= {}".format(len(cnts)))
#
# bounding box
rectWithArea = []
for cnt in cnts :
	rect = cv2.minAreaRect(cnt)
	box = cv2.boxPoints(rect)
	box = np.int0(box)
	area = box[1][0]* box[1][1]
	rectWithArea.append((box,area))
sortedTups = sorted(rectWithArea, key=lambda t :t[1], reverse=True)
sortedBoxes = [t[0] for t in sortedTups]

#cntWithArea = []
#for cnt in cnts :
#	area = cv2.contourArea(cnt)
#	cntWithArea.append((cnt,area))
#sortedTups = sorted(cntWithArea, key=lambda t :t[1], reverse=True)
#sortedCnts = [t[0] for t in sortedTups]
#for i in range(10):
#	print (sortedTups[i][1])
#
fish = image.copy()

cv2.drawContours(fish, sortedBoxes, -1, (0,255,0), 2)

#cv2.imshow("fish", fish)
cv2.waitKey(0)


import base64
import zlib

def file2b64(inFile,gzip=True):
	if type(inFile)==file:
		if 'r' not in inFile.mode:
			raise IOError("File isn't open for reading")
	elif type(inFile)==str:
		path=inFile
		inFile=open(path,'r')
	else:
		raise ValueError("inFile must be a str or file object (and be open for reading) ")
	fileValue=getFileValue(inFile)
	value=''
	if gzip:
		value=zlib.compress(fileValue)
	else:
		value=fileValue
	return base64.b64encode(value)

def getFileValue(inFile):
	if type(inFile)==file:
		if 'r' not in inFile.mode:
			raise IOError("File isn't open for reading")
	else:
		raise ValueError("inFile must be a file object")
	value=""
	while True:
		line=inFile.read()
		if line=='':
			break
		else:
			value+=line

	return value


import pytest

from vyper import compiler
from vyper.exceptions import (
    EnumDeclarationException,
    InvalidOperation,
    NamespaceCollision,
    StructureException,
    TypeMismatch,
    UnknownAttribute,
)

fail_list = [
    (
        """
event Action:
    pass

enum Action:
    BUY
    SELL
    """,
        NamespaceCollision,
    ),
    (
        """
enum Action:
    pass
    """,
        EnumDeclarationException,
    ),
    (
        """
enum Action:
    BUY
    BUY
    """,
        EnumDeclarationException,
    ),
    ("enum Foo:\n" + "\n".join([f"    member{i}" for i in range(257)]), EnumDeclarationException),
    (
        """
enum Roles:
    USER
    STAFF
    ADMIN

@external
def foo(x: Roles) -> bool:
    return x in [Roles.USER, Roles.ADMIN]
    """,
        TypeMismatch,
    ),
    (
        """
enum Roles:
    USER
    STAFF
    ADMIN

@external
def foo(x: Roles) -> Roles:
    return x.USER  # can't dereference on enum instance
    """,
        StructureException,
    ),
    (
        """
enum Roles:
    USER
    STAFF
    ADMIN

@external
def foo(x: Roles) -> bool:
    return x >= Roles.STAFF
    """,
        InvalidOperation,
    ),
    (
        """
enum Functions:
    def foo():nonpayable
    """,
        EnumDeclarationException,
    ),
    (
        """
enum Numbers:
    a:constant(uint256) = a
    """,
        EnumDeclarationException,
    ),
    (
        """
enum Numbers:
    12
    """,
        EnumDeclarationException,
    ),
    (
        """
enum Roles:
    ADMIN
    USER

@external
def foo() -> Roles:
    return Roles.GUEST
    """,
        UnknownAttribute,
    ),
    (
        """
enum A:
    a
enum B:
    a
    b

@internal
def foo():
    a: A = B.b
    """,
        TypeMismatch,
    ),
]

@pytest.mark.parametrize("bad_code", fail_list)
def test_fail_cases(bad_code):
    with pytest.raises(bad_code[1]):
        compiler.compile_code(bad_code[0])

valid_list = [
    """
enum Action:
    BUY
    SELL
    """,
    """
enum Action:
    BUY
    SELL
@external
def run() -> Action:
    return Action.BUY
    """,
    """
enum Action:
    BUY
    SELL

struct Order:
    action: Action
    amount: uint256

@external
def run() -> Order:
    return Order({
        action: Action.BUY,
        amount: 10**18
        })
    """,
    "enum Foo:\n" + "\n".join([f"    member{i}" for i in range(256)]),
    """
a: constant(uint256) = 1

enum A:
    a
    """,
]

@pytest.mark.parametrize("good_code", valid_list)
def test_enum_success(good_code):
    assert compiler.compile_code(good_code) is not None


import random
import string
import pytest

from selenium import webdriver as webdriver
from selenium.webdriver.chrome.service import Service as ChromeService
from selenium.webdriver.common.by import By

from page_objects.home_page import HomePage

def pytest_addoption(parser):
    parser.addoption(
        "--browser",
        default="chrome",
        help="Options: firefox, chrome, opera. Default: chrome",
        choices=("chrome", "firefox", "opera")
    )
    parser.addoption(
        "--base_url",
        help="base_url"
    )

@pytest.fixture()
def base_url(request):
    return request.config.getoption("--base_url")

@pytest.fixture()
def driver(request):
    global browser
    browser_name = request.config.getoption("--browser")
    service = ChromeService()

    if browser_name == "chrome":
        browser = webdriver.Chrome(service=service)
    elif browser_name == "firefox":
        browser = webdriver.Firefox(service=service)
    elif browser_name == "opera":
        browser = webdriver.Chrome(service=service)

    yield browser

    browser.close()

@pytest.fixture()
def clear_cart(driver, base_url):
    home_page = HomePage(driver)
    home_page.open(base_url)

    home_page.click_cart()
    product_number = home_page.number_of_products_in_cart()
    if product_number > 0:
        for product in range(product_number):
            driver.find_element(By.CSS_SELECTOR, '[title="Remove"]')

    yield

    home_page.click_cart()
    product_number = home_page.number_of_products_in_cart()
    if product_number > 0:
        for product in range(product_number):
            driver.find_element(By.CSS_SELECTOR, '[title="Remove"]')

@pytest.fixture()
def random_email():
    random_email = ''
    for x in range(10):
        random_email += ''.join(random.choice(string.ascii_lowercase))
    yield random_email + "@gmail.com"


# coding: utf-8
# data exploration and cleaning

import pandas as pd
import numpy as np
import seaborn as sns
import streamlit as st
import matplotlib.pyplot as plt
from datetime import datetime
import os

st.title("Downtime machine analysis")
st.markdown(
    "Data exploration app to gain insight into the frequency, duration, and distribution of the downtime events"
)

# st.date_input("Date", datetime.now())
# st.header('Built with Streamlit')
# st.subheader('App')
# uploaded_file = st.file_uploader("Choose a CSV file", type="csv")
# if uploaded_file is not None:
#      data = pd.read_csv(uploaded_file)
#      st.write(data)

DIR_NAME = os.path.dirname(__file__)
DATA_DIR = os.path.join(DIR_NAME, "input_data/")

FILENAME = [f for f in os.listdir(DATA_DIR) if f.endswith(".csv")]

MISSING_VALUES = ["xx", "nan", "0", "???"]
COLNAMES = [
    "equip",
    "equiptype",
    "dept",
    "area",
    "date",
    "shifts",
    "reason",
    "min",
    "prodfix",
    "enginfix",
    "scheduled",
    "prodfam",
    "prodtype",
    "comment",
    "downtimetype",
]
DATE_COLUMN = "date"

# use
# for name in glob.glob('dir/file?.txt'):
# instead of filename[0]

@st.cache
def load_data(nrows=None):
    """
    Load data and do predefined type of cleaning

    Args:
        nrows (int): Limit a number of rows to be read in. Can be useful for large datasets. Defaults to None.

    Returns:
        dataframe: Clean data
    """
    data = pd.read_csv(
        os.path.join(DATA_DIR, FILENAME[0]), nrows=nrows, na_values=MISSING_VALUES
    )
    data.columns = COLNAMES
    # lowercase = lambda x: str(x).lower()
    # data.rename(lowercase, axis='columns', inplace=True)
    data.index = pd.to_datetime(data[DATE_COLUMN])
    data["equip"] = data["equip"].str.replace(r'[\s_" "-]', "").str.lower()
    data["equiptype"] = data["equiptype"].str.replace(r"[\s_]", "-").str.lower()
    data["dept"] = data["dept"].str.replace(" ", "").str.lower()
    data["area"] = data["area"].str.replace(r'[\s|_|" "]', "").str.lower()
    data["shifts"] = data["shifts"].str.lower().str.replace(" ", "")
    di = {
        "evening": "eve",
        "evenings": "eve",
        "wkend": "wknd",
        "days": "day",
        "weekend": "wknd",
        "o/t": "overtime",
    }
    data = data.replace({"shifts": di})
    # replace all other values to NaN
    allowed_values = list(set(di.values()))
    data.loc[~data["shifts"].isin(allowed_values), "shifts"] = "NaN"
    # data['area'] = data['area'].str.replace(r'cell2' 'cell-2')
    data["prodtype"] = data["prodtype"].str.replace(r"[\s_]", "-").str.lower()
    data["prodfam"] = data["prodfam"].str.replace(r"[\s_]", "-").str.lower()
    data["reason"] = data["reason"].str.replace(" ", "")
    return data

data = load_data()

st.markdown("Loading data...done!")  # include progress bar

st.markdown("Average monthly downtime, min")
st.area_chart(data.resample("M").mean())
st.text("Total number of entries: %s" % len(data))

# select columns to subset
ALLCOLS = data.columns.tolist()
DEFAULTCOLS = [
    "equip",
    "equiptype",
    "dept",
    "area",
    "shifts",
    "scheduled",
    "reason",
    "min",
]
selectedcols = st.multiselect(
    "Select columns of interest", options=ALLCOLS, default=DEFAULTCOLS
)
data_cols = data[selectedcols]

# select a year
YEAR = data_cols.index.year
year = st.sidebar.selectbox("Select year", tuple(range(YEAR.min(), YEAR.max() + 1, 1)))
data_year = data_cols[str(year)]

# select a dept
DEPT = data_year.dept.unique()
dept = st.sidebar.selectbox("Select department", DEPT)
data_dept = data_year[data_year.dept == str(dept)]

# select a dept
SCHED = data_dept.scheduled.unique()
sched = st.sidebar.selectbox("Select type", SCHED)
data_sched = data_dept[data_dept.scheduled == str(sched)]

# display data
st.dataframe(data_sched)
st.text("Number of rows: %s" % len(data_sched))

selectedcols = [s for s in selectedcols if s not in ("scheduled", "min", "dept")]
selectedcol = st.selectbox("Select a column to display", selectedcols)
st.bar_chart(data_sched.groupby(selectedcol)["min"].sum().sort_values())
st.text("Shown cumulative downtime for column: %s" % (selectedcol))
st.text("Number of %ss: %s" % (selectedcol, len(data_sched.groupby(selectedcol))))

MAX = data_sched["min"].max()
MINMIN, MAXMIN = st.sidebar.slider(
    "Select range for %s, min" % selectedcol, 0, int(MAX), [0, int(MAX)], 10
)

st.sidebar.text("Distribution of downtime, min ")
data_sched["min"][(data_sched["min"] > MINMIN) & (data_sched["min"] < MAXMIN)].hist(
    grid=False, bins=25, color="r"
)
st.sidebar.pyplot()

NEW_DATA_DIR = "output_data"
if not os.path.exists(NEW_DATA_DIR):
    os.makedirs(NEW_DATA_DIR)

if st.button("Save clean data"):
    NEW_FILENAME = "clean_" + FILENAME[0]
    data.to_csv(os.path.join(NEW_DATA_DIR, NEW_FILENAME))
    st.text("Data has been sucessfully saved")

# Buttons
if st.button("About us"):
    st.text("Data exploration app build with https://www.streamlit.io/")


import yfinance as yf
import streamlit as st
import pandas as  pd

st.write("""
    ## Simple Stock Price App
    Shown are the stock **closing price** and ***volume*** of Google!

""")

tickerSymbol = 'GOOGL'
tickerData = yf.Ticker(tickerSymbol)

tickerDf = tickerData.history(period= 'id', start='2010-5-31', end='2023-5-31')
# Open High Low Close Volume Dividens Stock price
st.write( """
Close Price """)
st.line_chart(tickerDf.Close)

st.write(""" Volume Price """)
st.line_chart(tickerDf.Volume)


import logging

import numpy as np

import pytest

from sherpa.utils.testing import requires_data, requires_fits, \
    requires_group
from sherpa.astro import ui

logger = logging.getLogger('sherpa')

# Single PHA file with a scalar backscal value"""

@pytest.fixture
def single_scalar_setup(make_data_path):

    ui.set_stat('wstat')

    infile = make_data_path('3c273.pi')
    ui.load_pha(1, infile)

    ui.set_source(1, ui.powlaw1d.pl)

    # The powerlaw slope and normalization are
    # intended to be "a reasonable approximation"
    # to the data, just to make sure that any statistic
    # calculation doesn't blow-up too much.
    #
    ui.set_par("pl.gamma", 1.782)
    ui.set_par("pl.ampl", 1.622e-4)

def filter_data():
    """Filter the data.

    Use a slightly-complex filter - e.g. not just the
    end points - to exercise the system.
    """

    ui.ignore(None, 0.5)
    ui.ignore(3, 4)
    ui.ignore(7, None)

def check_stat(nbins, expected, idval=None):

    # check the filter sizes (mainly so that these tests
    # get flagged as in need of a look if anything changes
    # in other parts of the code, such as filtering and binning
    #
    assert ui.get_data(idval).get_dep(True).size == nbins

    stat = ui.calc_stat(idval)

    # this used to check to a given number of decimal places
    # rather than relative
    assert stat == pytest.approx(expected)

def check_stat2(expected):
    """Is calc_stat() == expected?"""

    stat = ui.calc_stat()

    # this used to check to a given number of decimal places
    # rather than relative
    assert stat == pytest.approx(expected)

@requires_data
@requires_fits
def test_wstat_single_scalar_grouped_all(clean_astro_ui, hide_logging, single_scalar_setup):

    # Used git commit 770359b5004374b969ebb63c173f293419397b4c
    # to create the oracle value, on a linux 64-bit machine.
    check_stat(46, 62.12275005769554)

@requires_data
@requires_fits
def test_wstat_single_scalar_grouped_filtered(clean_astro_ui, hide_logging, single_scalar_setup):
    filter_data()

    # Used git commit 770359b5004374b969ebb63c173f293419397b4c
    # to create the oracle value, on a linux 64-bit machine.
    check_stat(35, 39.84715083397662)

@requires_data
@requires_fits
def test_wstat_single_scalar_ungrouped_all(clean_astro_ui, hide_logging, single_scalar_setup):
    ui.ungroup()

    # Used git commit 770359b5004374b969ebb63c173f293419397b4c
    # to create the oracle value, on a linux 64-bit machine.
    check_stat(1024, 652.9968167212116)

@requires_data
@requires_fits
def test_wstat_single_scalar_ungrouped_filtered(clean_astro_ui, hide_logging, single_scalar_setup):
    ui.ungroup()
    filter_data()

    # Used git commit 770359b5004374b969ebb63c173f293419397b4c
    # to create the oracle value, on a linux 64-bit machine.
    check_stat(375, 416.0601496345599)

# Two PHA files with a scalar backscal value

@pytest.fixture
def two_scalar_setup(make_data_path):

    ui.set_stat('wstat')

    infile1 = make_data_path('3c273.pi')
    infile2 = make_data_path('9774.pi')
    ui.load_pha(1, infile1)
    ui.load_pha(2, infile2)

    # Since 9774.pi isn't grouped, group it. Note that this
    # call groups the background to 20 counts per bin. In this
    # case we do not want that; instead we want to use the same
    # grouping scheme as the source file.
    #
    # Note: this is related to issue 227
    #
    ui.group_counts(2, 20)
    ui.set_grouping(2, bkg_id=1, val=ui.get_grouping(2))

    # There's no need to have the same model in both datasets,
    # but assume the same source model can be used, with a
    # normalization difference.
    #
    ui.set_source(1, ui.powlaw1d.pl1)
    ui.set_source(2, ui.const1d.c2 * ui.get_source(1))

    # The powerlaw slope and normalization are
    # intended to be "a reasonable approximation"
    # to the data, just to make sure that any statistic
    # calculation doesn't blow-up too much.
    #
    # Note: the model values for 3c273 are slighly different
    #       to the single-PHA-file case, so stat results are
    #       slightly different
    #
    ui.set_par("pl1.gamma", 1.7)
    ui.set_par("pl1.ampl", 1.6e-4)
    ui.set_par("c2.c0", 45)

@requires_group
@requires_data
@requires_fits
def test_wstat_two_scalar_grouped_all(clean_astro_ui, hide_logging, two_scalar_setup):

    # Used git commit 770359b5004374b969ebb63c173f293419397b4c
    # to create the oracle value, on a linux 64-bit machine.
    exp1 = 66.20114213871206
    exp2 = 401.75572944361613
    check_stat(46, exp1, 1)
    check_stat(148, exp2, 2)
    check_stat2(exp1 + exp2)

@requires_group
@requires_data
@requires_fits
def test_wstat_two_scalar_grouped_filtered(clean_astro_ui, hide_logging, two_scalar_setup):
    filter_data()

    # Used git commit 770359b5004374b969ebb63c173f293419397b4c
    # to create the oracle value, on a linux 64-bit machine.
    exp1 = 45.93447168433694
    exp2 = 127.35556915677182
    check_stat(35, exp1, 1)
    check_stat(120, exp2, 2)
    check_stat2(exp1 + exp2)

@requires_group
@requires_data
@requires_fits
def test_wstat_two_scalar_ungrouped_all(clean_astro_ui, hide_logging, two_scalar_setup):
    ui.ungroup(1)
    ui.ungroup(2)

    # Used git commit 770359b5004374b969ebb63c173f293419397b4c
    # to create the oracle value, on a linux 64-bit machine.
    exp1 = 657.2275371837611
    exp2 = 880.8442022201893
    check_stat(1024, exp1, 1)
    check_stat(1024, exp2, 2)
    check_stat2(exp1 + exp2)

@requires_group
@requires_data
@requires_fits
def test_wstat_two_scalar_ungrouped_filtered(clean_astro_ui, hide_logging, two_scalar_setup):
    ui.ungroup(1)
    ui.ungroup(2)
    filter_data()

    # Used git commit 770359b5004374b969ebb63c173f293419397b4c
    # to create the oracle value, on a linux 64-bit machine.
    exp1 = 420.50951039235736
    exp2 = 397.4089421041855
    check_stat(375, exp1, 1)
    check_stat(375, exp2, 2)
    check_stat2(exp1 + exp2)

# PHA file that has been grouped.
#
# This is the error reported in issue #227, whereby the
# grouping in the source and background files is different,
# which causes an error. It is not clear yet whether this
# should be supported or not, but a regression test is added
# to check this behavior.
#
# The settings are the same as the test_wstat_two_scalar
# case, for the 9774.pi file. It would be easier to move this
# into test_wstat_two_scalar (since it avoids having multiple
# copies of the settings, and lets the wstat be tested on
# > 2 data sets), but for now keep as a separate set of tests
# to make sure it is obvious what is and isn't failing.

@pytest.fixture
def group_setup(make_data_path):

    ui.set_stat('wstat')

    infile = make_data_path('9774.pi')
    ui.load_pha(1, infile)

    ui.group_counts(1, 20)

    # Unlike the test_wstat_two_scalar case, the grouping
    # is not copied over.
    # ui.set_grouping(1, bkg_id=1, val=ui.get_grouping(1))

    ui.set_source(1, ui.const1d.c1 * ui.powlaw1d.pl1)

    # These should be the same as test_wstat_two_scalar
    #
    ui.set_par("pl1.gamma", 1.7)
    ui.set_par("pl1.ampl", 1.6e-4)
    ui.set_par("c1.c0", 45)

@requires_group
@requires_data
@requires_fits
def test_wstat_group_grouped_all(clean_astro_ui, hide_logging, group_setup):

    # Used git commit 770359b5004374b969ebb63c173f293419397b4c
    # to create the oracle value, on a linux 64-bit machine.
    expval = 401.75572944361613
    check_stat(148, expval, 1)

@requires_group
@requires_data
@requires_fits
def test_wstat_group_grouped_filtered(clean_astro_ui, hide_logging, group_setup):
    filter_data()

    # Used git commit 770359b5004374b969ebb63c173f293419397b4c
    # to create the oracle value, on a linux 64-bit machine.
    expval = 127.35556915677182
    check_stat(120, expval, 1)

@requires_group
@requires_data
@requires_fits
def test_wstat_group_ungrouped_all(clean_astro_ui, hide_logging, group_setup):
    ui.ungroup(1)

    # Used git commit 770359b5004374b969ebb63c173f293419397b4c
    # to create the oracle value, on a linux 64-bit machine.
    expval = 880.8442022201893
    check_stat(1024, expval, 1)

@requires_group
@requires_data
@requires_fits
def test_wstat_group_ungrouped_filtered(clean_astro_ui, hide_logging, group_setup):
    ui.ungroup(1)
    filter_data()

    # Used git commit 770359b5004374b969ebb63c173f293419397b4c
    # to create the oracle value, on a linux 64-bit machine.
    expval = 397.4089421041855
    check_stat(375, expval, 1)

# Single PHA file with an array of backscal values.
#
# This really should use a grating PHA dataset, but it's not
# obvious we have one (along with the necessary responses) in
# the sherpa-test-data/ repository. So for now "hack" in
# one. The statistic values were calculated by changing the
# backscal by 0.9 but leaving it as a scalar. As the tests
# currently fail, they have not been validated.

@pytest.fixture
def single_array_setup(make_data_path):

    ui.set_stat('wstat')

    infile = make_data_path('3c273.pi')
    ui.load_pha(1, infile)

    # Change the backscale value slightly so that the
    # results are different to other runs with this file.
    #
    nbins = ui.get_data(1).get_dep(False).size
    bscal = 0.9 * np.ones(nbins) * ui.get_backscal(1)
    ui.set_backscal(1, backscale=bscal)

    ui.set_source(1, ui.powlaw1d.pl)

    # The powerlaw slope and normalization are
    # intended to be "a reasonable approximation"
    # to the data, just to make sure that any statistic
    # calculation doesn't blow-up too much.
    #
    ui.set_par("pl.gamma", 1.7)
    ui.set_par("pl.ampl", 1.7e-4)

@requires_data
@requires_fits
def test_wstat_single_array_grouped_all(clean_astro_ui, hide_logging, single_array_setup):

    # Used git commit 770359b5004374b969ebb63c173f293419397b4c
    # to create the oracle value, on a linux 64-bit machine.
    check_stat(46, 71.21845954979574)

@requires_data
@requires_fits
def test_wstat_single_array_grouped_filtered(clean_astro_ui, hide_logging, single_array_setup):
    filter_data()

    # Used git commit 770359b5004374b969ebb63c173f293419397b4c
    # to create the oracle value, on a linux 64-bit machine.
    check_stat(35, 45.6311990089982)

@requires_data
@requires_fits
def test_wstat_single_array_ungrouped_all(clean_astro_ui, hide_logging, single_array_setup):
    ui.ungroup()

    # Used git commit 770359b5004374b969ebb63c173f293419397b4c
    # to create the oracle value, on a linux 64-bit machine.
    check_stat(1024, 663.0160968458746)

@requires_data
@requires_fits
def test_wstat_single_array_ungrouped_filtered(clean_astro_ui, hide_logging, single_array_setup):
    ui.ungroup()
    filter_data()

    # Used git commit 770359b5004374b969ebb63c173f293419397b4c
    # to create the oracle value, on a linux 64-bit machine.
    check_stat(375, 420.8390856766203)


import functools
import math
import time

import pandas as pd

def get_scores_dict():
    start_time = time.time()
    ranks = pd.read_csv("ranks.zip", compression="zip")
    end_time = time.time()
    print(f"took {end_time - start_time} sec to read dataframe with hands scores")

    start_time = time.time()
    ranks_score_dict = {}
    for row in ranks.itertuples(index=True, name="Pandas"):
        ranks_score_dict[row.hands] = row.value
    end_time = time.time()
    print(f"took {end_time - start_time} to create dict with hands scores")

    return ranks_score_dict

HANDS_RANKS_SCORE_DICT = None

def get_hand_score(hand, do_floor):
    """
    Scores a given hand. Implements a singleton on variable HANDS_RANKS_SCORE_DICT
    so that it is loaded into memory only once.
    :param hand: a hand.
    :param do_floor: floors card scores.
    :return:  the score of the hand.
    """
    global HANDS_RANKS_SCORE_DICT
    if HANDS_RANKS_SCORE_DICT is None:
        HANDS_RANKS_SCORE_DICT = get_scores_dict()
    return (
        HANDS_RANKS_SCORE_DICT[str(hand).replace(",", "")]
        if not do_floor
        else math.floor(HANDS_RANKS_SCORE_DICT[str(hand).replace(",", "")])
    )

def left_hand_points(
    left_hand,
    right_hand,
    do_floor,
):
    if get_hand_score(left_hand, do_floor) > get_hand_score(right_hand, do_floor):
        return 1
    elif get_hand_score(left_hand, do_floor) < get_hand_score(right_hand, do_floor):
        return -1
    else:
        return 0

def compare_cards(x, y, suit_to_number={"H": 0, "S": 1, "C": 2, "D": 3}):
    suit_x, rank_x = x[0], int(x[1:])
    suit_y, rank_y = y[0], int(y[1:])
    assert 2 <= rank_x <= 14 and 2 <= rank_y <= 14
    assert suit_x in suit_to_number.keys() and suit_y in suit_to_number.keys()
    if rank_x > rank_y:
        return 1
    elif rank_x < rank_y:
        return -1
    else:
        return suit_to_number[suit_x] - suit_to_number[suit_y]

def append_cards(initial_hand, new_cards):
    new_hand = initial_hand + new_cards
    new_hand.sort(key=functools.cmp_to_key(compare_cards))
    return new_hand


#Kalman Filter
import cv2
import numpy as np
import math

collision_coordinates = [[2,3], [1,7], []]

def lat_to_cart(lat, long):
    x = long * 60 * 1852 * math.cos(lat)
    y = lat * 60 * 1852
    x = x * 0.175
    y = y * 0.175
    print(x,y)

#method of
def coordinate_mapping():
    img = cv2.imread('test.jpg', 0)
    edges = cv2.Canny(img, 100, 255)

    indices = np.where(edges != [0])
    coordinates = zip(indices[0], indices[1])

    xcoord = indices[0]
    ycoords = indices[1]

    #print(indices)
    return xcoord, ycoords


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Nov 13 19:32:15 2021

@author: ron
"""

from .base import AbstractTechnicalIndicator
from collections import deque
import numpy as np

class SMA(AbstractTechnicalIndicator):

    def __init__(self, period):
        """
        Parameters
        ----------
        ready: to indicate whether data is sufficent to calucalte the result
        """
        self.data = deque(maxlen=period)
        self.data_store = []
        self.lastest_val = 0
        self.period = period
        self.ready = False

    def update(self, price):
        self.data.append(price)
        #data is sufficient
        if not self.ready and len(self.data) == self.period:
            self.ready = True

        if self.ready:
            self.lastest_val = np.mean(self.data)
            self.data_store.append(self.lastest_val)

    def __bool__(self):
        return self.ready


import sys
import os
import json

import numpy as np
import tensorflow as tf
from tensorflow.contrib.layers import xavier_initializer
from tqdm import tqdm

def load_params_dict(filename):
    with open(filename, 'r') as file:
        params_dict = json.load(file)
    return params_dict

class BaseModel(object):
    """ Code from mem2nn-tensorflow. """
    def __init__(self, vocab_size, params, *args):
        self.vocab_size = vocab_size
        ## dirs ##
        self.save_dir = params.save_dir
        self.load_dir = params.load_dir

        ## set params ##
        self.action = params.action
        self.params = params;
        self.num_steps = params.lm_num_steps
        ## build graph ##
        self.graph = tf.Graph()
        gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=self.params.gpu_fraction)
        self.sess = tf.Session(graph=self.graph, config=tf.ConfigProto(gpu_options=gpu_options))
        with self.graph.as_default():
            default_init = xavier_initializer()
            with tf.variable_scope('Language_model', initializer=default_init):
                print("Building Language Model...")
                self.global_step = tf.Variable(0, name='global_step', trainable=False)
                self.min_validation_loss = tf.Variable(np.inf, name='validation_loss', trainable=False)
                self.new_validation_loss = tf.placeholder('float32', name='new_validation_loss');
                self.assign_min_validation_loss = self.min_validation_loss.assign(self.new_validation_loss).op
                if self.action == 'train':
                    self.build(eval_flag=False)
                elif self.action == 'test':
                    self.build(eval_flag=True)
                self.merged = tf.summary.merge_all()
                self.init_op = tf.global_variables_initializer()

        ## init saver ##
        with self.graph.as_default():
            self.saver = tf.train.Saver()

        ## init variables ##
        if not self.load_dir == '':
            print("Loading model ...")
            self.load()
        else:
            summary_dir = os.path.join(self.save_dir, "summary")
            if tf.gfile.Exists(summary_dir):
                tf.gfile.DeleteRecursively(summary_dir)
            print("Init model ...")
            self.sess.run(self.init_op)

        ## init saver ##
        if self.action == 'train':
            summary_dir = os.path.join(self.save_dir, "summary")
            self.summary_writer = tf.summary.FileWriter(logdir=summary_dir, graph=self.sess.graph)

        ## train & eval run output ##

    def __del__(self):
        if hasattr(self, "sess"):
            self.sess.close()

    def build(self):
        raise NotImplementedError()

    def get_feed_dict(self, batch, is_train):
        raise NotImplementedError()

    def save_params(self):
        raise NotImplementedError()

    def train_batch(self, feed_dict):
        train_list = [self.merged, self.opt_op, self.global_step, self.final_state, self.loss]
        return self.sess.run(train_list, feed_dict=feed_dict)

    def test_batch(self, feed_dict):
        eval_list = [self.loss, self.global_step, self.final_state, self.log_perp]
        return self.sess.run(eval_list, feed_dict=feed_dict)

    def pre_train(self, train_data, val_data):
        assert self.action == 'train'
        params = self.params
        num_epochs = params.lm_num_epoch

        min_loss = self.sess.run(self.min_validation_loss)
        print("Training %d epochs ..." % num_epochs)
        try:
            for epoch_no in range(num_epochs):
                ## training process ##
                ## fetch initial state ##
                iters = 0
                losses = 0.0
                for step, (x, y) in enumerate(train_data.get_batch()):
                    if step == 0:
                        init_feed_dict = {self.x: x}
                        state = self.sess.run(self.initial_state, feed_dict=init_feed_dict)
                    feed_dict = self.get_feed_dict((x, y), is_train=True, state=state)
                    summary, _, global_step, final_state, cur_loss = self.train_batch(feed_dict)
                    losses += cur_loss
                    iters += self.num_steps
                    if step % (train_data.epoch_size // 10) == 10:
                        print('[%d] perp: %f' % (global_step, np.exp(losses / iters)))
                    state = final_state
                self.summary_writer.add_summary(summary, global_step)
                ## evaluation process, including train and val ##
                if (epoch_no + 1) % params.lm_period == 0:
                    self.eval(train_data, name='Training')

                if (epoch_no + 1) % params.lm_val_period == 0:
                    loss = np.inf
                    if val_data:
                        loss , _, _ = self.eval(val_data, name='Validation')
                    if loss <= min_loss:
                        self.sess.run(self.assign_min_validation_loss, {self.new_validation_loss: loss})
                        min_loss = loss
                        self.save()
            print("Training completed.")

        except KeyboardInterrupt:
            loss = np.inf
            if val_data:
                loss, _, _ = self.eval(val_data, name='Validation')
            if loss <= min_loss:
                self.sess.run(self.assign_min_validation_loss, {self.new_validation_loss: loss})
                min_loss = loss
                self.save()
            print("Stop the training by console!")

    def eval(self, data, name):
        losses = 0.0
        iters = 0
        log_perps = []
        for step, (x, y) in enumerate(data.get_batch()):
            if step == 0:
                init_feed_dict = {self.x: x}
                state = self.sess.run(self.initial_state, feed_dict=init_feed_dict)
            feed_dict = self.get_feed_dict((x, y), is_train=False, state=state)
            batch_loss, global_step, state, log_perp = self.test_batch(feed_dict)
            losses += batch_loss
            iters += self.num_steps
            log_perps.append(log_perp)
        perp = np.exp(losses / iters)
        #print(np.exp(np.sum(np.array(log_perps).transpose(), axis = 1) / iters))
        log_perps = np.array(log_perps).transpose()
        #print("[%s] step %d, Perp = %.4f" % \
        #      (name, global_step, perp))
        return perp, log_perps, iters

    def save(self):
        assert self.action == 'train'
        print("[lm] Saving model to dir %s" % self.save_dir)
        self.saver.save(self.sess, os.path.join(self.save_dir, 'run'), self.global_step)

    def load(self):
        checkpoint = tf.train.get_checkpoint_state(self.load_dir)
        if checkpoint is None:
            print("Error: No saved model found. Please train first.")
            sys.exit(0)
        self.saver.restore(self.sess, checkpoint.model_checkpoint_path)


#!/usr/bin/env python3

"""Parse and save tables in a web page into tab-separated-values (tsv) files.

Takes input from stdin.

Writes files in the current directory based on "id" attribute of each table.

NOTE: to fit into tsv, cell values \t \n \r are made into spaces instead.
To exactly preserve values, we'll have to use csv.

Based on https://srome.github.io/Parsing-HTML-Tables-in-Python-with-BeautifulSoup-and-pandas/

Requires beautifulsoup4==4.9.1.
"""

import argparse
import logging
import sys
from bs4 import BeautifulSoup

def parse_text(txt, leave_html):
    """Parse all tables in the given HTML text"""
    soup = BeautifulSoup(txt, 'html.parser')
    idx = 1
    for table in soup.find_all('table'):
        _parse_html_table(table, f'table{idx:03d}', leave_html)
        idx += 1

def _clean_text(txt):
    """Make \t, \n, \r into spaces"""
    return txt.replace("\n", " ").replace("\t", " ").replace("\r", " ")

def _get_text(elem, leave_html):
    """Return elem.get_text() if leave_html is False, else str(elem)

    elem should be a td or th element.
    """
    assert elem.name in (u'td', u'th')
    val = None
    if leave_html:
        # Take everything, but not including the outer td or th tag
        val = ''.join([str(elem1) for elem1 in elem.contents])
    else:
        # only the text
        val = elem.get_text()
    return val

def _parse_html_table(table, backup_id, leave_html):
    """Parse tables out of the beautifulsoup table and write them into files."""
    n_columns = 0
    column_names = []

    table_id = table.get('id', backup_id)
    # logging.info(f"*** table {table_id}")
    # Find number of rows and columns
    # we also find the column titles if we can
    for row in table.find_all('tr'):
        # Determine the number of rows in the table
        td_tags = row.find_all('td')
        if len(td_tags) > 0:
            if n_columns == 0:
                # Set the number of columns for our table
                # TODO(dan): Handle colspan > 1
                n_columns = len(td_tags)

        # Handle column names if we find them
        th_tags = row.find_all('th')
        if len(th_tags) > 0 and len(column_names) == 0:
            assert len(column_names) == 0, "found more column names"
            for th in th_tags:
                column_names.append(_get_text(th, leave_html))

    # TODO(dan): Handle colspan > 1
    # Safeguard on Column Titles
    # if len(column_names) > 0 and len(column_names) != n_columns:
    #    raise Exception("Column titles do not match the number of columns")

    # TODO(dan): Handle table without id attribute
    filename = f"{table_id}.tsv"
    logging.info(f"Write {filename}..")
    with open(filename, 'w') as the_tsv:
        if not column_names:
            # there were no column names, so print a fake header col1, col2, ..
            print('\t'.join([f"col{col}" for col in range(n_columns)]),
                  file=the_tsv)
        for row in table.find_all('tr'):
            # always look for td or th because sometimes people use either in
            # a row, possibly for formatting reasons
            # this has the side effect of printing any existing table headers
            columns = row.find_all(['td', 'th'])
            col_texts = [_get_text(col, leave_html) for col in columns]
            if col_texts:
                print('\t'.join([_clean_text(col) for col in col_texts]),
                      file=the_tsv)

def main():
    parser = argparse.ArgumentParser(
        description='Convert HTML tables to tab-separated values files')
    parser.add_argument('--leave-html',
                        action='store_true',
                        default=False,
                        help='Leave the HTML in each cell')
    args = parser.parse_args()

    logging.basicConfig(format='%(name)s: %(message)s', level=logging.INFO)
    parse_text(sys.stdin.read(), args.leave_html)

main()


import pygame
from stagehandler import Stagehandler
from level import Level
from gameloop.clock import Clock
from gameloop.eventqueue import EventQueue
from gameloop.renderer import Renderer
from gameloop.gameloop import GameLoop

class Game: # pylint: disable=too-many-instance-attributes
    """Class for the game.
    Attributes:
        stagehandler: Handles reading stages.json file and current number of stage and returns necessary information
        single_stage: Bool that is True if stage was specifically selected in the UI
        level_map: Array that contains the map of the level
        cellsize: Integer that tells how wide (in px) a single square on the map is
        pickup_amount: Integer that tells how many pickups are to be collected in a stage
        enemy_info: Dictionary telling propetries of enemies in the stage
        height: Integer that tells the height of the stage in squares
        width: Integer that tells the width of the stage in squares
        display_height: Integer that tells the height of the window in pixels
        display_width: Integer that tells the width of the window in pixels
        display: Sets the display width and height
    """
    def __init__(self, stage=0, single_stage=False): # pylint: disable=too-many-statements
        """Constructor for the class
        Args:
            stage: Specifies the stagenumber if a stage was specifically selected in the UI
            single_stage: Bool that is True if stage was specifically selected in the UI
        """
        self.stagehandler = Stagehandler()
        self.stagehandler.current_stage = stage
        self.single_stage = single_stage
        self.level_map = self.stagehandler.get_stagemap()
        self.cellsize = self.stagehandler.get_stage_cellsize()
        self.pickup_amount = self.stagehandler.get_stage_pickup_amount()
        self.enemy_info = self.stagehandler.get_stage_enemy_info()
        self.height = len(self.level_map)
        self.width = len(self.level_map[0])
        self.display_height = self.height * self.cellsize
        self.display_width = self.width * self.cellsize
        self.display = pygame.display.set_mode((self.display_width, self.display_height))

        pygame.display.set_caption("World's Hardest Game")

        level = Level(self.level_map, self.cellsize, self.pickup_amount, self.enemy_info)
        eventqueue = EventQueue()
        clock = Clock()
        renderer = Renderer(self.display, level)
        gameloop = GameLoop(level, renderer, eventqueue, clock, self.cellsize)

        pygame.init()

        if gameloop.start() is False:
            self.__init__(stage, True)

        if self.single_stage or gameloop.quit:
            pygame.quit()
        else:
            self.next_level()

    def next_level(self):
        """Changes the stage to the next one unless the final stage has been completed
        """
        if self.stagehandler.next_stage():
            self.game_complete()
        else:
            self.__init__(self.stagehandler.current_stage)

    def game_complete(self):
        """Exits the game keeping the UI running
        """
        pygame.quit()


from typing import List

class Solution:
    def maximumProduct2(self, nums: List[int]) -> int:

        len_ = len(nums)
        prod = 1
        if(len_ > 3):
            numsSorted = sorted(nums, reverse=True)
            print(numsSorted)
            for i in range(3):
                prod = prod * numsSorted[i]
        else:
            for i in range(len_):
                prod = prod * nums[i]

        return prod

    def maximumProduct(self, nums:List[int]) -> int:

        numSorted = sorted(nums, reverse=True)
        len_ = len(nums)
        if(len == 3):
                return numSorted[0] * numSorted[1] * numSorted[2]
        else:
            return max((numSorted[0] * numSorted[1] * numSorted[2]), (numSorted[0]*numSorted[len_ -1]*numSorted[len_ -2]))

# Time Complexity -> O(n log n) - to sort the nums array
# Space Complexity -> O(log n) -> to sort the nums array

s = Solution()
nums = [-4,-3,-2,50,60]
prod = s.maximumProduct(nums)
print(prod)


from lims.utils import tmpID
from dependencies.dependency import _createObjectByType
from magnitude import mg

def compare_containers(a, b):
    a_capacity = a.getCapacity().lower().split(" ", 1)
    b_capacity = b.getCapacity().lower().split(" ", 1)
    a_magnitude = mg(float(a_capacity[0]), a_capacity[1])
    b_magnitude = mg(float(b_capacity[0]), b_capacity[1])
    return cmp(
        a.getCapacity() and a_magnitude or mg(0, 'ml'),
        b.getCapacity() and b_magnitude or mg(0, 'ml')
    )

def set_container_preservation(context, container, data):
    # If container is pre-preserved, set the partition's preservation,
    # and flag the partition to be transitioned below.
    if container:
        if type(container) in (list, tuple):
            container = container[0]
        proxies = context.bika_setup_catalog(UID=container)
        container = [_p.getObject() for _p in proxies]
        container = container[0] if container else None
        if container:
            prepreserved = container.getPrePreserved()
            preservation = container.getPreservation()
            data['prepreserved'] = prepreserved
            if prepreserved and preservation:
                return preservation.UID()
    return data.get('preservation_uid', '')

def create_samplepartition(context, data, analyses=[]):
    partition = _createObjectByType('SamplePartition', context, data['part_id'])
    partition.unmarkCreationFlag()
    # Determine if the sampling workflow is enabled
    workflow_enabled = context.bika_setup.getSamplingWorkflowEnabled()
    # Sort containers and select smallest
    container = data.get('container_uid', None)
    if container:
        containers = []
        if type(container[0]) is str:
            # UIDs
            containers = context.bika_setup_catalog(UID=container)
            containers = [_p.getObject() for _p in containers]
        elif hasattr(container[0], 'getObject'):
            # Brains
            containers = [_p.getObject() for _p in container]
        elif hasattr(container[0], 'portal_type'):
            containers = [c for c in container]

        if containers:
            try: containers.sort(lambda a, b: compare_containers(a, b))
            except: pass
            container = containers[0]
    # Set the container and preservation
    preservation = set_container_preservation(context, container, data)
    # Add analyses
    partition_services = data['services']
    analyses = [a for a in analyses if a.getServiceUID() in partition_services]
    if analyses:
        partition.edit(Analyses=analyses)
    # Set some generated values
    partition.edit(
        Container=container,
        Preservation=preservation,
    )
    # Attach partition to analyses
    if analyses:
        for analysis in analyses:
            analysis.setSamplePartition(partition)
    # Perform the appropriate workflow action
    workflow_action =  'sampling_workflow' if workflow_enabled \
        else 'no_sampling_workflow'
    context.portal_workflow.doActionFor(partition, workflow_action)
    # Return the created partition
    return partition


import lxml.html
import urllib.request as urllib2
import pprint
import http.cookiejar as cookielib
from io import BytesIO
import lxml.html
from PIL import Image, ImageFilter, ImageOps, ImageChops
import pytesseract
import requests
#import base64

__author__ = "Dzhud"

def parse_form(html):
    tree = lxml.html.fromstring(html)
    data = {}
    for e in tree.cssselect('form input'):
        if e.get('name'):
            data[e.get('name')] = e.get('value')
    return data

REGISTER_URL = 'http://tracuunnt.gdt.gov.vn/tcnnt/mstdn.jsp'
ty = cookielib.CookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(ty))
html = opener.open(REGISTER_URL).read()
form = parse_form(html)
#Run this below to check if all is right. Should return the fields
#from the webpage including `'captcha': None,`
#pprint.pprint(form)

def get_captcha(html):
    tree = lxml.html.fromstring(html)
    img_data = tree.cssselect('div img')[0].get('src')
    #For base64 images
    #binary_img_data = base64.b64decode(img_data)
    img_link = 'http://tracuunnt.gdt.gov.vn'+ img_data
    response = requests.get(img_link)
    img = Image.open(BytesIO(response.content))
    return img

# Include tesseract executable in your path
pytesseract.pytesseract.tesseract_cmd = r"C:\Program Files\Tesseract-OCR\tesseract.exe"

img = get_captcha(html)
# To see original image
img.save('captcha_initial.png')
img = img.filter(ImageFilter.MinFilter(3))
img = ImageOps.crop(img)
# (1, 0) and (-2, 0) seem to work fine too for offset()
img = ImageChops.offset(img, 0, -1)
img = img.effect_spread(1)
img = img.filter(ImageFilter.SHARPEN)
# To see newly adjusted image
img.save('newImage1.png')

#Convert image to text
wb = pytesseract.image_to_string(img)
print(wb)


from PIL import Image
from fastapi import File, UploadFile, Body

from app.db.images import create_images, get_images_by_id, get_images_all, delete_image_by_id
from app.models.api import ResolutionImagesModel
from app.utils import async_request_file
from app.views import api_router

import shutil

@api_router.get("/images", status_code=200, tags=['api'])
async def get_images():
    return await get_images_all()

@api_router.get("/images/{img_id}", status_code=200, tags=['api'])
async def get_image(img_id: int):
    return await get_images_by_id(img_id)

@api_router.post("/images", status_code=201, tags=['api'])
async def create_file(file: UploadFile = File(None), url: str = Body(None)):
    if file:
        with open(f'files/{file.filename}', 'wb') as buffer:
            shutil.copyfileobj(file.file, buffer)
            im = Image.open(f'files/{file.filename}')
            (width, height) = im.size
            return await create_images(name=file.filename,
                                       url=None,
                                       picture=f'files/{file.filename}',
                                       width=width,
                                       height=height)
    if url:
        filename = await async_request_file(url)
        im = Image.open(f'files/{filename}')
        (width, height) = im.size
        return await create_images(name=filename,
                                   url=url,
                                   picture=f'files/{filename}',
                                   width=width,
                                   height=height)

@api_router.post("/images/{img_id}/resize", status_code=200, tags=['api'])
async def resize_image(img_id: int, res: ResolutionImagesModel):
    file = await get_images_by_id(img_id)
    url = file['url']
    img = Image.open(file['picture'])
    width = res.width
    height = res.height
    resized_img = img.resize((width, height), Image.ANTIALIAS)
    new_file_name = file['name'].split('.')[0]
    file_formate = file['name'].split('.')[-1]
    resized_img.save(f'files/{new_file_name}.{file_formate}_{width}_{height}.{file_formate}')
    return await create_images(name=f'{new_file_name}.{file_formate}_{width}_{height}.{file_formate}',
                               url=url,
                               picture=f'files/{new_file_name}.{file_formate}_{width}_{height}.{file_formate}',
                               width=width,
                               height=height,
                               parent_picture=img_id)

@api_router.delete("/images/{img_id}", status_code=201, tags=['api'])
async def delete_image(img_id: int):
    await delete_image_by_id(img_id)
    return


def list_merge(input_1=None, input_2=None, input_3=None, input_4=None, input_5=None, input_6=None, input_7=None, input_8=None, input_9=None, input_10=None, **kwargs):
    """
    Merge 2-10 different data paths into a single output data path. For example, if IP addresses are stored in the fields sourceAddress, destinationAddress, and deviceAddress, then those three fields could be merged together to form a single list of IP addresses.

    Args:
        input_1 (CEF type: *)
        input_2 (CEF type: *)
        input_3 (CEF type: *)
        input_4 (CEF type: *)
        input_5 (CEF type: *)
        input_6 (CEF type: *)
        input_7 (CEF type: *)
        input_8 (CEF type: *)
        input_9 (CEF type: *)
        input_10 (CEF type: *)

    Returns a JSON-serializable object that implements the configured data paths:
        *.item (CEF type: *): A combined list of all the values from all the input lists
    """
    ############################ Custom Code Goes Below This Line #################################
    import json
    import phantom.rules as phantom

    outputs = []

    # loop through all the inputs and use the index to track which input is being processed
    for index, input_value in enumerate([input_1, input_2, input_3, input_4, input_5, input_6, input_7, input_8, input_9, input_10]):

        # skip the input if no datapath is provided or the datapath does not resolve to anything
        if not input_value:
            phantom.debug("skipping input_{} because it is falsy".format(index+1))
            continue

        # if the input is not a list just append the single item
        if not isinstance(input_value, list):
            outputs.append({"item": input_value})
            phantom.debug("merged 1 items from input_{}".format(index+1))
            continue

        # keep track of how many items were merged from each input
        item_count = 0

        # iterate through the list and append each item in its own dictionary
        for item in input_value:
            if item:
                phantom.debug("input_value is {} and item is {}".format(input_value, item))
                outputs.append({"item": item})
                item_count += 1
        phantom.debug("merged {} items from input_{}".format(item_count, index+1))

    phantom.debug("merged results: {}".format(outputs))

    # Return a JSON-serializable object
    assert json.dumps(outputs)  # Will raise an exception if the :outputs: object is not JSON-serializable
    return outputs