###############################################
# DATA ANALYSIS WITH PYTHON
###############################################
# - NumPy
# - Pandas
# - Data Visualization: Matplotlib & Seaborn
# - Advanced Functional Exploratory Data Analysis EDA

#############################################
# NUMPY
#############################################

# Why Numpy?
# Creating Numpy Arrays
# Attibutes of Numpy Arrays
# Reshaping
# Index Selection
# Slicing
# Fancy Index
# Conditions on Numpy
# Mathematical Operations

#############################################
# Why NumPy?
#############################################
import numpy as np
a = [1, 2, 3, 4]
b = [2, 3, 4, 5]

ab = []

for i in range(0, len(a)):
    ab.append(a[i] * b[i])

a = np.array([1, 2, 3, 4])
b = np.array([2, 3, 4, 5])
a * b

#############################################
# Creating Numpy Arrays
#############################################
import numpy as np

np.array([1, 2, 3, 4, 5])
type(np.array([1, 2, 3, 4, 5]))
np.zeros(10, dtype=int)
np.random.randint(0, 10, size=10)
np.random.normal(10, 4, (3, 4))

#############################################
# Attibutes of Numpy Arrays
#############################################
import numpy as np

# ndim: boyut sayısı
# shape: boyut bilgisi
# size: toplam eleman sayısı
# dtype: array veri tipi

a =np.random.randint(10, size=5)
a.ndim
a.shape
a.size
a.dtype

#############################################
# Reshaping
#############################################
import numpy as np

np.random.randint(1, 10, size=9)
np.random.randint(1, 10, size=9).reshape(3, 3)

ar = np.random.randint(1, 10, size=9)
ar.reshape(3, 3)

#############################################
# Index Selection
#############################################
import numpy as np
a = np.random.randint(10, size=10)
a[0]
a[0:5]
a[0] = 999

m = np.random.randint(10, size=(3, 5))

m[0, 0]
m[1, 1]
m[2, 3]

m[2, 3] = 999

m[2, 3] = 2.9

m[:, 0]
m[1, :]
m[0:2, 0:3]

#############################################
# Fancy Index
#############################################
import numpy as np

v = np.arange(0, 30, 3)
v[1]
v[4]

catch = [1, 2, 3]

v[catch]

#############################################
# Conditions on Numpy
#############################################
import numpy as np
v = np.array([1, 2, 3, 4, 5])

#######################
# Klasik döngü ile
#######################
ab = []
for i in v:
    if i < 3:
        ab.append(i)

#######################
# By Numpy
#######################
v < 3

v[v < 3]
v[v > 3]
v[v != 3]
v[v == 3]
v[v >= 3]

#############################################
# Mathematical Operations
#############################################
import numpy as np
v = np.array([1, 2, 3, 4, 5])

v / 5
v * 5 / 10
v ** 2
v - 1

np.subtract(v, 1)
np.add(v, 1)
np.mean(v)
np.sum(v)
np.min(v)
np.max(v)
np.var(v)
v = np.subtract(v, 1)

v

#######################
# Solving Equations with Two Unknowns with NumPy
#######################

# 5*x0 + x1 = 12
# x0 + 3*x1 = 10

a = np.array([[5, 1], [1, 3]])
b = np.array([12, 10])

np.linalg.solve(a, b)

#############################################
# PANDAS
#############################################

# Pandas Series
# Reading Data
# Quick Look at Data
# Selection in Pandas
# Aggregation & Grouping
# Apply and Lambda
# Join

#############################################
# Pandas Series
#############################################
import pandas as pd

s = pd.Series([10, 77, 12, 4, 5])
type(s)
s.index
s.dtype
s.size
s.ndim
s.values
type(s.values)
s.head(3)
s.tail(3)

#############################################
# Reading Data
#############################################
import pandas as pd

df = pd.read_csv("PycharmProjects/pythonProject/datasets/advertising.csv")
df.head()
# pandas cheatsheet

#############################################
# Quick Look at Data
#############################################
import pandas as pd
import seaborn as sns

df = sns.load_dataset("PycharmProjects/pythonProject/datasets/titanic.csv")
df.head()
df.tail()
df.shape
df.info()
df.columns
df.index
df.describe().T
df.isnull().values.any()
df.isnull().sum()
df["sex"].head()
df["sex"].value_counts()

#############################################
# Selection in Pandas
#############################################
import pandas as pd
import seaborn as sns
df = sns.load_dataset("PycharmProjects/pythonProject/datasets/titanic.csv")
df.head()

df.index
df[0:13]
df.drop(0, axis=0).head()

delete_indexes = [1, 3, 5, 7]
df.drop(delete_indexes, axis=0).head(10)

# df = df.drop(delete_indexes, axis=0)
# df.drop(delete_indexes, axis=0, inplace=True)

#######################
# Converting a Variable to an Index
#######################

df["age"].head()
df.age.head()

df.index = df["age"]

df.drop("age", axis=1).head()

df.drop("age", axis=1, inplace=True)
df.head()

#######################
# Converting an Index to a Variable
#######################

df.index

df["age"] = df.index

df.head()
df.drop("age", axis=1, inplace=True)

df.reset_index().head()
df = df.reset_index()
df.head()

#######################
# Operations on Variables
#######################
import pandas as pd
import seaborn as sns
pd.set_option('display.max_columns', None)
df = sns.load_dataset("titanic")
df.head()

"age" in df

df["age"].head()
df.age.head()

df["age"].head()
type(df["age"].head())

df[["age"]].head()
type(df[["age"]].head())

df[["age", "alive"]]

col_names = ["age", "adult_male", "alive"]
df[col_names]

df["age2"] = df["age"]**2
df["age3"] = df["age"] / df["age2"]

df.drop("age3", axis=1).head()

df.drop(col_names, axis=1).head()

df.loc[:, ~df.columns.str.contains("age")].head()

#######################
# iloc & loc
#######################
import pandas as pd
import seaborn as sns
pd.set_option('display.max_columns', None)
df = sns.load_dataset("titanic")
df.head()

# iloc: integer based selection
df.iloc[0:3]
df.iloc[0, 0]

# loc: label based selection
df.loc[0:3]

df.iloc[0:3, 0:3]
df.loc[0:3, "age"]

col_names = ["age", "embarked", "alive"]
df.loc[0:3, col_names]

#######################
# Conditional Selection
#######################
import pandas as pd
import seaborn as sns
pd.set_option('display.max_columns', None)
df = sns.load_dataset("titanic")
df.head()

df[df["age"] > 50].head()
df[df["age"] > 50]["age"].count()

df.loc[df["age"] > 50, ["age", "class"]].head()

df.loc[(df["age"] > 50) & (df["sex"] == "male"), ["age", "class"]].head()

df["embark_town"].value_counts()

df_new = df.loc[(df["age"] > 50) & (df["sex"] == "male")
       & ((df["embark_town"] == "Cherbourg") | (df["embark_town"] == "Southampton")),
       ["age", "class", "embark_town"]]

df_new["embark_town"].value_counts()

#############################################
# Aggregation & Grouping
#############################################

# - count()
# - first()
# - last()
# - mean()
# - median()
# - min()
# - max()
# - std()
# - var()
# - sum()
# - pivot table

import pandas as pd
import seaborn as sns
pd.set_option('display.max_columns', None)
df = sns.load_dataset("titanic")
df.head()

df["age"].mean()

df.groupby("sex")["age"].mean()

df.groupby("sex").agg({"age": "mean"})
df.groupby("sex").agg({"age": ["mean", "sum"]})

df.groupby("sex").agg({"age": ["mean", "sum"],
                       "survived": "mean"})

df.groupby(["sex", "embark_town"]).agg({"age": ["mean"],
                       "survived": "mean"})

df.groupby(["sex", "embark_town", "class"]).agg({"age": ["mean"],
                       "survived": "mean"})

df.groupby(["sex", "embark_town", "class"]).agg({
    "age": ["mean"],
    "survived": "mean",
    "sex": "count"})

#######################
# Pivot table
#######################
import pandas as pd
import seaborn as sns
pd.set_option('display.max_columns', None)
df = sns.load_dataset("titanic")
df.head()

df.pivot_table("survived", "sex", "embarked")

df.pivot_table("survived", "sex", ["embarked", "class"])

df.head()

df["new_age"] = pd.cut(df["age"], [0, 10, 18, 25, 40, 90])

df.pivot_table("survived", "sex", ["new_age", "class"])

pd.set_option('display.width', 500)

#############################################
# Apply and Lambda
#############################################
import pandas as pd
import seaborn as sns
pd.set_option('display.max_columns', None)
pd.set_option('display.width', 500)
df = sns.load_dataset("titanic")
df.head()

df["age2"] = df["age"]*2
df["age3"] = df["age"]*5

(df["age"]/10).head()
(df["age2"]/10).head()
(df["age3"]/10).head()

for col in df.columns:
    if "age" in col:
        print(col)

for col in df.columns:
    if "age" in col:
        print((df[col]/10).head())

for col in df.columns:
    if "age" in col:
        df[col] = df[col]/10

df.head()

df[["age", "age2", "age3"]].apply(lambda x: x/10).head()

df.loc[:, df.columns.str.contains("age")].apply(lambda x: x/10).head()

df.loc[:, df.columns.str.contains("age")].apply(lambda x: (x - x.mean()) / x.std()).head()

def standart_scaler(col_name):
    return (col_name - col_name.mean()) / col_name.std()

df.loc[:, df.columns.str.contains("age")].apply(standart_scaler).head()

# df.loc[:, ["age","age2","age3"]] = df.loc[:, df.columns.str.contains("age")].apply(standart_scaler)

df.loc[:, df.columns.str.contains("age")] = df.loc[:, df.columns.str.contains("age")].apply(standart_scaler)

df.head()

#############################################
# Join
#############################################
import numpy as np
import pandas as pd
m = np.random.randint(1, 30, size=(5, 3))
df1 = pd.DataFrame(m, columns=["var1", "var2", "var3"])
df2 = df1 + 99

pd.concat([df1, df2])

pd.concat([df1, df2], ignore_index=True)

#######################
# Merge
#######################

df1 = pd.DataFrame({'employees': ['john', 'dennis', 'mark', 'maria'],
                    'group': ['accounting', 'engineering', 'engineering', 'hr']})

df2 = pd.DataFrame({'employees': ['mark', 'john', 'dennis', 'maria'],
                    'start_date': [2010, 2009, 2014, 2019]})

pd.merge(df1, df2)
pd.merge(df1, df2, on="employees")

# Amaç: Her çalışanın müdürünün bilgisine erişmek istiyoruz.
df3 = pd.merge(df1, df2)

df4 = pd.DataFrame({'group': ['accounting', 'engineering', 'hr'],
                    'manager': ['Caner', 'Mustafa', 'Berkcan']})

pd.merge(df3, df4)

#############################################
# Data Visualization: MATPLOTLIB & SEABORN
#############################################

#############################################
# MATPLOTLIB
#############################################

# Categorical Variable: countplot, bar
# Numerical Variable: hist, boxplot

#############################################
# Categorical Variable Visualization
#############################################
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
pd.set_option('display.max_columns', None)
pd.set_option('display.width', 500)
df = sns.load_dataset("titanic")
df.head()

df['sex'].value_counts().plot(kind='bar')
plt.show()

#############################################
# Numerical Variable Visualization
#############################################
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
pd.set_option('display.max_columns', None)
pd.set_option('display.width', 500)
df = sns.load_dataset("titanic")
df.head()

plt.hist(df["age"])
plt.show()

plt.boxplot(df["fare"])
plt.show()

#############################################
# Matplotlib Properties
#############################################
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
pd.set_option('display.max_columns', None)
pd.set_option('display.width', 500)

#######################
# plot
#######################

x = np.array([1, 8])
y = np.array([0, 150])

plt.plot(x, y)
plt.show()

plt.plot(x, y, 'o')
plt.show()

x = np.array([2, 4, 6, 8, 10])
y = np.array([1, 3, 5, 7, 9])

plt.plot(x, y)
plt.show()

plt.plot(x, y, 'o')
plt.show()

#######################
# marker
#######################

y = np.array([13, 28, 11, 100])

plt.plot(y, marker='o')
plt.show()

plt.plot(y, marker='*')
plt.show()

markers = ['o', '*', '.', ',', 'x', 'X', '+', 'P', 's', 'D', 'd', 'p', 'H', 'h']

#######################
# line
#######################

y = np.array([13, 28, 11, 100])
plt.plot(y, linestyle="dashdot", color="r")
plt.show()

#######################
# Multiple Lines
#######################

x = np.array([23, 18, 31, 10])
y = np.array([13, 28, 11, 100])
plt.plot(x)
plt.plot(y)
plt.show()

#######################
# Labels
#######################

x = np.array([80, 85, 90, 95, 100, 105, 110, 115, 120, 125])
y = np.array([240, 250, 260, 270, 280, 290, 300, 310, 320, 330])
plt.plot(x, y)
# Title
plt.title("This is the main topic")

# Naming the X-axis
plt.xlabel("Naming the X-axis")

plt.ylabel("Naming the Y-axis")

plt.grid()
plt.show()

#######################
# Subplots
#######################

# plot 1
x = np.array([80, 85, 90, 95, 100, 105, 110, 115, 120, 125])
y = np.array([240, 250, 260, 270, 280, 290, 300, 310, 320, 330])
plt.subplot(1, 2, 1)
plt.title("1")
plt.plot(x, y)

# plot 2
x = np.array([8, 8, 9, 9, 10, 15, 11, 15, 12, 15])
y = np.array([24, 20, 26, 27, 280, 29, 30, 30, 30, 30])
plt.subplot(1, 2, 2)
plt.title("2")
plt.plot(x, y)
plt.show()

# Showing 3 charts as one row and 3 columns
# plot 1
x = np.array([80, 85, 90, 95, 100, 105, 110, 115, 120, 125])
y = np.array([240, 250, 260, 270, 280, 290, 300, 310, 320, 330])
plt.subplot(1, 3, 1)
plt.title("1")
plt.plot(x, y)

# plot 2
x = np.array([8, 8, 9, 9, 10, 15, 11, 15, 12, 15])
y = np.array([24, 20, 26, 27, 280, 29, 30, 30, 30, 30])
plt.subplot(1, 3, 2)
plt.title("2")
plt.plot(x, y)

# plot 3
x = np.array([80, 85, 90, 95, 100, 105, 110, 115, 120, 125])
y = np.array([240, 250, 260, 270, 280, 290, 300, 310, 320, 330])
plt.subplot(1, 3, 3)
plt.title("3")
plt.plot(x, y)

plt.show()

#############################################
# SEABORN
#############################################
import pandas as pd
import seaborn as sns
from matplotlib import pyplot as plt
df = sns.load_dataset("tips")
df.head()

df["sex"].value_counts()
sns.countplot(x=df["sex"], data=df)
plt.show()

df['sex'].value_counts().plot(kind='bar')
plt.show()

#############################################
# Numerical Variable Visualization
#############################################

sns.boxplot(x=df["total_bill"])
plt.show()

df["total_bill"].hist()
plt.show()

#############################################
# ADVANCED FUNCTIONAL EDA
#############################################
# 1. General
# 2. Analysis of Categorical Variables
# 3. Analysis of Numerical Variables
# 4. Analysis of Target Variable
# 5. Analysis of Correlation

#############################################
# 1. General
#############################################
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
pd.set_option('display.max_columns', None)
pd.set_option('display.width', 500)
df = sns.load_dataset("titanic")
df.head()
df.tail()
df.shape
df.info()
df.columns
df.index
df.describe().T
df.isnull().values.any()
df.isnull().sum()

def check_df(dataframe, head=5):
    print("##################### Shape #####################")
    print(dataframe.shape)
    print("##################### Types #####################")
    print(dataframe.dtypes)
    print("##################### Head #####################")
    print(dataframe.head(head))
    print("##################### Tail #####################")
    print(dataframe.tail(head))
    print("##################### NA #####################")
    print(dataframe.isnull().sum())
    print("##################### Quantiles #####################")
    print(dataframe.describe([0, 0.05, 0.50, 0.95, 0.99, 1]).T)

check_df(df)

df = sns.load_dataset("flights")
check_df(df)

#############################################
# 2. Analysis of Categorical Variables
#############################################
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
pd.set_option('display.max_columns', None)
pd.set_option('display.width', 500)
df = sns.load_dataset("titanic")
df.head()

df["survived"].value_counts()
df["sex"].unique()
df["class"].nunique()

cat_cols = [col for col in df.columns if str(df[col].dtypes) in ["category", "object", "bool"]]

num_but_cat = [col for col in df.columns if df[col].nunique() < 10 and df[col].dtypes in ["int", "float"]]

cat_but_car = [col for col in df.columns if df[col].nunique() > 20 and str(df[col].dtypes) in ["category", "object"]]

cat_cols = cat_cols + num_but_cat

cat_cols = [col for col in cat_cols if col not in cat_but_car]

df[cat_cols].nunique()

[col for col in df.columns if col not in cat_cols]

df["survived"].value_counts()
100 * df["survived"].value_counts() / len(df)

def cat_summary(dataframe, col_name):
    print(pd.DataFrame({col_name: dataframe[col_name].value_counts(),
                        "Ratio": 100 * dataframe[col_name].value_counts() / len(dataframe)}))
    print("##########################################")

cat_summary(df, "sex")

for col in cat_cols:
    cat_summary(df, col)

def cat_summary(dataframe, col_name, plot=False):
    print(pd.DataFrame({col_name: dataframe[col_name].value_counts(),
                        "Ratio": 100 * dataframe[col_name].value_counts() / len(dataframe)}))
    print("##########################################")

    if plot:
        sns.countplot(x=dataframe[col_name], data=dataframe)
        plt.show(block=True)

cat_summary(df, "sex", plot=True)

for col in cat_cols:
    if df[col].dtypes == "bool":
        print("sdfsdfsdfsdfsdfsd")
    else:
        cat_summary(df, col, plot=True)

df["adult_male"].astype(int)

for col in cat_cols:
    if df[col].dtypes == "bool":
        df[col] = df[col].astype(int)
        cat_summary(df, col, plot=True)

    else:
        cat_summary(df, col, plot=True)

def cat_summary(dataframe, col_name, plot=False):

    if dataframe[col_name].dtypes == "bool":
        dataframe[col_name] = dataframe[col_name].astype(int)

        print(pd.DataFrame({col_name: dataframe[col_name].value_counts(),
                            "Ratio": 100 * dataframe[col_name].value_counts() / len(dataframe)}))
        print("##########################################")

        if plot:
            sns.countplot(x=dataframe[col_name], data=dataframe)
            plt.show(block=True)
    else:
        print(pd.DataFrame({col_name: dataframe[col_name].value_counts(),
                            "Ratio": 100 * dataframe[col_name].value_counts() / len(dataframe)}))
        print("##########################################")

        if plot:
            sns.countplot(x=dataframe[col_name], data=dataframe)
            plt.show(block=True)

cat_summary(df, "adult_male", plot=True)

def cat_summary(dataframe, col_name):
    print(pd.DataFrame({col_name: dataframe[col_name].value_counts(),
                        "Ratio": 100 * dataframe[col_name].value_counts() / len(dataframe)}))
    print("##########################################")

cat_summary(df, "sex")

#############################################
# 3. Analysis of Numerical Variables
#############################################

import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
pd.set_option('display.max_columns', None)
pd.set_option('display.width', 500)
df = sns.load_dataset("titanic")
df.head()

cat_cols = [col for col in df.columns if str(df[col].dtypes) in ["category", "object", "bool"]]
num_but_cat = [col for col in df.columns if df[col].nunique() < 10 and df[col].dtypes in ["int", "float"]]
cat_but_car = [col for col in df.columns if df[col].nunique() > 20 and str(df[col].dtypes) in ["category", "object"]]
cat_cols = cat_cols + num_but_cat
cat_cols = [col for col in cat_cols if col not in cat_but_car]

df[["age","fare"]].describe().T

num_cols = [col for col in df.columns if df[col].dtypes in ["int","float"]]
num_cols = [col for col in num_cols if col not in cat_cols]

def num_summary(dataframe, numerical_col):
    quantiles = [0.05, 0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80, 0.90, 0.95, 0.99]
    print(dataframe[numerical_col].describe(quantiles).T)

num_summary(df, "age")

for col in num_cols:
    num_summary(df, col)

def num_summary(dataframe, numerical_col, plot=False):
    quantiles = [0.05, 0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80, 0.90, 0.95, 0.99]
    print(dataframe[numerical_col].describe(quantiles).T)

    if plot:
        dataframe[numerical_col].hist()
        plt.xlabel(numerical_col)
        plt.title(numerical_col)
        plt.show(block=True)

num_summary(df, "age", plot=True)

for col in num_cols:
    num_summary(df, col, plot=True)

#############################################
# Capturing Variables and Generalizing Operations
#############################################

import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
pd.set_option('display.max_columns', None)
pd.set_option('display.width', 500)
df = sns.load_dataset("titanic")
df.head()
df.info()

# docstring

def grab_col_names(dataframe, cat_th=10,  car_th=20):
    """
    It gives the names of categorical, numerical and categorical but cardinal variables in the data set.

    Parameters
    ----------
    dataframe: dataframe
        dataframe where to get variable names
    cat_th: int, float
        class threshold for numeric but categorical variables
    car_th: int, float
        class threshold for categorical but cardinal variables

    Returns
    -------
    cat_cols: list
        Categorical variable list
    num_cols: list
        Numeric variable list
    cat_but_car: list
        Categorical view cardinal variable list

    Notes
    ------
    cat_cols + num_cols + cat_but_car = total variable numbers
    num_but_cat is in cat_cols

    """
    # cat_cols, cat_but_car
    cat_cols = [col for col in dataframe.columns if str(dataframe[col].dtypes) in ["category", "object", "bool"]]

    num_but_cat = [col for col in dataframe.columns if dataframe[col].nunique() < 10 and dataframe[col].dtypes in ["int", "float"]]

    cat_but_car = [col for col in dataframe.columns if
                   dataframe[col].nunique() > 20 and str(dataframe[col].dtypes) in ["category", "object"]]

    cat_cols = cat_cols + num_but_cat
    cat_cols = [col for col in cat_cols if col not in cat_but_car]

    num_cols = [col for col in dataframe.columns if dataframe[col].dtypes in ["int", "float"]]
    num_cols = [col for col in num_cols if col not in cat_cols]

    print(f"Observations: {dataframe.shape[0]}")
    print(f"Variables: {dataframe.shape[1]}")
    print(f'cat_cols: {len(cat_cols)}')
    print(f'num_cols: {len(num_cols)}')
    print(f'cat_but_car: {len(cat_but_car)}')
    print(f'num_but_cat: {len(num_but_cat)}')

    return cat_cols, num_cols, cat_but_car

cat_cols, num_cols, cat_but_car = grab_col_names(df)

def cat_summary(dataframe, col_name):
    print(pd.DataFrame({col_name: dataframe[col_name].value_counts(),
                        "Ratio": 100 * dataframe[col_name].value_counts() / len(dataframe)}))
    print("##########################################")

cat_summary(df, "sex")

for col in cat_cols:
    cat_summary(df, col)

def num_summary(dataframe, numerical_col, plot=False):
    quantiles = [0.05, 0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80, 0.90, 0.95, 0.99]
    print(dataframe[numerical_col].describe(quantiles).T)

    if plot:
        dataframe[numerical_col].hist()
        plt.xlabel(numerical_col)
        plt.title(numerical_col)
        plt.show(block=True)

for col in num_cols:
    num_summary(df, col, plot=True)

# BONUS
df = sns.load_dataset("titanic")
df.info()

for col in df.columns:
    if df[col].dtypes == "bool":
        df[col] = df[col].astype(int)

cat_cols, num_cols, cat_but_car = grab_col_names(df)

def cat_summary(dataframe, col_name, plot=False):
    print(pd.DataFrame({col_name: dataframe[col_name].value_counts(),
                        "Ratio": 100 * dataframe[col_name].value_counts() / len(dataframe)}))
    print("##########################################")

    if plot:
        sns.countplot(x=dataframe[col_name], data=dataframe)
        plt.show(block=True)

for col in cat_cols:
    cat_summary(df, col, plot=True)

for col in num_cols:
    num_summary(df, col, plot=True)

#############################################
# 4. Analysis of Target Variable
#############################################
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
pd.set_option('display.max_columns', None)
pd.set_option('display.width', 500)
df = sns.load_dataset("titanic")

for col in df.columns:
    if df[col].dtypes == "bool":
        df[col] = df[col].astype(int)

def cat_summary(dataframe, col_name, plot=False):
    print(pd.DataFrame({col_name: dataframe[col_name].value_counts(),
                        "Ratio": 100 * dataframe[col_name].value_counts() / len(dataframe)}))
    print("##########################################")

    if plot:
        sns.countplot(x=dataframe[col_name], data=dataframe)
        plt.show(block=True)

def grab_col_names(dataframe, cat_th=10,  car_th=20):
    """
    It gives the names of categorical, numerical and categorical but cardinal variables in the data set.

    Parameters
    ----------
    dataframe: dataframe
        dataframe where to get variable names
    cat_th: int, float
        class threshold for numeric but categorical variables
    car_th: int, float
        class threshold for categorical but cardinal variables

    Returns
    -------
    cat_cols: list
        Categorical variable list
    num_cols: list
        Numeric variable list
    cat_but_car: list
        Categorical view cardinal variable list

    Notes
    ------
    cat_cols + num_cols + cat_but_car = total variable numbers
    num_but_cat is in cat_cols

    """
    # cat_cols, cat_but_car
    cat_cols = [col for col in df.columns if str(df[col].dtypes) in ["category", "object", "bool"]]

    num_but_cat = [col for col in df.columns if df[col].nunique() < 10 and df[col].dtypes in ["int", "float"]]

    cat_but_car = [col for col in df.columns if
                   df[col].nunique() > 20 and str(df[col].dtypes) in ["category", "object"]]

    cat_cols = cat_cols + num_but_cat
    cat_cols = [col for col in cat_cols if col not in cat_but_car]

    num_cols = [col for col in df.columns if df[col].dtypes in ["int", "float"]]
    num_cols = [col for col in num_cols if col not in cat_cols]

    print(f"Observations: {dataframe.shape[0]}")
    print(f"Variables: {dataframe.shape[1]}")
    print(f'cat_cols: {len(cat_cols)}')
    print(f'num_cols: {len(num_cols)}')
    print(f'cat_but_car: {len(cat_but_car)}')
    print(f'num_but_cat: {len(num_but_cat)}')

    return cat_cols, num_cols, cat_but_car

cat_cols, num_cols, cat_but_car = grab_col_names(df)

df.head()

df["survived"].value_counts()
cat_summary(df, "survived")

#######################
# Analysis of the Target Variable with Categorical Variables
#######################

df.groupby("sex")["survived"].mean()

def target_summary_with_cat(dataframe, target, categorical_col):
    print(pd.DataFrame({"TARGET_MEAN": dataframe.groupby(categorical_col)[target].mean()}), end="\n\n\n")

target_summary_with_cat(df, "survived", "pclass")

for col in cat_cols:
    target_summary_with_cat(df, "survived", col)

#######################
# Analysis of the Target Variable with Numerical Variables
#######################

df.groupby("survived")["age"].mean()

df.groupby("survived").agg({"age":"mean"})

def target_summary_with_num(dataframe, target, numerical_col):
    print(dataframe.groupby(target).agg({numerical_col: "mean"}), end="\n\n\n")

target_summary_with_num(df, "survived","age")

for col in num_cols:
    target_summary_with_num(df, "survived", col)

#############################################
# 5. Analysis of Correlation
#############################################
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
pd.set_option('display.max_columns', None)
pd.set_option('display.width', 500)
df = pd.read_csv("datasets/breast_cancer.csv")
df = df.iloc[:, 1:-1]
df.head()

num_cols = [col for col in df.columns if df[col].dtype in [int, float]]

corr = df[num_cols].corr()

sns.set(rc={'figure.figsize': (12, 12)})
sns.heatmap(corr, cmap="RdBu")
plt.show()

#######################
# Deleting Highly Correlation Variables
#######################

cor_matrix = df.corr().abs()

#           0         1         2         3
# 0  1.000000  0.117570  0.871754  0.817941
# 1  0.117570  1.000000  0.428440  0.366126
# 2  0.871754  0.428440  1.000000  0.962865
# 3  0.817941  0.366126  0.962865  1.000000

#     0        1         2         3
# 0 NaN  0.11757  0.871754  0.817941
# 1 NaN      NaN  0.428440  0.366126
# 2 NaN      NaN       NaN  0.962865
# 3 NaN      NaN       NaN       NaN

upper_triangle_matrix = cor_matrix.where(np.triu(np.ones(cor_matrix.shape), k=1).astype(bool))
drop_list = [col for col in upper_triangle_matrix.columns if any(upper_triangle_matrix[col]>0.90) ]
cor_matrix[drop_list]
df.drop(drop_list, axis=1)

def high_correlated_cols(dataframe, plot=False, corr_th=0.90):
    corr = dataframe.corr()
    cor_matrix = corr.abs()
    upper_triangle_matrix = cor_matrix.where(np.triu(np.ones(cor_matrix.shape), k=1).astype(bool))
    drop_list = [col for col in upper_triangle_matrix.columns if any(upper_triangle_matrix[col] > corr_th)]
    if plot:
        import seaborn as sns
        import matplotlib.pyplot as plt
        sns.set(rc={'figure.figsize': (15, 15)})
        sns.heatmap(corr, cmap="RdBu")
        plt.show()
    return drop_list

high_correlated_cols(df)
drop_list = high_correlated_cols(df, plot=True)
df.drop(drop_list, axis=1)
high_correlated_cols(df.drop(drop_list, axis=1), plot=True)

# Let's try it on a dataset of about 600mb with more than 300 variables.
# https://www.kaggle.com/c/ieee-fraud-detection/data?select=train_transaction.csv

df = pd.read_csv("datasets/fraud_train_transaction.csv")
len(df.columns)
df.head()

drop_list = high_correlated_cols(df, plot=True)

len(df.drop(drop_list, axis=1).columns)

type(adsa)


"""
The :mod:`sklearn.QuantumUtility` module includes quantum routines like tomography, amplitude estimation and
phase estimation
"""

from collections import Counter
import math
import random
import re
import time
from scipy.stats import truncnorm
import numpy as np
from bisect import bisect
import warnings

# import gc

warnings.simplefilter('always', UserWarning)
import matplotlib.pyplot as plt
from multiprocessing import *

# random.seed(a=31337)

class QuantumState(object):
    """This class simulates a simple Quantum Register

    Parameters
    ----------

    registers: list, ndarray.
        List of values that represents the superposition of the states of the quantum state.

    amplitudes: list, ndarray.
        List of values that represents the amplitudes of each register.
    """

    def __init__(self, registers, amplitudes):
        super(QuantumState, self).__init__()
        self.registers = registers

        # Amplitudes must be normalized to have the right probabilities for each register
        self.norm_factor = math.sqrt(sum([pow(x, 2) for x in amplitudes]))
        self.amplitudes = [x / self.norm_factor for x in amplitudes]

        # Each register_i appears with probability amplitude_i^2
        self.probabilities = [pow(x, 2) for x in self.amplitudes]
        assert (len(self.registers) == len(self.amplitudes))
        assert (abs(sum(self.probabilities) - 1) < 0.0000000001)

    def measure(self, n_times=1):
        # return random.choices(self.registers, weights=self.probabilities, k=n_times)
        LocalProcRandGen = np.random.RandomState()
        return LocalProcRandGen.choice(self.registers, p=self.probabilities, size=n_times)
        # return np.random.choice(self.registers, p=self.probabilities, size=n_times)

    def get_state(self):
        return {self.registers[i]: self.probabilities[i] for i in range(len(self.registers))}

def estimate_wald(measurements):
    counter = Counter(measurements)
    estimate = {x: counter[x] / len(measurements) for x in counter}
    return estimate

## Introduce +-epsilon error in a value
def introduce_error(value, epsilon):
    return value + truncnorm.rvs(-epsilon, epsilon, size=1)

def coupon_collect(quantum_state):
    counter = 0
    collection_dict = {value: 0 for value in quantum_state.get_state().keys()}

    # Until you don't collect all the values, keep sampling and increment the counter
    while sum(collection_dict.values()) != len(collection_dict):
        value = quantum_state.measure()[0]
        if not collection_dict[value]:
            collection_dict[value] = 1
        counter += 1
    return counter

def make_gaussian_est(vec, noise):
    """This function is used to estimate vec with tomography or with Gaussian Noise Approximation.

    Parameters
    ----------
    vec: array-like that has to be estimated.

    noise: float. It represent the error that you want to introduce to estimate the representation of vector V.
    """
    noise_per_component = noise / np.sqrt(len(vec))
    if noise_per_component != 0:
        errors = truncnorm.rvs(-noise_per_component, noise_per_component, size=len(vec))
        # print('noise_per_comp:', noise_per_component, 'errors:', errors)
        somma = lambda x, y: x + y
        # new_vec = np.array([vec[i] + errors[i] for i in range(len(vec))])
        new_vec = np.apply_along_axis(somma, 0, vec, errors)
    return new_vec

def tomography(A, noise, true_tomography=True, stop_when_reached_accuracy=True, N=None, norm='L2',
               incremental_measure=True, faster_measure_increment=0):
    """ Tomography function (real and approximate).

    Parameters
    ----------
    A: array-like that has to be estimated.

    noise: float value.
        It represent the error that you want to introduce to estimate the representation of vector V.

    true_tomography: bool, default=True.
        If true means that the quantum estimations are are done with real tomography,
        otherwise the estimations are approximated with a Truncated Gaussian Noise.

    stop_when_reached_accuracy: bool value, default=True.
        If True it stops the execution of the tomography when the L2(or Linf)-norm of the
        difference between V and its estimation is less or equal then delta. Otherwise
        N measures are done (very memory intensive for large vectors).

    N: int value, default=None.
        Number of measures of the quantum state. If None it is computed in the function itself.

    norm: string value, default='L2'
        If true_tomography is True:
            'L2':
                L2-tomography is computed
            'inf':
                L-inf tomography is computed

    incremental_measure: bool, default=True.
        If True the tomography is computed incrementally up to N measures. If False, the routine is
        performed once using N measures.

    faster_measure_increment: int, default=0.
        It speeds up the tomography increasing the number of measurements in the incremental way.

    Returns
    -------
    A_est: array-like that was estimated.

    Notes
    -----
    This method returns an estimation of the true array/matrix A using quantum tomography algorithm 4.1 proposed in
    "A Quantum Interior Point Method for LPs and SDPs" paper, or using an approximation of the tomography computation.
    """

    assert noise >= 0

    if noise == 0:
        return A

    if true_tomography == False:
        if len(A.shape) == 2:
            vector_A = A.reshape(A.shape[0] * A.shape[1])
            vector_B = make_gaussian_est(vector_A, noise)
            A_est = vector_B.reshape(A.shape[0], A.shape[1])
        else:
            A_est = make_gaussian_est(A, noise)

    else:
        if len(A.shape) == 2:
            A_est = np.array([np.array(list(
                real_tomography(A[idx], delta=noise, stop_when_reached_accuracy=stop_when_reached_accuracy,
                                N=N, norm=norm, incremental_measure=incremental_measure,
                                faster_measure_increment=faster_measure_increment).values())[-1]) for idx in
                              range(len(A))])
        else:
            A_est = np.array(
                list(real_tomography(A, delta=noise, stop_when_reached_accuracy=stop_when_reached_accuracy, N=N,
                                     norm=norm, incremental_measure=incremental_measure,
                                     faster_measure_increment=faster_measure_increment).values())[-1])

    return A_est

def create_rand_vec(n_vec, len_vec, scale=None, type='uniform'):
    v = []
    for i in range(n_vec):
        if type == 'uniform':
            vv = np.random.uniform(-1, 1, (len_vec))
        elif type == 'exp':
            vv = np.random.exponential(scale=scale, size=len_vec)
        vv = vv / np.linalg.norm(vv, ord=2)
        v.append(vv)

    return v

def __mu(p, matrix):
    def s(p, A):
        if p == 0:
            result = np.max([np.count_nonzero(A[i]) for i in range(len(A))])
        else:
            norms = np.sum(np.power(np.abs(A), p), axis=1)
            result = max(norms)
            del norms
        # gc.collect()
        return result

    s1 = s(2 * p, matrix)
    s2 = s(2 * (1 - p), matrix.T)
    mu = np.sqrt(s1 * s2)

    # gc.collect()
    return mu

def linear_search(matrix, start=0.0, end=1.0, step=0.05):
    domain = [i for i in np.arange(start, end, step)] + [end]
    values = [__mu(i, matrix) for i in domain]
    best_p = domain[values.index(min(values))]
    return best_p, min(values)

def best_mu(matrix, start=0.0, end=1.0, step=0.05):
    p, val = linear_search(matrix, start=start, end=end, step=step)
    val_list = [val, np.linalg.norm(matrix)]
    index = val_list.index(min(val_list))
    if index == 0:
        best_norm = f"p={p}"
    elif index == 1:
        best_norm = "Frobenius"

    return best_norm, val_list[index]

def L2_tomogrphy_fakeSign(V, N=None, delta=None):
    d = len(V)
    # index = np.arange(0,d)
    if N is None:
        N = (36 * d * np.log(d)) / (delta ** 2)

    q_state = QuantumState(amplitudes=V, registers=V)
    P = estimate_wald(q_state.measure(n_times=int(N)))
    # P = {V[k]: v for (k, v) in P.items()}

    # Manage the mismatch problem of the length of the measurements for some values
    if len(P) < d:
        keys = set(list(P.keys()))
        v_set = set(V)
        missing_values = list(v_set - keys)
        P.update({l: 0 for l in missing_values})

    P_sqrt = {k: (-np.sqrt(v) if k < 0 else np.sqrt(v)) for (k, v) in P.items()}

    x_sign = list(map(P_sqrt.get, V))
    # x_sign = np.array(list(P_sqrt.values()))
    # print(np.linalg.norm(x_sign,ord=2))
    return x_sign

def real_tomography(V, N=None, delta=None, stop_when_reached_accuracy=True, norm='L2', incremental_measure=True,
                    faster_measure_increment=0):
    """ Official version of the tomography function.

    Parameters
    ----------
    V: array-like that has to be estimated.

    N: int value, default=None.
        Number of measures of the quantum state. If None it is computed in the function itself.

    delta: float value, default=None.
        It represent the error that you want to introduce to estimate the representation of vector V.

    stop_when_reached_accuracy: bool, default=True.
        If True it stops the execution of the tomography when the L2-norm of the
        difference between V and its estimation is less or equal then delta. Otherwise
        N measures are done (very memory intensive for large vectors).

    norm: string, default='L2'.
        If 'L2':
            perform L2-norm tomography.
        If 'inf':
            perform L-inf tomography.

    incremental_measure: bool, default=True.
        If True the tomography is computed incrementally up to N measures. If False, the routine is
        performed once using N measures.

    faster_measure_increment: int, default=0.
        It speeds up the tomography increasing the number of measurements in the incremental way.

    Returns
    -------
    dict_res: dictionary of shape {N_measure: vector_estimation}.

    Notes
    -----
    This method returns an estimation of the true array V using quantum tomography algorithm 4.1 proposed in
    "A Quantum Interior Point Method for LPs and SDPs" paper.

    """
    if np.isclose(np.linalg.norm(V), 1, rtol=1e-2):
        pass
    else:
        V = V / np.linalg.norm(V, ord=2)
    d = len(V)
    index = np.arange(0, d)
    if N is None:
        if norm == 'L2':
            N = int((36 * d * np.log(d)) / (delta ** 2))
        elif norm == 'inf':
            N = int((36 * np.log(d)) / (delta ** 2))

    q_state = QuantumState(amplitudes=V, registers=index)
    dict_res = {}
    if incremental_measure:

        measure_indexes = np.geomspace(1, N, num=100, dtype=np.int64)

        measure_indexes = check_measure(measure_indexes, faster_measure_increment=faster_measure_increment)

        for i in measure_indexes:

            P = estimate_wald(q_state.measure(n_times=int(i)))
            P_i = np.zeros(d)
            P_i[list(P.keys())] = np.sqrt(list(P.values()))

            # Part2 of algorithm 4.1
            max_index = max(index)
            digits = len(str(max_index)) + 1
            registers = [str(j).zfill(digits) for j in index] + [re.sub('0', '1', str(j).zfill(digits), 1) for j in
                                                                 index]

            amplitudes = np.asarray([V[k] + P_i[k] for k in range(len(V))] + [V[k] - P_i[k] for k in range(len(V))])

            amplitudes *= 0.5

            new_quantum_state = QuantumState(registers=registers, amplitudes=amplitudes)

            measure = new_quantum_state.measure(n_times=int(i))

            str_ = [str(ind).zfill(digits) for ind in index]
            dictionary = dict(Counter(measure))

            if len(dictionary) < len(registers):
                keys = set(list(dictionary.keys()))
                tot_keys = set(registers)
                missing_keys = list(tot_keys - keys)
                dictionary.update({l: 0 for l in missing_keys})

            d_ = list(map(dictionary.get, str_))

            P_i = [P_i[e] if x > 0.4 * P_i[e] ** 2 * i else P_i[e] * -1 for e, x in enumerate(d_)]

            dict_res.update({i: P_i})
            if stop_when_reached_accuracy:
                if norm == 'L2':
                    sample = np.linalg.norm(V - P_i, ord=2)
                elif norm == 'inf':
                    sample = np.linalg.norm(V - P_i, ord=np.inf)
                if sample > delta:
                    pass
                else:
                    break

    else:
        P = estimate_wald(q_state.measure(n_times=int(N)))
        P_i = np.zeros(d)
        # indexes = [list(q_state.registers).index(key) for key in list(P.keys())]
        P_i[list(P.keys())] = np.sqrt(list(P.values()))
        # P_i[indexes] = np.sqrt(list(P.values()))

        # Part2 of algorithm 4.1
        max_index = max(index)
        digits = len(str(max_index)) + 1
        registers = [str(j).zfill(digits) for j in index] + [re.sub('0', '1', str(j).zfill(digits), 1) for j in index]

        amplitudes = np.asarray([V[k] + P_i[k] for k in range(len(V))] + [V[k] - P_i[k] for k in range(len(V))])

        amplitudes *= 0.5

        new_quantum_state = QuantumState(registers=registers, amplitudes=amplitudes)

        measure = new_quantum_state.measure(n_times=int(N))

        str_ = [str(ind).zfill(digits) for ind in index]
        dictionary = dict(Counter(measure))

        if len(dictionary) < len(registers):
            keys = set(list(dictionary.keys()))
            tot_keys = set(registers)
            missing_keys = list(tot_keys - keys)
            dictionary.update({l: 0 for l in missing_keys})

        d_ = list(map(dictionary.get, str_))

        P_i = [P_i[e] if x > 0.4 * P_i[e] ** 2 * N else P_i[e] * -1 for e, x in enumerate(d_)]

        dict_res.update({N: P_i})

        # print('error_tomography:',np.linalg.norm(V - P_i, ord=2))

    return dict_res

def auxiliary_fun(q_state, i):
    P = q_state.measure(n_times=int(i))
    return P

def vectorize_aux_fun(dic, i):
    return np.sqrt(dic[i]) if i in dic else 0

def check_measure(arr, faster_measure_increment):
    incr = 5 + faster_measure_increment

    for i in range(len(arr) - 1):
        if arr[i + 1] == arr[i]:
            arr[i + 1] += incr
        if arr[i + 1] <= arr[i]:
            arr[i + 1] = arr[i] + incr
    return arr

def check_division(v, n_jobs):
    a = float(v) / n_jobs
    d = a - int(a)
    remaining = int(d * n_jobs)
    process_values = [int(a) for _ in range(n_jobs)]
    for i in range(remaining):
        process_values[i] += 1
    return process_values

def amplitude_est_dist(w0, w1):
    c = -np.ceil(w1 - w0)
    f = -np.floor(w1 - w0)
    distance = min(np.abs(c + w1 - w0), np.abs(f + w1 - w0))
    return distance

def amplitude_estimation(a, epsilon=0.01, gamma=None, M=None, nqubit=False, plot_distribution=False):
    """ Official version of the amplitude estimation function.

    Parameters
    ----------
    a: float or int value.
         Value that has to be estimated by the amplitude estimation. It must be in the range of [0,1].

    epsilon: float value, default=0.01
        Error that you want to tolerate in the estimate of amplitude estimation.

    gamma: float value, default=None.
        It represent the probability of failure of amplitude estimation. If specified, median evaluation is performed
        to boost the probability of success of this routine.

    M: int value, default=None.
        The number of iteration executes in the routine.

    nqubit: bool value, default=False.
        If True, the routine returns also the number of qubits to represent an estimate with the specified precision
        and the parameter M (see the amplitude estimation routine description for more details).

    plot_distribution: bool value, default=False.
        If True, a plot of the probability distribution for the output of amplitude estimation is done.

    Returns
    -------
    a_tilde: float value.
        Estimate of the probability of measure a "good" state.

    Notes
    -----
    This method performs the amplitude estimation routine following the approach described in "Quantum Amplitude Amplification
    and Estimation" paper. Amplitude estimation is the problem of estimating the probability that a measurements of
    a quantum state yields a good state.
    """
    if gamma:
        return median_evaluation(amplitude_estimation, gamma=gamma, Q=None, a=a, epsilon=epsilon, M=M,
                                 nqubit=False, plot_distribution=plot_distribution)

    if M == None:
        M = (math.ceil((np.pi / (2 * epsilon)) * (1 + np.sqrt(1 + 4 * epsilon))))
        n_qubits = np.ceil(np.log2(M))
        # M = int(2 ** n_qubits)  # If M is a power of 2. In Mosca book they said this.
    else:
        n_qubits = np.ceil(np.log2(M))

        warnings.warn(
            "Attention! The value of M that will be considered is the one you passed. Epsilon in this case is "
            "useless")

    theta_a = math.asin(np.sqrt(a))
    p = []
    theta_j = []

    for j in range(M):
        theta_est = np.pi * j / M
        theta_j.append(theta_est)
        distance = amplitude_est_dist(theta_est / np.pi, theta_a / np.pi)
        if distance != 0:
            p_aj = np.abs(math.sin(M * distance * np.pi) / (M * math.sin(distance * np.pi))) ** 2
        else:
            p_aj = 1
        p.append(p_aj)
    # sum_at_one = np.sum(p)
    theta_tilde = random.choices(theta_j, weights=p, k=1)[0]

    if plot_distribution:
        relative_error = epsilon * max(theta_a, 1)
        plt.annotate((float('%.2f' % (theta_j[p.index(max(p))])), float('%.2f' % (max(p)))),
                     xy=(theta_j[p.index(max(p))], max(p)))
        plt.bar(theta_j, p, 0.001)
        plt.axvline(theta_a - relative_error, c='red', ls='dashed')
        plt.axvline(theta_a + relative_error, c='red', ls='dashed')
        plt.xlim(theta_j[p.index(max(p))] - 0.03, theta_j[p.index(max(p))] + 0.03)
        plt.title(r'Probability distribution for the output of amplitude estimation for $\theta$ = ' + str(
            float('%.2f' % (theta_a))) + r' with $\epsilon$ =' + str(epsilon) + r'$\rightarrow$ M=' + str(M),
                  fontdict={'family': 'serif',
                            'color': 'darkblue',
                            'weight': 'bold',
                            'size': 8})
        plt.xlabel(r'$\hat a$')
        plt.ylabel("probability")

        plt.show()
    if nqubit:
        return theta_tilde, n_qubits, M
    a_tilde = np.sin(theta_tilde) ** 2
    return a_tilde

def median_evaluation(func, gamma=0.1, Q=None, *args, **kwargs):
    """Median evaluation.

    Parameters
    ----------
    func: Callable.
        The function that you want to execute Q time.

    gamma: float value, default=0.1.
        The probability that the median estimation gives a value satisfying the error bounds.

    Q: int value, default=None.
        Number of iterations to execute func.

    args:
        list of parameters values to pass to the callable function.

    kwargs:
        arguments of type key->value to pass to the callable function.

    Returns
    -------
    final_estimate : float value.
        Median estimation of the function passed as arguments.

    Notes
    -----
    This procedure at high level computes Q times the result of the callable func passed, and extract the median
    to be more accurate in the estimation. It is used to boost the probability and precision for the estimation.
    """
    if Q == None:
        z = np.log(1 / gamma) / (2 * (8 / np.pi ** 2 - 0.5) ** 2)
        Q = np.ceil(z)
        if Q % 2 == 0:
            Q += 1

    estimates = [func(*args, **kwargs) for _ in range(int(Q))]
    final_estimate = np.median(estimates)
    return final_estimate

def wrapper_phase_est_arguments(argument, type='sv'):
    if type == 'sv':
        theta_i = 2 * math.acos(argument)
        return theta_i
    if type == 'distance':
        theta_i = math.asin(np.sqrt(argument))
        return theta_i

def unwrap_phase_est_arguments(argument, eps, type='sv'):
    if type == 'sv':
        return math.cos(argument * (eps + np.pi) / 2)
    if type == 'distance':
        return math.sin(argument * np.pi) ** 2

def phase_estimation(omega, m=None, epsilon=None, gamma=0.1, plot_distribution=False, nqubit=False):
    """ Official version of the phase estimation function.

        Parameters
        ----------
        omega: float or int value.
             Value that has to be estimated by the phase estimation. It must be in the range of [0,1).

        m: int value, default=None.
            The number of qubits you want to use for the computation.

        epsilon: float value, default=None.
            Precision that you want to have in the phase estimation procedure. If not None, it's used to compute m.

        nqubit: bool value, default=False.
            If True, the routine returns the estimation of omega, the k_est and also the number of qubits to represent
            an estimate with the specified precision and the parameter M.

        gamma: float value, default=0.1.
            It represent the probability of success of the routine.

        plot_distribution: bool value, default=False.
            If True, a plot of the probability distribution for the output of phase estimation is done.

        Returns
        -------
        omega_tilde: float value.
            Estimate of the true omega value.

        Notes
        -----
        This method performs the phase estimation routine following the approach described in "An Introduction
        to Quantum computing" book and "Quantum Algorithm for Unsupervised ML and NN" thesis.
        Phase estimation is the problem of estimating the phase of the eigenvectors of a unitary U using m qubits of
        precision. From this method is derived the :mod:`sklearn.QuantumUtility.Utility.amplitude_estimation` procedure.
    """
    # omega must be between [0,1). The estimation must have shape of x/2^n
    assert m != None or epsilon != None, "Attention! You need to specify the number of qubits m or the precision epsilon."
    if m != None and nqubit == True:
        warnings.warn("Attention! You are specifying that you want to return also the number of qubits" \
                      " used, but you are already specifying it with the m parameter.")
    if epsilon != None:
        warnings.warn('Attention! The m value is computed using the epsilon parameter passed.')
        # From Nielsen and Chuang (eq. 5.35).
        m = int(np.ceil(np.log2(1 / epsilon)) + np.ceil(np.log2(2 + 1 / (2 * gamma))))

    p = []
    omega_k = []
    M = 2 ** m  # in P.E., M is fixed in this way
    if omega == 1 or np.isclose(omega, 1):
        return (M - 1) / M
    for k in range(M):
        omega_est = k / M
        omega_k.append(omega_est)
        # Landman
        try:
            p.append(np.abs((math.sin((M * omega - k) * np.pi)) / (M * (math.sin((omega - k / M) * np.pi)))) ** 2)
        except:
            # if the division is 0/0 -> case when M*omega is an integer.
            p.append(1)

    # sum_at_one = np.sum(p)
    omega_tilde = random.choices(omega_k, weights=p, k=1)[0]

    k_est = omega_tilde * M

    if plot_distribution:
        # rel_error = epsilon * max(omega, 1)
        rel_error = epsilon * omega
        plt.annotate((float('%.4f' % (omega_k[p.index(max(p))])), float('%.2f' % (max(p)))),
                     xy=(omega_k[p.index(max(p))], max(p)))
        plt.bar(omega_k, p, 0.0001)

        plt.axvline(omega - rel_error, c='red', ls='dashed')
        plt.axvline(omega + rel_error, c='red', ls='dashed')
        plt.axvline(omega, c='yellow', ls='dashed', label=r'$\omega$=' + str(omega))

        mask = (np.array(omega_k) >= (omega - rel_error)) & (np.array(omega_k) <= omega + rel_error)
        print(np.array(omega_k)[mask], len(omega_k), omega + rel_error, omega - rel_error)
        print(len(np.array(omega_k)[mask]) * 100 / len(omega_k))
        plt.xlim(omega_k[p.index(max(p))] - 0.006, omega_k[p.index(max(p))] + 0.006)
        if epsilon:
            '''plt.title(r'Probability distribution for the output of phase estimation for $\theta$ = ' + str(
                float('%.2f' % (omega))) + r' with $\epsilon$ =' + str(epsilon) + r'$\rightarrow$ M=' + str(M),
                      fontdict={'family': 'serif',
                                'color': 'darkblue',
                                'weight': 'bold',
                                'size': 8})'''
        else:
            '''plt.title(r'Probability distribution for the output of phase estimation for $\theta$ = ' + str(
                float('%.2f' % (omega))) + r' with M=' + str(M),
                      fontdict={'family': 'serif',
                                'color': 'darkblue',
                                'weight': 'bold',
                                'size': 8})'''
        plt.xlabel(r'$\hat \omega$')
        plt.ylabel("probability")
        plt.legend()

        plt.show()
    if nqubit:
        return omega_tilde, k_est, m, M

    return omega_tilde

def ipe(x, y, epsilon, Q=1, gamma=0.1):
    """Official version of the Robust Inner Product Estimation Routine ((R)IPE).

    Parameters
    ----------
    x: ndarray of shape (n,).
        One of the two vector needed to compute the inner product.

    y: ndarray of shape (n,).
        The second vector needed to compute the inner product.

    epsilon: float value, default=None.
        Precision that you want to have in the inner product estimation procedure.

    Q: int value, default=1.
        Number of iteration for the median evaluation of implicit amplitude estimation output.

    gamma: float value, default=0.1.
        Probability of success to insert in median evaluation.

    Returns
    -------
    s: float value.
        Estimate of the inner product between x and y vectors.

    Notes
    -----
    This method performs the Robust Inner Product Estimation as described in the supplemental material of
    "Quantum algorithms for feedforward neural networks" paper. Implicitly it uses amplitude estimation routine
    :mod:`sklearn.QuantumUtility.Utility.amplitude_estimation` and if the number of iteration Q are >1, implicitly it
    uses also median evaluation to boost the amplitude estimation output.
    """
    a = (np.linalg.norm(x) ** 2 + np.linalg.norm(y) ** 2 - 2 * np.inner(x, y)) / (2 * (
            np.linalg.norm(x) ** 2 + np.linalg.norm(y) ** 2))

    epsilon_a = epsilon * max(1, np.abs(np.inner(x, y))) / (np.linalg.norm(x) ** 2 + np.linalg.norm(y) ** 2)
    if math.isclose(a, 0.0, abs_tol=1e-15):
        a = 0
    a_tilde = amplitude_estimation(a=a, gamma=gamma, epsilon=epsilon_a)
    s = (np.linalg.norm(x) ** 2 + np.linalg.norm(y) ** 2) * (1 - 2 * a_tilde) / 2
    return s

def consistent_phase_estimation(omega, epsilon, gamma, n=None, shift=None):
    """Official version of the Consistent Phase Estimation routine.

    Parameters
    ----------
    omega: float value.
        Value that you want to estimate. It must be a value in the range of [0,1).

    epsilon: float value.
        The accuracy value that you want to have in your estimations.

    gamma: float value.
        Probability error that you want to have.

    n: int value, default=None.
        Number of qubits that you want to use in the routine. If None, this value is computed using the accuracy value
        passed.

    shift: int value, default=None.
        If not None, this fix the shift of the algorithm, otherwise it is computed inside the function.

    Returns
    -------
    estimate: float value.
        Estimate of the omega value.

    Notes
    -----
    This method performs the Consistent Phase Estimation as described in the "Inverting Well Conditioned Matrices in
    Quantum Logspace" paper. Implicitly it uses normal phase estimation routine :mod:`sklearn.QuantumUtility.Utility.phase_estimation`.
    It is useful because different from the normal phase estimation, this is consistent in the sense that it return almost
    always the same result.
    """
    if n == None:
        n = int(np.ceil(np.log2(1 / epsilon)) + np.ceil(np.log2(2 + 1 / (2 * gamma))))

    C = gamma / n
    delta_prime = (epsilon * C) / 2
    L = np.floor(2 / C)
    # shift = random.randint(1, L)
    if shift == None:
        shift = int(L / 2) + 1
    intervals = np.arange(-1 - shift * delta_prime, 1 + epsilon - shift * delta_prime, epsilon)
    intervals = np.append(intervals, 1 + epsilon - shift * delta_prime)
    pe_estimate = phase_estimation(omega=omega, epsilon=delta_prime, gamma=gamma)
    index = bisect(intervals, pe_estimate)
    section = (intervals[(index - 1)], intervals[index])
    estimate = np.mean(section)

    if estimate < 0:
        estimate = 0
    # print('true_value:', omega, 'pe_estimate:', pe_estimate, 'consistent_pe_estimate:', estimate)
    return estimate


from Config import api_config as api
import requests
import json
import csv
from datetime import datetime

class getData:
    def __init__(self):
        pass

    def getApiResponseData(self):
        try:
            # Configure api url from api.config.py file.
            url = api.api_config["url"] + "?" + "resource_id=" +  api.api_config["resource_id"] + "&limit=" + str(api.api_config["limit"])
            print("Loading api url.")
            # Getting response from api using GET method.
            res = requests.get(url)
            print("Getting response from api.")
            # Checking response code if returning 200 then will go for further opertions.
            if res.status_code == 200:
                print("Got ok response from api.")
                # Loading json response data.
                jsondata = json.loads(res.content.decode('UTF-8'))
                print("Loading response data in json.")
                # We will open a file for writing.
                datafile = open("Files/Output_file.csv", 'w', newline='')
                print("Opening a file for writing.")
                # Create csv writer object.
                csv_wrt = csv.writer(datafile)
                print("Create csv writer object.")
                count = 0
                # Filtering out the json response data through loop.
                for data in jsondata['result']['records']:
                    if count == 0:
                        # Loading header for CSV file.
                        header = data.keys()
                        # Writing header of CSV file.
                        csv_wrt.writerow(header)
                        count +=1
                    # Writing data of CSV file
                    csv_wrt.writerow(data.values())
                # Colsing the opened file.
                datafile.close()
                print('completed')
        except Exception as e:
            raise e


from datetime import datetime
from database import Database
from itertools import groupby
from helper import Helper

class Chart:
    def __init__(self,sensors,rows,start_date,end_date,main_type,variable,opt):
        self.helper = Helper()
        self.options = {}
        self.options['tick'] = self.calculate_tickness(start_date,end_date);
        self.rows = rows
        self.main_type = main_type
        self.variable = variable
        self.opt = opt
        self.database = Database()
        self.sensors = sensors
        self.options['sensor_ids'] = self.helper.pluck(sensors,'id')
        self.title = self.resolve_title(sensors[0])
        self.options['title'] = self.title

    def calculate_tickness(self,start_date,end_date):
        difference = self.string_to_datetime(end_date) - self.string_to_datetime(start_date)
        difference_in_seconds = difference.total_seconds()
        hours = divmod(difference_in_seconds,3600)[0]
        if hours < 24 :
            return 1000 * 60 * 60
        else :
            return 1000 * 60 * 60 * 24

    def string_to_datetime(self,date):
        return datetime.strptime(date,'%Y-%m-%d %H:%M:%S')

    def resolve_title(self, sensor) :
        if self.variable == 'stream' :
            return 'Corriente'
        elif self.variable == 'power':
            if sensor['name'] == 'P1':
                return 'Potencia líneas'
            else:
                return f"Potencia {sensor['name']}"
        else :
            if self.main_type == 'LL' :
                return 'Tensión LL'
            else :
                return 'Tensión LN'

    def make_chart_data(self):
        self.make_series()
        self.get_y_axis()

    def make_series(self):
        series = []
        i = 0

        self.sensors =  sorted(self.sensors, key=self.sensor_key_func)
        for key, sensor in groupby(self.sensors, self.sensor_key_func):
            ss = list(sensor)
            series.append({
                'name'  : f"{ss[0]['name']} ({ss[0]['disposition'] != None and ss[0]['disposition']['unit'] or 'N/A'})",
                'data' : self.get_points(ss),
                'turboThreshold' : 0,
                'type' :'spline',
                'zIndex' : (i + 100)
            })
            i+=1

        self.options['series'] = series

    def sensor_key_func(self,k):
        return k['name']

    def point_key_func(self,k):
        return k['grouper']

    def get_points(self,sensors):
        values = []
        for sensor in sensors:
            for value in list(map(self.map_point,filter(lambda row: row['sensor_id'] == sensor['id'], self.rows))):
                values.append(value)

        pp = []
        for key, points in groupby(sorted(values, key=self.point_key_func), self.point_key_func):
            pp.append(self.format_point(points))
        return pp

    def map_point(self,row):
        return {
            'grouper' : row['date'].strftime('%Y-%m-%d %H:%M:%S'),
            'date' : row['date'],
            'result' : row['result']
        }

    def format_point(self,rows):
        rows = list(rows)
        row = rows[0]
        return {
            'x' : int(row['date'].strftime('%s'))*1000,
            'y' : self.calculate_avg(rows),
            'name' : row['date'].strftime('%Y-%m-%d %H:%M:%S')
        }

    def calculate_avg(self,rows):
        avg = 0
        for row in rows:
            avg = avg + row['result']
        return avg / float(len(rows))

    def get_y_axis(self):
        self.options['yAxis'] = [{
                'title' :{
                    'text' : self.title
                },
                'stacklabels' : {
                    'enabled' : True,
                    'style' : {
                        'fontWeight' : 'bold',
                        'color' : 'gray'
                    }
                },
                'opposite' : False
            }]


import pygame
import Levels
from random import randint
import sys

class GameClass:
    def __init__(self):
        self.WIDTH = 1000  # ширина игрового окна
        self.HEIGHT = 800  # высота игрового окна
        self.FPS = 60  # частота кадров в секунду
        # создаем игру и окно
        pygame.init()
        pygame.font.init()  # для текста!
        pygame.mixer.init()  # для звука
        pygame.mixer.music.load('Sources/Sounds/PPK - Ressurection .wav')
        pygame.mixer.music.set_volume(0.3)
        pygame.mixer.music.play(-1)
        self.sound_of_touch = pygame.mixer.Sound('Sources/Sounds/Запись.wav')
        self.org_screen = pygame.display.set_mode((self.WIDTH, self.HEIGHT))
        self.screen = self.org_screen.copy()
        self.delta_x = 0
        self.delta_y = 0
        self.shake_ticks = 0
        pygame.display.set_caption("DMD907")
        self.clock = pygame.time.Clock()
        self.PURPURN = (0, 0, 100)
        self.nlo_img = pygame.image.load('Sources/Images/nlo.png').convert_alpha()
        self.bullet_img = pygame.image.load('Sources/Images/bullet.png').convert_alpha()
        self.background_image = pygame.image.load('Sources/Images/фон.jpg').convert_alpha()
        self.background_image = pygame.transform.scale(self.background_image, (self.WIDTH, self.HEIGHT))
        self.is_running = True

        file = open('score.data', 'r+')
        self.score = [0] * 3
        for i, line in enumerate(file):
            self.score[i] = float(line)
        file.close()

        self.objects = []
        self.objects_dict = {}
        self.scripts = []
        self.scripts_dict = {}
        self.handlers = []
        self.handlers_dict = {}

        self.current_level = Levels.MenuLevel(self)

    def add_object(self, obj, name: str = ''):
        self.objects.append(obj)
        if name:
            self.objects_dict[name] = obj

    def add_handler(self, handler, name: str = ''):
        self.handlers.append(handler)
        if name:
            self.handlers_dict[name] = handler

    def add_script(self, script, name: str = ''):
        self.scripts.append(script)
        if name:
            self.scripts_dict[name] = script

    def fps(self):
        self.clock.tick(self.FPS)

    def run_scripts(self):
        for script in self.scripts:
            if script.run():
                return
        for o in self.objects:
            if o.run_scripts():
                return

    def events(self):
        all_events = pygame.event.get()
        # все события за текущий кадр
        for handler in self.handlers:
            handler.process(all_events)

    def update(self):
        for o in self.objects:
            if o.update():
                break

    def draw(self):
        self.screen.fill(self.PURPURN)
        self.screen.blit(self.background_image, (0, 0))

        for o in self.objects:
            o.draw()
        if self.shake_ticks > 0:
            self.shake_ticks -= 1
            if self.shake_ticks % 5 == 0:
                self.delta_x = randint(-10, 10)
                self.delta_y = randint(-10, 10)
        else:
            self.delta_x = 0
            self.delta_y = 0
        self.org_screen.blit(self.screen, (self.delta_x, self.delta_y))

    def start(self):
        while self.is_running:
            self.fps()
            self.run_scripts()
            self.events()
            self.update()
            self.draw()
            pygame.display.flip()

    def clear(self):
        self.objects.clear()
        self.objects_dict.clear()
        self.handlers.clear()
        self.handlers_dict.clear()
        self.scripts.clear()
        self.scripts_dict.clear()


import logging
from six import iteritems, string_types
from spynnaker.pyNN.exceptions import InvalidParameterType
from spynnaker.pyNN.models.pynn_population_common import PyNNPopulationCommon
from spynnaker.pyNN.utilities.constants import SPIKES
from spinn_front_end_common.utilities import globals_variables
from spinn_front_end_common.utilities.exceptions import ConfigurationException

from spynnaker8.models.recorder import Recorder
from spynnaker8.models.populations import IDMixin, PopulationBase
from spynnaker8.models.populations.population_view import PopulationView
from spynnaker8.utilities import DataHolder

from pyNN import descriptions

logger = logging.getLogger(__name__)

class Population(PyNNPopulationCommon, Recorder, PopulationBase):
    """ PyNN 0.8/0.9 population object
    """

    def __init__(self, size, cellclass, cellparams=None, structure=None,
                 initial_values=None, label=None):
        # pylint: disable=too-many-arguments
        size = self._roundsize(size, label)

        # hard code initial values as required
        if initial_values is None:
            initial_values = {}

        if isinstance(cellclass, DataHolder):
            self._vertex_holder = cellclass
            self._vertex_holder.add_item(
                'label', self.create_label(
                    self._vertex_holder.data_items['label'], label))
            assert cellparams is None
        # cellparams being retained for backwards compatibility, but use
        # is deprecated
        elif issubclass(cellclass, DataHolder):
            if cellparams is None:
                internal_params = dict()
            else:
                internal_params = dict(cellparams)
            cell_label = None
            if 'label' in internal_params:
                cell_label = internal_params['label']
            internal_params['label'] = self.create_label(cell_label, label)
            self._vertex_holder = cellclass(**internal_params)
            # emit deprecation warning
        else:
            raise TypeError(
                "cellclass must be an instance or subclass of BaseCellType,"
                " not a %s" % type(cellclass))

        if 'n_neurons' in self._vertex_holder.data_items:
            if size is None:
                size = self._vertex_holder.data_items['n_neurons']
            elif size != self._vertex_holder.data_items['n_neurons']:
                raise ConfigurationException(
                    "Size parameter is {} but the {} expects a size of {}"
                    "".format(size, cellclass,
                              self._vertex_holder.data_items['n_neurons']))
        else:
            if size is None:
                raise ConfigurationException(
                    "Size parameter can not be None for {}".format(cellclass))
            self._vertex_holder.add_item('n_neurons', size)

        # convert between data holder and model (uses ** so that its taken
        # the dictionary to be the parameters themselves)
        vertex = self._vertex_holder.build_model()(
            **self._vertex_holder.data_items)

        # build our initial objects
        super(Population, self).__init__(
            spinnaker_control=globals_variables.get_simulator(),
            size=size, vertex=vertex,
            structure=structure, initial_values=initial_values)
        Recorder.__init__(self, population=self)

        # annotations used by neo objects
        self._annotations = dict()

    def __iter__(self):
        """ Iterate over local cells
        """
        for _id in range(self._size):
            yield IDMixin(self, _id)

    def __getitem__(self, index_or_slice):
        if isinstance(index_or_slice, int):
            return IDMixin(self, index_or_slice)
        else:
            return PopulationView(
                self, index_or_slice, label="view over {}".format(self.label))

    def all(self):
        """ Iterator over cell IDs on all MPI nodes."""
        for _id in range(self._size):
            yield IDMixin(self, _id)

    @property
    def annotations(self):
        """ The annotations given by the end user
        """
        return self._annotations

    @property
    def celltype(self):
        """ Implements the PyNN expected celltype property

        :return: The celltype this property has been set to
        """
        return self._vertex_holder

    def can_record(self, variable):
        """ Determine whether `variable` can be recorded from this population.
        """
        return variable in self._get_all_possible_recordable_variables()

    def record(self, variables, to_file=None, sampling_interval=None,
               indexes=None):
        """ Record the specified variable or variables for all cells in the\
            Population or view.

        :param variables: either a single variable name or a list of variable\
            names. For a given celltype class, `celltype.recordable` contains\
            a list of variables that can be recorded for that celltype.
        :type variables: str or list(str)
        :param to_file: a file to automatically record to (optional).\
            `write_data()` will be automatically called when `end()` is called.
        :type to_file: a Neo IO instance
        :param sampling_interval: a value in milliseconds, and an integer\
            multiple of the simulation timestep.
        """
        if indexes is not None:
            logger.warn(
                "record indexes parameter is non-standard PyNN, so may not "
                "be portable to other simulators. "
                "It is now deprecated and replaced with views")
        self._record_with_indexes(
            variables, to_file, sampling_interval, indexes)

    def _record_with_indexes(
            self, variables, to_file, sampling_interval, indexes):
        """ Same as record but without non-standard PyNN warning

        This method is non-standard PyNN and is intended only to be called by\
        record in a Population, View or Assembly
        """
        if variables is None:  # reset the list of things to record
            if sampling_interval is not None:
                raise ConfigurationException(
                    "Clash between parameters in record."
                    "variables=None turns off recording,"
                    "while sampling_interval!=None implies turn on recording")
            if indexes is not None:
                logger.warning(
                    "View.record with variable None is non-standard PyNN. "
                    "Only the neurons in the view have their record turned "
                    "off. Other neurons already set to record will remain "
                    "set to record")

            # note that if record(None) is called, its a reset
            Recorder._turn_off_all_recording(self, indexes)
            # handle one element vs many elements
        elif isinstance(variables, string_types):
            # handle special case of 'all'
            if variables == "all":
                logger.warning(
                    "This is non-standard PyNN, and therefore may not be "
                    "portable to other simulators.")

                # get all possible recordings for this vertex
                variables = self._get_all_possible_recordable_variables()

                # iterate though them
                for variable in variables:
                    self._record(variable, sampling_interval, to_file, indexes)
            else:
                # record variable
                self._record(variables, sampling_interval, to_file, indexes)

        else:  # list of variables, so just iterate though them
            for variable in variables:
                self._record(variable, sampling_interval, to_file, indexes)

    def write_data(self, io, variables='all', gather=True, clear=False,
                   annotations=None):
        """ Write recorded data to file, using one of the file formats\
            supported by Neo.

        :param io: \
            a Neo IO instance, or a string for where to put a neo instance
        :type io: neo instance or str
        :param variables: \
            either a single variable name or a list of variable names.\
            Variables must have been previously recorded, otherwise an\
            Exception will be raised.
        :type variables: str or list(str)
        :param gather: pointless on sPyNNaker
        :param clear: \
            clears the storage data if set to true after reading it back
        :param annotations: annotations to put on the neo block
        """
        # pylint: disable=too-many-arguments
        if not gather:
            logger.warning("sPyNNaker only supports gather=True. We will run "
                           "as if gather was set to True.")

        if isinstance(io, string_types):
            io = self._get_io(io)

        data = self._extract_neo_block(variables, None, clear, annotations)
        # write the neo block to the file
        io.write(data)

    def describe(self, template='population_default.txt', engine='default'):
        """ Returns a human-readable description of the population.

        The output may be customized by specifying a different template\
        together with an associated template engine (see\
        :mod:`pyNN.descriptions`).

        If `template` is None, then a dictionary containing the template\
        context will be returned.
        """
        vertex_context = self._vertex.describe()

        context = {
            "label": self.label,
            "celltype": vertex_context,
            "structure": None,
            "size": self.size,
            "size_local": self.size,
            "first_id": self._first_id,
            "last_id": self._last_id,
        }
        context.update(self._annotations)
        if self.size > 0:
            context.update({
                "local_first_id": self._first_id,
                "cell_parameters": {}})
        if self._structure:
            context["structure"] = self._structure.describe(template=None)
        return descriptions.render(engine, template, context)

    def _end(self):
        """ Do final steps at the end of the simulation
        """
        for variable in self._write_to_files_indicators:
            if self._write_to_files_indicators[variable] is not None:
                self.write_data(
                    io=self._write_to_files_indicators[variable],
                    variables=[variable])

    def get_data(
            self, variables='all', gather=True, clear=False, annotations=None):
        """ Return a Neo `Block` containing the data\
            (spikes, state variables) recorded from the Assembly.

        :param variables: either a single variable name or a list of variable\
            names. Variables must have been previously recorded, otherwise an
            Exception will be raised.
        :type variables: str or list
        :param: Whether to collect data from all MPI nodes or just the current\
            node (irrelevant on sPyNNaker, which always behaves as if True)
        :type gather: bool
        :param: Whether recorded data will be deleted from the `Assembly`.
        :type clear: bool
        :param annotations: annotations to put on the neo block
        :type annotations: dict
        :rtype: neo.Block
        """
        if not gather:
            logger.warning("sPyNNaker only supports gather=True. We will run "
                           "as if gather was set to True.")

        return self._extract_neo_block(variables, None, clear, annotations)

    def get_data_by_indexes(
            self, variables, indexes, clear=False, annotations=None):
        """ Return a Neo `Block` containing the data\
            (spikes, state variables) recorded from the Assembly.

        :param variables: either a single variable name or a list of variable\
            names. Variables must have been previously recorded, otherwise an
            Exception will be raised.
        :type variables: str or list
        :param indexes: List of neuron indexes to include in the data.
            Clearly only neurons recording will actually have any data
            If None will be taken as all recording as get_data
        :type indexes: list (int)
        :param clear: Whether recorded data will be deleted.
        :type clear: bool
        :param annotations: annotations to put on the neo block
        :type annotations: dict
        :rtype: neo.Block
        """
        return self._extract_neo_block(variables, indexes, clear, annotations)

    def spinnaker_get_data(self, variable):
        """ Public accessor for getting data as a numpy array, instead of\
            the neo based object

        :param variable: \
            either a single variable name or a list of variable names.\
            Variables must have been previously recorded, otherwise an\
            Exception will be raised.
        :return: numpy array of the data
        """
        logger.warning(
            "This call is non-standard PyNN and therefore may not be "
            "portable to other simulators. Nor do we guarantee that this "
            "function will exist in future releases.")
        if isinstance(variable, list):
            if len(variable) == 1:
                variable = variable[0]
            else:
                msg = "Only one type of data at a time is supported"
                raise ConfigurationException(msg)
        if variable == SPIKES:
            return self._get_spikes()
        return self._get_recorded_pynn7(variable)

    def get_spike_counts(self, gather=True):
        """ Return the number of spikes for each neuron.
        """
        spikes = self._get_spikes()
        return PyNNPopulationCommon.get_spike_counts(self, spikes, gather)

    def find_units(self, variable):
        """ Get the units of a variable

        :param variable: The name of the variable
        :return: The units of the variable
        """
        return self._get_variable_unit(variable)

    def set(self, **kwargs):
        for parameter, value in iteritems(kwargs):
            try:
                super(Population, self).set(parameter, value)
            except InvalidParameterType:
                super(Population, self).initialize(parameter, value)

    def initialize(self, **kwargs):
        for parameter, value in iteritems(kwargs):
            super(Population, self).initialize(parameter, value)

    @property
    def initial_values(self):
        if not self._vertex_population_initializable:
            raise KeyError(
                "Population does not support the initialisation")
        return self._vertex.initial_values

    # NON-PYNN API CALL
    def get_initial_value(self, variable, selector=None):
        """ See AbstractPopulationInitializable.get_initial_value
        """
        if not self._vertex_population_initializable:
            raise KeyError(
                "Population does not support the initialisation of {}".format(
                    variable))
        return self._vertex.get_initial_value(variable, selector)

    # NON-PYNN API CALL
    def set_initial_value(self, variable, value, selector=None):
        """ See AbstractPopulationInitializable.set_initial_value
        """
        if not self._vertex_population_initializable:
            raise KeyError(
                "Population does not support the initialisation of {}".format(
                    variable))
        if globals_variables.get_not_running_simulator().has_ran \
                and not self._vertex_changeable_after_run:
            raise Exception("Population does not support changes after run")
        self._vertex.set_initial_value(variable, value, selector)

    # NON-PYNN API CALL
    def get_initial_values(self, selector=None):
        """ See AbstractPopulationInitializable.get_initial_values
        """
        if not self._vertex_population_initializable:
            raise KeyError("Population does not support the initialisation")
        return self._vertex.get_initial_values(selector)

    def get(self, parameter_names, gather=False, simplify=True):
        if simplify is not True:
            logger.warning("The simplify value is ignored if not set to true")
        return super(Population, self).get(parameter_names, gather)

    @property
    def all_cells(self):
        cells = []
        for _id in range(self._size):
            cells.append(IDMixin(self, _id))
        return cells

    @property
    def position_generator(self):
        def gen(i):
            return self.positions[:, i]
        return gen

    @staticmethod
    def create(cellclass, cellparams=None, n=1):
        """ Pass through method to the constructor defined by PyNN.\
        Create n cells all of the same type.\
        Returns a Population object.

        :param cellclass: see Population.__init__
        :param cellparams: see Population.__init__
        :param n: see Population.__init__(size...)
        :return: A New Population
        """
        return Population(size=n, cellclass=cellclass, cellparams=cellparams)