import pandas as pd
import os
import pickle
import numpy as np
from collections.abc import Iterable
from scipy.stats import norm
from .base import _BaseSumStats, _H5SSConnection
from .utils import _MultiWindowPlot, _recompute_maf
from .plot import (manhattan, qqplot, afplot, zzplot, pzplot)
from .exceptions import sumstatswarn

pd.options.mode.chained_assignment = None

class SumStats(_BaseSumStats):
    """Class for summary statistics of a single GWAS.

    :param path: Path to the file containing summary statistics. Should be a csv, or tab-delimited txt-file (.gz supported).
    :type path: str
    :param phenotype: Phenotype name
    :type phenotype: str
    :param gwas_n: Optional N subjects in the GWAS, for N-based meta analysis (if there is no N column in the summary statistics)
    :type gwas_n: int
    :param column_names: Optional dictionary of column names, if these are not automatically recognised. Keys should be: ['rsid', 'chr', 'bp', 'ea', 'oa', 'maf', 'b', 'se', 'p', 'hwe', 'info', 'n', 'eaf', 'oaf']
    :type column_names: dict
    :param data: Dataset for the new SumStats object, in general, don't specify this.
    :type data: dict
    :param low_ram: Whether to use the low_ram option for this SumStats object. Use this only when running into MemoryErrors. Enabling this option will read/write data from local storage rather then RAM. It will save lots of RAM, but it will significantly decrease processing speed.
    :type low_ram: bool
    :param tmpdir: Which directory to store the temporary files if low_ram is enabled.
    :type tmpdir: str
    :param kwargs: other keyword arguments to be passed to pandas.read_csv() method

    """

    def __init__(self, path, phenotype=None, gwas_n=None, column_names=None, data=None, low_ram=False,
                 tmpdir='sumstats_temporary', **kwargs):
        assert isinstance(path, str) or (path is None),  "path should be str"
        assert isinstance(phenotype, str) or (phenotype is None), "phenotype should be str"
        assert isinstance(gwas_n, int) or gwas_n is None, "gwas_n should be int"
        assert isinstance(low_ram, bool), "low_ram should be True or False"
        assert isinstance(tmpdir, str), "tmpdir should be str"
        super().__init__()
        self.low_ram = low_ram
        self.tmpdir = tmpdir
        self.plot_funcs = {'manhattan': self.manhattan, 'qqplot': self.qqplot, 'pzplot': self.pzplot}
        if phenotype is None:
            self.phenotype_name = path.split('/')[-1].split('.')[0]
        else:
            self.phenotype_name = phenotype
        self.column_names = column_names
        if path is not None:
            if path.endswith('.pickle'):
                with open(path, 'rb') as f:
                    sumstatsobj = pickle.load(f)
                if sumstatsobj.low_ram:
                    if not os.path.isfile(sumstatsobj.data.path):
                        raise ImportError(
                            'The data that was stored in {} using low_ram does not exist anymore.'.format(path))
                if self.__version__ != sumstatsobj.__version__:
                    if self.__version__.split('.')[0] != sumstatsobj.__version__.split('.')[0]:
                        pass  # What to do in case of major version differences: No problem yet.
                    else:
                        sumstatswarn('Imported sumstats version {} does not match current version {}'.format(
                            sumstatsobj.__version__, self.__version__))
                for k, v in {m: getattr(sumstatsobj, m) for m in dir(sumstatsobj) if
                             (not m.startswith('__'))}.items():
                    setattr(self, k, v)
                self.__version__ = sumstatsobj.__version__
            else:
                if 'sep' not in kwargs.keys():
                    if path.endswith('.txt') or path.endswith('.txt.gz') or path.endswith('.tsv') or path.endswith('.tsv.gz'):
                            self.data = pd.read_csv(path, sep='\t', **kwargs)
                    elif path.endswith('.csv') or path.endswith('.csv.gz'):
                        self.data = pd.read_csv(path, **kwargs)
                    else:
                        raise ImportError('Only .txt(.gz), .tsv(.gz) or .csv(.gz) files allowed when \'sep\' is not specified.')
                else:
                    self.data = pd.read_csv(path, **kwargs)
            self.gwas_n = gwas_n

            self._sync_columns()
            self._split()
        else:
            self.phenotype_name = phenotype
            self.data = data
            self.reset_index()
        self.qc_result = {}
        self.columns = self.data[1].columns

    def _sync_columns(self):
        """Internal function to synchronize column names and types with the defaults.

        :return: None

        """
        self.data.columns = [x.lower() for x in self.data.columns]
        if (len(self.data.columns)) != len(list(set(self.data.columns))):
            raise KeyError('Duplicate column names not allowed! (not case-sensitive)')
        required = ['rsid', 'chr', 'bp', 'ea', 'oa', 'maf', 'b', 'se', 'p']
        column_name_variants = {
            'rsid': ['rsid', 'rs', 'rsnumber'],
            'chr': ['chr', 'chromosome'],
            'bp': ['bp', 'pos'],
            'ea': ['a1', 'ea', 'effectallele', 'allele1', 'ref', 'refallele'],
            'oa': ['a2', 'oa', 'otherallele', 'allele0', 'allele2', 'alt', 'altallele'],
            'maf': ['maf', 'minorallelefrequency', 'minfreq'],
            'b': ['b', 'beta', 'effect'],
            'se': ['se', 'stderr'],
            'p': ['p', 'pval', 'pvalue', 'p_bolt_lmm_inf', 'p-value'],
            'hwe': ['p_hwe', 'hwe', 'hwe_p'],
            'info': ['info', 'rsq', 'r2'],
            'n': ['n', 'nchrobs', 'samplesize', 'totalsamplesize']
        }
        column_name_variants_af = {
            'eaf': ['a1freq', 'a1frq', 'a1f', 'eaf', 'freq1'],
            'oaf': ['a2freq', 'a2frq', 'a2f', 'oaf', 'freq2'],
        }
        column_types = {'rsid': str, 'chr': np.uint8, 'bp': np.uint32, 'ea': str, 'oa': str, 'maf': np.float32,
                        'b': np.float64, 'se': np.float64, 'p': np.float64, 'hwe': np.float32, 'info': np.float32,
                        'n': np.uint32, 'eaf': np.float32, 'oaf': np.float32}
        if self.column_names is not None:
            for k, v in self.column_names:
                if (k not in column_name_variants) and (k not in column_name_variants_af):
                    raise KeyError('{} not recognized as column type'.format(k))
                elif k in column_name_variants:
                    column_name_variants[k] = [v]
                elif k in column_name_variants_af:
                    column_name_variants_af[k] = [v]
        found, found_af, not_found = [], [], []
        for target, options in column_name_variants.items():
            for opt in options:
                if opt in self.data.columns:
                    self.data.rename(columns={opt: target}, inplace=True)
                    found += [target]
                    break
            if (target == 'rsid') and ('rsid' not in found):
                for opt in ['snpname', 'snp', 'snpid', 'cptid', 'markername']:
                    if opt in self.data.columns:
                        self.data.rename(columns={opt: target}, inplace=True)
                        found.append(target)
                        sumstatswarn(
                            'Assuming column {} represents rsid as no other rsid column was found.'.format(opt))
                        break
        if self.data['rsid'].isnull().values.any():
            sumstatswarn('Missing RSIDs were replaced with CHR:BP')
            midx = self.data['rsid'].isnull()
            self.data.loc[midx, 'rsid'] = self.data.loc[midx, 'chr'].map(str) + ':' + self.data.loc[midx, 'bp'].map(str)
        for target, options in column_name_variants_af.items():
            for opt in options:
                if opt in self.data.columns:
                    self.data.rename(columns={opt: target}, inplace=True)
                    found_af.append(target)
        if 'maf' not in found:
            if len(found_af) == 0:
                raise KeyError('No allele frequency column found')
            if ('eaf' in found_af) and ('oaf' not in found_af):
                self.data['oaf'] = 1 - self.data['eaf']
            elif ('eaf' not in found_af) and ('oaf' in found_af):
                self.data['eaf'] = 1 - self.data['oaf']
            self.data['maf'] = self.data[['eaf', 'oaf']].min(axis=1)
            found.append('maf')
        for target in required:
            if target not in found:
                not_found.append(target)
        if 'n' not in self.data.columns:
            if self.gwas_n is None:
                sumstatswarn('No N found or specified, samplesize based meta analysis not possible')
            else:
                self.data['n'] = self.gwas_n
        else:
            if np.any(self.data['n'].isna()):
                nnna = np.sum(self.data['n'].isna())
                sumstatswarn('{} missing values in sample size column were mean imputed'.format(nnna))
                self.data.loc[self.data['n'].isna(), 'n'] = self.data['n'].mean()
        if np.any(self.data['bp'].isna()):
            naidx = np.where(self.data['bp'].isna())
            self.data.loc[naidx[0], 'bp'] = (self.data.loc[naidx[0] - 1, 'bp'] + 1).tolist()
            if np.any(self.data['bp'].isna()):
                raise ValueError('Too many missing values in basepair column')
            else:
                sumstatswarn('{} missing values in basepair column were imputed to be (bp at the previous row)+1'.format(len(naidx)))
        if np.any(self.data['chr'].isna()):
            naidx = np.where(self.data['chr'].isna())
            self.data.loc[naidx[0], 'chr'] = (self.data.loc[naidx[0] - 1, 'chr']).tolist()
            if np.any(self.data['chr'].isna()):
                raise ValueError('Too many missing values in chromosome column')
            else:
                sumstatswarn('{} missing values in chromosome column were imputed to be the same as chromsome of the previous row'.format(len(naidx)))
        try:
            self.data['chr'] = self.data['chr'].astype(int)
        except ValueError:
            self.data['chr'] = self.data['chr'].str.replace('X', '23')
            try:
                self.data['chr'] = self.data['chr'].astype(int)
            except ValueError:
                raise ValueError('Could not convert chromosome column to integer')
        if len(not_found) > 0:
            raise KeyError('Could not find columns: {}'.format(', '.join(not_found)))
        self.data['ea'] = self.data['ea'].str.upper()
        self.data['oa'] = self.data['oa'].str.upper()
        self.variables = list(self.data.columns)
        for column, targettype in column_types.items():
            if column in self.data.columns:
                self.data[column].astype(targettype, copy=False)

    def _split(self):
        """ Internal function to split the initial large dataset into chunks based on chromosome

        :return: None

        """
        if not self.low_ram:
            new_data = {}
        else:
            new_data = _H5SSConnection(self.phenotype_name, tmpdir=self.tmpdir)
        for c in range(1, 24):
            new_data[c] = self.data.loc[self.data['chr'] == c, :].copy()
            if len(new_data[c]) == 0:
                sumstatswarn('No data found for chromosome {}'.format(c))
        self.data = new_data

    def qc(self, maf=None, hwe=None, info=None, **kwargs):
        """Basic GWAS quality control function.

        :param maf: Minor allele frequency cutoff, will drop SNPs where MAF < cutoff. Default: 0.1
        :type maf: float or None
        :param hwe: Hardy-Weinberg Equilibrium cutoff, will drop SNPs where HWE < cutoff, if specified and HWE column is present in the data.
        :type hwe: float or None
        :param info: Imputation quality cutoff, will drop SNPs where Info < cutoff, if specified and Info column is present in the data.
        :type info: float or None
        :param kwargs: Other columns to filter on, keyword should be column name, SNPs whill be dropped where the value < argument.
        :return: None

        """
        assert isinstance(maf, float) or (maf is None), "maf should be float"
        assert isinstance(hwe, float) or (hwe is None), "hwe should be float"
        assert isinstance(info, float) or (info is None), "info should be float"
        qc_vals = dict(maf=.01)
        qc_info = dict(org_len=0, maf=0, new_len=0)
        if maf is not None:
            qc_vals['maf'] = maf
        if hwe is not None:
            if 'hwe' not in self.variables:
                sumstatswarn('HWE qc cutoff specified but HWE column was not found, skipping QC step.')
            else:
                qc_vals['hwe'] = hwe
                qc_info['hwe'] = 0
        if info is not None:
            if 'info' not in self.variables:
                sumstatswarn('Info qc cutoff specified but info column was not found, skipping QC step.')
            else:
                qc_vals['info'] = info
                qc_info['info'] = 0
        for k, v in kwargs.items():
            if k not in self.variables:
                raise KeyError('Column {} not found.'.format(k))
            else:
                qc_vals[k] = v
                qc_info[k] = 0
        for c in range(1, 24):
            data = self.data[c]
            qc_info['org_len'] += len(data)
            for var, cutoff in qc_vals.items():
                qc_info[var] += len(data.loc[data[var] >= cutoff, :])
            for var, cutoff in qc_vals.items():
                data = data.loc[data[var] >= cutoff, :].copy()
            qc_info['new_len'] += len(data)
            self.data[c] = data
        self.qc_result = qc_info

    def merge(self, other, how='inner', low_memory=False):
        """Merge with other SumStats object(s).

        :param other: Other sumstats object, or list of other SumStats objects.
        :type other: :class:`pysumstats.plot.SumStats` or list
        :param how: Type of merge.
        :type how: str
        :param low_memory: Enable to use a more RAM-efficient merging method (WARNING: still untested)
        :type low_memory: bool
        :return: :class:`pysumstats.plot.MergedSumStats` object

        """
        assert isinstance(other, SumStats) or isinstance(other, list), "other should be SumStats or list"
        if isinstance(other, list):
            for o in other:
                assert isinstance(o, SumStats), "items in other should be SumStats"
            merged = self.merge(other[0], how=how, low_memory=low_memory)
            merged.merge(other[1:], how=how, inplace=True, low_memory=low_memory)
            return merged
        else:
            if self.phenotype_name == other.phenotype_name:
                sumstatswarn('Phenotype names were equal, converting to {0}_x and {0}_y'.format(self.phenotype_name))
                self.phenotype_name = self.phenotype_name + '_x'
                other.phenotype_name = other.phenotype_name + '_y'
            merge_info = dict(xlen=0, ylen=0, newlen=0)
            joined = [x for x in self.variables if x in other.variables]
            for x in [x for x in self.variables if x not in joined]:
                sumstatswarn(
                    'Could not find column {} in pysumstats for {}, column dropped'.format(x, other.phenotype_name))
            for x in [x for x in other.variables if x not in joined]:
                sumstatswarn(
                    'Could not find column {} in pysumstats for {}, column dropped'.format(x, self.phenotype_name))
            if not self.low_ram:
                merged_data = {}
            else:
                merged_data = _H5SSConnection(
                    '{}{}{}{}{}'.format(self.phenotype_name[0], self.phenotype_name[-1], other.phenotype_name[0],
                                        other.phenotype_name[1], len(os.listdir('sumstats_temporary'))),
                    tmpdir=self.tmpdir)
            if 'n' not in self.columns:
                sumstatswarn('No sample size column found for {}, using 1 for calculation of overall MAF'.format(self.phenotype_name))
            if 'n' not in other.columns:
                sumstatswarn('No sample size column found for {}, using 1 for calculation of overall MAF'.format(other.phenotype_name))
            if low_memory and (not how == 'inner'):
                raise NotImplementedError('low_memory only allows for an inner merge.')
            for c in range(1, 24):
                data_x = self.data[c].copy()
                merge_info['xlen'] += len(data_x)
                data_y = other.data[c].copy()
                merge_info['ylen'] += len(data_y)
                data_m_x = data_x.rename(columns={k: '{}_{}'.format(k, self.phenotype_name) for k in data_x.columns if
                                                  k not in ['rsid', 'chr', 'bp']})
                data_m_y = data_y.rename(columns={k: '{}_{}'.format(k, other.phenotype_name) for k in data_y.columns if
                                                  k not in ['rsid', 'chr', 'bp']})
                if low_memory:
                    if np.any(data_m_x.duplicated(subset='rsid')) or np.any(data_m_y.duplicated(subset='rsid')):
                        raise KeyError('Duplicated rsids in either dataset will cause low_memory merge to fail')
                    merged_data[c] = data_m_x.loc[data_m_x['rsid'].isin(data_m_y['rsid'].tolist()), :]
                    data_m_y = data_m_y.loc[data_m_y['rsid'].isin(data_m_x['rsid'].tolist()), :]
                    merged_data[c].sort_values(by='rsid', inplace=True)
                    data_m_y.sort_values(by='rsid', inplace=True)
                    merged_data[c].reset_index(inplace=True, drop=True)
                    data_m_y.reset_index(inplace=True, drop=True)
                    data_m_y = data_m_y[
                        [x + '_' + other.phenotype_name for x in joined if x not in ['rsid', 'chr', 'bp']]]
                    for x in data_m_y.columns:
                        merged_data[c][x] = data_m_y[x]
                        data_m_y.drop(axis=1, labels=[x], inplace=True)
                    merged_data[c].sort_values(by='bp', inplace=True)
                else:
                    joined_x = ['rsid', 'chr', 'bp'] + [x + '_' + self.phenotype_name for x in joined if
                                                        x not in ['rsid', 'chr', 'bp']]
                    joined_y = ['rsid', 'chr_y', 'bp_y'] + [x + '_' + other.phenotype_name for x in joined if
                                           x not in ['rsid', 'chr', 'bp']]
                    data_m_y = data_m_y.rename(columns={'chr': 'chr_y', 'bp': 'bp_y'})
                    merged_data[c] = data_m_x[joined_x].merge(data_m_y[joined_y], on='rsid', how=how)
                    merged_data[c].loc[merged_data[c]['chr'].isna(), 'chr'] = merged_data[c].loc[merged_data[c]['chr'].isna(), 'chr_y']
                    merged_data[c].loc[merged_data[c]['bp'].isna(), 'bp'] = merged_data[c].loc[
                        merged_data[c]['bp'].isna(), 'bp_y']
                    merged_data[c] = merged_data[c].drop(axis=1, labels=['chr_y', 'bp_y'])
                merged_data[c]['maf'] = _recompute_maf(merged_data[c], [self.phenotype_name, other.phenotype_name])
                merge_info['newlen'] += len(merged_data[c])
            return MergedSumStats(data=merged_data, phenotypes=[self.phenotype_name, other.phenotype_name],
                                  merge_info=merge_info, variables=joined,
                                  xy=[self.phenotype_name, other.phenotype_name], low_ram=self.low_ram,
                                  tmpdir=self.tmpdir)

    def describe(self, columns=None, per_chromosome=False):
        """Get a summary of the data.

        :param columns: List of column names to print summary for (default: ['b', 'se', 'p'])
        :type columns: list.
        :param per_chromosome: Enable to return a list of summary dataframes per chromosome
        :type per_chromosome: bool.
        :return: pd.Dataframe, or list

        """
        assert isinstance(columns, list) or (columns is None), "columns should be a list"
        assert isinstance(per_chromosome, bool), "per_chromosome should be True or False"
        if columns is None:
            columns = ['b', 'se', 'p']
        if (not isinstance(columns, list)) and isinstance(columns, str):
            columns = [columns]
        sum_cols = []
        for c in columns:
            if c in self.data[1].columns:
                sum_cols.append(c)
            else:
                sumstatswarn('{} not found in columns, skipping.'.format(c))
        return self._get_summary(sum_cols, per_chromosome)

    def manhattan(self, **kwargs):
        """Generate a manhattan plot using this sumstats data

        :param kwargs: keyworded arguments to be passed to :func:`pysumstats.plot.manhattan`
        :return: None, or (fig, ax)

        """
        manhattan(self[['rsid', 'chr', 'bp', 'p']], **kwargs)

    def qqplot(self, **kwargs):
        """Generate a QQ-plot using this sumstats data

        :param kwargs: keyworded arguments to be passed to :func:`pysumstats.plot.qqplot`
        :return: None, or (fig, ax)

        """
        qqplot(self['p'].values, **kwargs)

    def pzplot(self, **kwargs):
        """Generate a PZ-plot using this sumstats data

        :param kwargs: keyworded arguments to be passed to :func:`pysumstats.plot.pzplot`
        :return: None, or (fig, ax)

        """
        pzplot(self[['b', 'se', 'p']], **kwargs)

class MergedSumStats(_BaseSumStats):
    """Class containing merged summary statistics. In general you will not create a MergedSumStats object manually.

    :param data: dataset containing merged summary statistics
    :type data: dict
    :param phenotypes: list of phenotype names.
    :type phenotypes: list
    :param merge_info: Dict with information on the merge
    :type merge_info: dict
    :param variables: list of variables contained in the data.
    :type variables: list
    :param xy: x and y suffixes (to be used in _allign)
    :type xy: list
    :param low_ram: Whether to use the low_ram option for this MergedSumStats object (passed down from SumStats). Use this only when running into MemoryErrors. Enabling this option will read/write data from local storage rather then RAM. It will save lots of RAM, but it will gratly decrease processing speed.
    :type low_ram: bool
    :param tmpdir: Which directory to store the temporary files if low_ram is enabled (passed down from SumStats).
    :type tmpdir: str
    :param allign: Enable to auto-allign SNPs
    :type allign: bool

    """

    def __init__(self, data, phenotypes, merge_info, variables, xy, low_ram=False, tmpdir='sumstats_temporary',
                 allign=True):
        super().__init__()
        self.pheno_names = phenotypes
        self.data = data
        self.info = merge_info
        self.variables = variables
        self.suffixes = xy
        self.columns = self.data[1].columns
        if allign:
            self._allign()
        self.reset_index()
        self.phenotype_name = None
        self.low_ram = low_ram
        self.tmpdir = tmpdir
        self.plot_funcs = {'manhattan': self.manhattan, 'qqplot': self.qqplot, 'pzplot': self.pzplot,
                           'afplot': self.afplot, 'zzplot': self.zzplot}

    def _allign(self, ynames=None):
        """Function to allign SNPs to the first phenotype.

        :param ynames: Optional argument of multiple phenotypes that should be alligned.
        :type ynames: list
        :return: None

        """
        if self.suffixes[0] is None:
            suffix_x = ''
        else:
            suffix_x = '_' + self.suffixes[0]
        if self.suffixes[1] is None:
            suffix_y = ''
        else:
            suffix_y = '_' + self.suffixes[1]
        eax, oax = 'ea{}'.format(suffix_x), 'oa{}'.format(suffix_x)
        if ynames is None:
            eay, oay, bys = 'ea{}'.format(suffix_y), 'oa{}'.format(suffix_y), ['b{}'.format(suffix_y)]
            eafys, oafys = ['eaf{}'.format(suffix_y)], ['oaf{}'.format(suffix_y)]
        else:
            eay, oay, bys = 'ea{}'.format(suffix_y), 'oa{}'.format(suffix_y), ['b{}'.format(y) for y in ynames]
            eafys, oafys = ['eaf{}'.format(y) for y in ynames], ['oaf{}'.format(y) for y in ynames]
        dropped = 0
        for c in self.data.keys():
            data = self.data[c].copy()
            data.loc[data[eax].isna(), eax] = data.loc[data[eax].isna(), eay]
            data.loc[data[oax].isna(), oax] = data.loc[data[oax].isna(), oay]
            data.loc[data[eay].isna(), eay] = data.loc[data[eay].isna(), eax]
            data.loc[data[oay].isna(), oay] = data.loc[data[oay].isna(), oax]
            org_len = len(data)
            for a in [eax, oax, eay, oay]:
                data[a] = data[a].astype(str)
            flip = data.loc[(data[eax] == data[oay]) & (data[oax] == data[eay]), bys[0]].index
            for by in bys:
                data.loc[flip, by] *= -1
            data.loc[flip, eay] = data.loc[flip, eax]
            data.loc[flip, oay] = data.loc[flip, oax]
            for fy in eafys + oafys:
                if fy in data.columns:
                    data.loc[flip, fy] = abs(1 - data.loc[flip, fy])
            data2 = data.loc[(data[eax] == data[eay]), :].copy()
            data2 = data2.loc[(data2[oax] == data2[oay]), :].copy()
            dropped += (org_len - len(data2))
            data2.drop(axis=1, labels=[eay, oay], inplace=True)
            data2.rename(columns={eax: 'ea', oax: 'oa'}, inplace=True)
            self.data[c] = data2
        self.columns = self.data[1].columns
        if dropped > 0:
            sumstatswarn('Dropped {} SNPs due to allele mismatch'.format(dropped))
        self.suffixes = [None, None]

    def meta_analyze(self, name='meta', method='ivw', debug=False):
        """Meta analyze all GWAS summary statistics contained in this object.
        WARNING: There appears to be an error somewhere in this function that causes incorrect result.
        For now running .meta_analyze() will instead run .gwama() with an identity matrix (functionally identical to an ivw meta_analysis)
        :param name: New phenotype name to use for the new SumStats object (default: 'meta')
        :type name: str
        :param method: Meta-analysis method to use, should be one of ['ivw', 'samplesize'], default: 'ivw'
        :type method: str
        :param debug: Run the meta_analyze function instead of .gwama() for debugging purposes
        :type debug: bool
        :return: :class:`pysumstats.SumStats` object.

        """
        assert isinstance(name, str), "name should be str"
        if not self.low_ram:
            new_data = {}
        else:
            new_data = _H5SSConnection(name, tmpdir=self.tmpdir)
        if method not in ['ivw', 'samplesize']:
            raise KeyError('method should be one of [\'ivw\', \'samplesize\']')
        if not debug:
            sumstatswarn("The meta_analyze function is currently bugged. .gwama() with an identity matrix is now performed instead.")
            id_mat = pd.DataFrame(np.identity(len(self.pheno_names)), columns=self.pheno_names, index=self.pheno_names)
            return self.gwama(cov_matrix=id_mat, name=name)
        else:
            sumstatswarn("Warning: Please do not use the results of this function without extensive verification!")
            columns = self.data[1].columns
            missing_n = []
            for p in self.pheno_names:
                if 'n_{}'.format(p) not in columns:
                    missing_n.append(p)
            if len(missing_n) > 0:
                if method == 'samplesize':
                    raise KeyError('Missing sample size column for {}.'.format(', '.join(missing_n)))
                else:
                    sumstatswarn('Missing sample size column for {}, output N will be incorrect.'.format(', '.join(missing_n)))
            for c in self.data.keys():
                data = self.data[c]

                n_dat = data[['rsid', 'chr', 'bp', 'ea', 'oa', 'maf']]
                if method == 'samplesize':
                    for p in self.pheno_names:
                        data.loc[data['b_{}'.format(p)].isna(), 'b_{}'.format(p)] = 0
                        data.loc[data['se_{}'.format(p)].isna(), 'se_{}'.format(p)] = 1
                        data.loc[:, 'z_{}'.format(p)] = data['b_{}'.format(p)] / data['se_{}'.format(p)]
                        data.loc[:, 'w_{}'.format(p)] = np.sqrt(data['n_{}'.format(p)])
                        data.loc[:, 'z_{}'.format(p)] = data['z_{}'.format(p)] * data['w_'.format(p)]
                    zsums = data[['z_'.format(x) for x in self.pheno_names]].sum(axis=1)
                    wsums = data[['w_'.format(x) for x in self.pheno_names]].sum(axis=1)
                    n_dat.loc[:, 'z'] = zsums / np.sqrt(np.sum(wsums ** 2))
                else:
                    for p in self.pheno_names:
                        data.loc[:, 'w_{}'.format(p)] = 1 / (data.loc[:, 'se_{}'.format(p)] ** 2)
                        data.loc[:, 'bw_{}'.format(p)] = data.loc[:, 'b_{}'.format(p)] * data.loc[:, 'w_{}'.format(p)]
                    bwsums = data[['bw_{}'.format(x) for x in self.pheno_names]].sum(axis=1)
                    wsums = data[['w_{}'.format(x) for x in self.pheno_names]].sum(axis=1)
                    n_dat.loc[:, 'se'] = np.sqrt(1 / wsums)
                    n_dat.loc[:, 'b'] = bwsums / wsums
                    n_dat.loc[:, 'z'] = n_dat.loc[:, 'b'] / n_dat.loc[:, 'se']
                n_dat.loc[:, 'p'] = norm.sf(abs(n_dat.loc[:, 'z'])) * 2
                n_dat['n'] = 0
                for p in self.pheno_names:
                    if 'n_{}'.format(p) in data.columns:
                        n_dat['n'] += data['n_{}'.format(p)]
                new_data[c] = n_dat
            new_columns = list(new_data[c].columns)
            new_data = SumStats(path=None, phenotype=name, data=new_data)
            new_data.variables = new_columns
            return new_data

    def gwama(self, cov_matrix=None, h2_snp=None, name='gwama'):
        """Multivariate meta analysis as described in Baselmans, et al. 2019.

        :param cov_matrix: Covariance matrix, defaults to generating a correlation matrix of Z-scores
        :type cov_matrix: pd.Dataframe
        :param h2_snp: Dict of SNP heritabilities per GWAS, to use as additional weights. Defaults to all 1's.
        :type h2_snp: dict
        :param name: New phenotype name to use in the new SumStats object (default: 'gwama')
        :return: :class:`pysumstats.SumStats` object

        """
        assert isinstance(cov_matrix, pd.DataFrame), "cov_matrix should be a pd.DataFrame"
        assert isinstance(name, str), "name should be a string"
        if h2_snp is None:
            sumstatswarn('h2-snp not specified, using ones instead. This will bias the estimates')
            h2_snp = {x: 1 for x in self.pheno_names}
        elif (not isinstance(h2_snp, pd.DataFrame)) or isinstance(h2_snp, dict) or (not isinstance(h2_snp, pd.Series)):
            raise KeyError('h2-snp should be a dataframe or dictionary with phenotype names as keys or column names.')
        else:
            for p in self.pheno_names:
                try:
                    _ = h2_snp[p]
                except KeyError:
                    raise KeyError('Could not find h2_snp value for {}'.format(p))
        if cov_matrix is None:
            cov_matrix = pd.DataFrame(index=self.pheno_names, columns=self.pheno_names)
            vars = {}
            for p in self.pheno_names:
                vars[p] = []
                cov_matrix.loc[p, p] = 1
                for c in range(1, 24):
                    data = self.data[c]
                    data['z_{}'.format(p)] = data['b_{}'.format(p)] / data['se_{}'.format(p)]
                    vars[p].append(self.data[1]['z_{}'.format(p)].var() * (len(data) / len(self)))
                    self.data[c] = data
            for p1, p2 in [(i, i2) for n, i in enumerate(self.pheno_names) for i2 in self.pheno_names[(n + 1):]]:
                tmp_statistics = {}
                for c in range(1, 24):
                    if len(self.data[c]) > 0:
                        covar = self.data[c][['z_{}'.format(p1), 'z_{}'.format(p2)]].cov().iloc[0, 1] * (
                                len(self.data[c]) / len(self))
                        tmp_statistics[c] = dict(p1_var=vars[p1][c], p2_var=vars[p2][c], covar=covar)
                tmp_df = pd.DataFrame.from_dict(tmp_statistics)
                cov_matrix.loc[p1, p2] = tmp_df.loc['covar', :].sum() / (
                        np.sqrt(tmp_df.loc['p1_var', :].sum()) * np.sqrt(tmp_df.loc['p2_var', :].sum()))
                cov_matrix.loc[p2, p1] = cov_matrix.loc[p1, p2]
        else:
            if not isinstance(cov_matrix, pd.DataFrame):
                raise KeyError(
                    'Incorrect cov_matrix specified, should be pandas dataframe with phenotypes as columns and indices.')
            if (not sorted(list(cov_matrix.index)) == sorted(list(self.pheno_names))) or (
                    not sorted(list(cov_matrix.columns)) == sorted(list(self.pheno_names))):
                raise IndexError(
                    'Incorrect cov_matrix specified, should be pandas dataframe with phenotypes as columns and indices.')
        if not self.low_ram:
            new_data = {}
        else:
            new_data = _H5SSConnection(name, tmpdir=self.tmpdir)
        i_cov_matrix = pd.DataFrame(np.linalg.inv(cov_matrix.to_numpy(dtype=np.float64)), index=cov_matrix.index,
                                    columns=cov_matrix.columns)
        for c in range(1, 24):
            n_dat = self.data[c][['rsid', 'chr', 'bp', 'ea', 'oa', 'maf']].copy()
            data = self.data[c]
            for p in self.pheno_names:
                data.loc[data['n_{}'.format(p)].isna(), 'n_{}'.format(p)] = 0
                data.loc[data['b_{}'.format(p)].isna(), 'b_{}'.format(p)] = 0
                data.loc[data['se_{}'.format(p)].isna(), 'se_{}'.format(p)] = 1
                data.loc[data['eaf_{}'.format(p)].isna(), 'eaf_{}'.format(p)] = 0
            for p in self.pheno_names:
                data['z_{}'.format(p)] = data['b_{}'.format(p)] / data['se_{}'.format(p)]
                data['w_{}'.format(p)] = np.sqrt(data['n_{}'.format(p)] * h2_snp[p])
                data['wz_{}'.format(p)] = data['w_{}'.format(p)] * data['z_{}'.format(p)]
            data['cw'] = np.array(
                [data['w_{}'.format(p)] * data['w_{}'.format(p2)] * cov_matrix.loc[p, p2]
                 for p in self.pheno_names for p2 in [x for x in self.pheno_names]]).sum(axis=0)
            n_dat['z'] = data[['wz_{}'.format(x) for x in self.pheno_names]].sum(axis=1) / np.sqrt(
                data['cw'])
            n_dat.loc[:, 'p'] = norm.sf(abs(n_dat.loc[:, 'z'])) * 2
            n_dat['n'] = np.array([np.sqrt(data['n_{}'.format(p)]) * np.sqrt(
                data['n_{}'.format(p2)]) * i_cov_matrix.loc[p, p2] for p in self.pheno_names for p2 in
                                   [x for x in self.pheno_names]]).sum(axis=0)
            n_obs = data[['n_{}'.format(p) for p in self.pheno_names]].sum(axis=1)
            n_dat['eaf'] = np.array(
                [data['n_{}'.format(p)] * data['eaf_{}'.format(p)] for p in self.pheno_names]).sum(
                axis=0) / n_obs
            n_dat['maf'] = pd.concat([n_dat['eaf'], (1 - n_dat['eaf'])], axis=1).min(axis=1)
            n_dat['b'] = n_dat['z'] / np.sqrt(n_obs * 2 * n_dat['eaf'] * (1 - n_dat['eaf']))
            n_dat['se'] = (1 / np.sqrt(n_obs)) * (1 / np.sqrt(2 * n_dat['eaf'] * (1 - n_dat['eaf'])))
            new_data[c] = n_dat
        new_columns = list(new_data[1].columns)
        new_data = SumStats(path=None, phenotype=name, data=new_data)
        new_data.variables = new_columns
        return new_data

    def merge(self, other, inplace=False, how='inner', low_memory=False):
        """ Merge with other SumStats or MergedSumstats object(s).

        :param other: :class:`pysumstats.SumStats`, or :class:`pysumstats.MergedSumStats` object, or a list of SumStats, MergedSumstats objects
        :type other: :class:`pysumstats.SumStats`, :class:`pysumstats.MergedSumStats`, or list.
        :param inplace: Enable to store the new data in the current MergedSumStats object. (currently not supported when low_ram is enabled)
        :type inplace: bool
        :param how: Type of merge, for now only implemented for merges with :class:`pysumstats.SumStats` objects
        :type how: str
        :param low_memory: Enable to use a more RAM-efficient merging method (WARNING: still untested)
        :type low_memory: bool
        :return: None, or :class:`pysumstats.MergedSumStats` object.

        """
        assert isinstance(other, SumStats) or isinstance(other, MergedSumStats) or isinstance(other, list), "other should be SumStats, MergedSumStats or list"
        assert isinstance(inplace, bool), "inplace should be True or False"
        assert isinstance(how, str), "how should be a string"
        assert isinstance(low_memory, bool), "low_memory should be True or False"
        if isinstance(other, list):
            for o in other:
                assert isinstance(o, SumStats) or isinstance(o, MergedSumStats), "items in other should be SumStats or MergedSumStats"
        if self.low_ram and inplace:
            sumstatswarn('Inplace merging with low_ram currently does not yield any performance benefits.')
        if isinstance(other, list):
            if inplace:
                for o in other:
                    self.merge(o, True, how=how, low_memory=low_memory)
            else:
                newmerged = self.merge(other[0], False, how=how, low_memory=low_memory)
                for o in other[1:]:
                    newmerged.merge(o, True, how=how, low_memory=low_memory)
                return newmerged
        else:
            if low_memory and (not how == 'inner'):
                raise NotImplementedError('low_memory only allows for an inner merge.')
            if other.phenotype_name is not None:
                if other.phenotype_name in self.pheno_names:
                    sumstatswarn(
                        'Phenotype name already in the dataset, converting to and {0}2'.format(other.phenotype_name))
                    other.phenotype_name = other.phenotype_name + '2'
                self.suffixes[1] = other.phenotype_name
                merge_info = dict(xlen=0, ylen=0, newlen=0)
                joined = [x for x in other.variables if x in self.variables]
                for x in [x for x in other.variables if x not in joined]:
                    sumstatswarn(
                        'Could not find column {} in pysumstats for {}, column dropped'.format(x, other.phenotype_name))
                merged_data = {}
                for c in range(1, 24):
                    data_x = self.data[c].copy()
                    merge_info['xlen'] += len(data_x)
                    data_y = other.data[c].copy()
                    merge_info['ylen'] += len(data_y)
                    data_m_y = data_y.rename(
                        columns={k: '{}_{}'.format(k, other.phenotype_name) for k in data_y.columns if
                                 k not in ['rsid', 'chr', 'bp']})
                    if low_memory:
                        if np.any(data_x.duplicated(subset='rsid')) or np.any(data_m_y.duplicated(subset='rsid')):
                            raise KeyError('Missing rsids in either dataset will cause low_memory merge to fail')
                        merged_data[c] = data_x.loc[data_x['rsid'].isin(data_m_y['rsid'].tolist()), :]
                        data_m_y = data_m_y.loc[data_m_y['rsid'].isin(data_x['rsid'].tolist()), :]
                        del data_y
                        merged_data[c].sort_values(by='rsid', inplace=True)
                        data_m_y.sort_values(by='rsid', inplace=True)
                        merged_data[c].reset_index(inplace=True, drop=True)
                        data_m_y.reset_index(inplace=True, drop=True)
                        data_m_y = data_m_y[
                            [x + '_' + other.phenotype_name for x in joined if x not in ['rsid', 'chr', 'bp']]]
                        for x in data_m_y.columns:
                            merged_data[c][x] = data_m_y[x]
                            data_m_y.drop(axis=1, labels=[x], inplace=True)
                        merged_data[c].sort_values(by='bp', inplace=True)
                    else:
                        joined_y = ['rsid', 'chr_y', 'bp_y'] + [x + '_' + other.phenotype_name for x in joined if
                                                                x not in ['rsid', 'chr', 'bp']]
                        data_m_y = data_m_y.rename(columns={'chr': 'chr_y', 'bp': 'bp_y'})
                        merged_data[c] = data_x.merge(data_m_y[joined_y], on='rsid', how=how)
                        merged_data[c].loc[merged_data[c]['chr'].isna(), 'chr'] = merged_data[c].loc[
                            merged_data[c]['chr'].isna(), 'chr_y']
                        merged_data[c].loc[merged_data[c]['bp'].isna(), 'bp'] = merged_data[c].loc[
                            merged_data[c]['bp'].isna(), 'bp_y']
                        merged_data[c] = merged_data[c].drop(axis=1, labels=['chr_y', 'bp_y'])
                    merged_data[c]['maf'] = _recompute_maf(merged_data[c], self.pheno_names + [other.phenotype_name])
                    merge_info['newlen'] += len(merged_data[c])
                    new_phenos = self.pheno_names + [other.phenotype_name]
                if not inplace:
                    return MergedSumStats(data=merged_data, phenotypes=new_phenos, merge_info=merge_info,
                                          variables=joined, xy=self.suffixes, low_ram=self.low_ram, tmpdir=self.tmpdir)
                else:
                    self.__init__(merged_data, new_phenos, merge_info, joined, self.suffixes, self.low_ram)
            else:
                for n, other_name in enumerate(other.pheno_names):
                    if other_name in self.pheno_names:
                        other.pheno_names[n] = other_name + '_y'
                        for c in range(1, 24):
                            for x in other.variables:
                                odat = other.data[c]
                                odat.rename(columns={x + '_' + other_name: x + '_' + other_name + '_y'}, inplace=True)
                                other.data[c] = odat
                if not self.low_ram:
                    merged_data = {}
                else:
                    tmpfilename = ''.join([''.join([j[0], j[1]]) for j in self.pheno_names]) + ''.join(
                        [''.join([j[0], j[1]]) for j in other.pheno_names])
                    merged_data = _H5SSConnection(
                        '{}{}'.format(tmpfilename, len(os.listdir('sumstats_temporary'))), tmpdir=self.tmpdir)
                for c in range(1, 24):
                    sdat = self.data[c]
                    odat = other.data[c]
                    s_id = pd.DataFrame(sdat['rsid'])
                    o_id = pd.DataFrame(odat['rsid'])
                    s_id['s_idx'] = s_id.index
                    o_id['o_idx'] = o_id.index
                    idm = s_id.merge(o_id, on='rsid')
                    sdat.rename(columns={'ea': 'ea_x', 'oa': 'oa_x'}, inplace=True)
                    odat.rename(columns={'ea': 'ea_y', 'oa': 'oa_y'}, inplace=True)
                    other_cols = [x for x in odat.columns if x not in ['rsid', 'chr', 'bp']]
                    sdat = sdat.loc[idm['s_idx'], :].reset_index(drop=True)
                    odat = odat.loc[idm['o_idx'], other_cols].reset_index(drop=True)
                    for oc in odat.columns:
                        sdat[oc] = odat[oc]
                    merged_data[c] = sdat
                    merged_data[c]['maf'] = _recompute_maf(merged_data[c], self.pheno_names + other.pheno_names)
                if inplace:
                    self.data = merged_data
                    self.columns = self.data[1].columns
                    self.suffixes = ['x', 'y']
                    self._allign(ynames=['_{}'.format(x) for x in other.pheno_names])
                else:
                    newmergedss = MergedSumStats(data=merged_data, phenotypes=self.pheno_names + other.pheno_names,
                                                 merge_info={}, xy=['x', 'y'], variables=self.variables,
                                                 low_ram=self.low_ram, tmpdir=self.tmpdir, allign=False)
                    newmergedss._allign(ynames=['_{}'.format(x) for x in other.pheno_names])
                    return newmergedss

    def describe(self, columns=None, per_chromosome=False):
        """ Get a summary of the data.

        :param columns: List of column names to print summary for (default: ['b', 'se', 'p'])
        :type columns: list
        :param per_chromosome: Enable to return a list of summary dataframes per chromosome
        :type per_chromosome: bool
        :return: pd.Dataframe, or list of pd.Dataframes

        """
        assert isinstance(columns, list) or (columns is None), "columns should be a list"
        assert isinstance(per_chromosome, bool), "per_chromosome should be True or False"
        if columns is None:
            columns = ['b', 'se', 'p']
        if (not isinstance(columns, list)) and isinstance(columns, str):
            columns = [columns]
        sum_cols = []
        for c in columns:
            if c not in self.data[1].columns:
                if '{}_{}'.format(c, self.pheno_names[0]) in self.data[1].columns:
                    for c2 in ['{}_{}'.format(c, p) for p in self.pheno_names]:
                        sum_cols.append(c2)
                else:
                    sumstatswarn('column {} not found'.format(c))
            else:
                sum_cols.append(c)
        return self._get_summary(sum_cols, per_chromosome)

    def prep_for_mr(self, exposure, outcome, filename=None, p_cutoff=None, bidirectional=False, **kwargs):
        """Save a pre-formatted file to use with the MendelianRandomization package in R.

        :param exposure: phenotype name to use as exposure.
        :type exposure: str
        :param outcome: phenotype name to use as outcome.
        :type outcome: str
        :param filename: Path to where the resulting file(s) should be stored, or list of paths if bidirectional=True
        :type filename: str, list or None
        :param p_cutoff: Optional p-value cut-off to apply. Will include SNPs where P > p_cutoff
        :type p_cutoff: float
        :param bidirectional: Enable to store two files (exposure=exposure, outcome=outcome), and (exposure=outcome, outcome=exposure)
        :type bidirectional: bool
        :param kwargs: Additional keyword arguments to be passed to pandas to_csv function.
        :return: None

        """
        assert isinstance(exposure, str), "exposure should be a string"
        assert isinstance(outcome, str), "exposure should be a string"
        assert isinstance(filename, str) or isinstance(filename, list) or (filename is None), "filename should be a string or list of strings"
        assert isinstance(p_cutoff, float), "p_cutoff should be a float"
        assert isinstance(bidirectional, bool), "bidirectional should be True or False"
        if isinstance(filename, list):
            for o in filename:
                assert isinstance(o, str), "filename should be a string or list of strings"
        if bidirectional and (not isinstance(filename, list)):
            raise ValueError(
                'If bidirectional, filename should be a list of filenames: (exp-outcome) name and (outcome-exp) name.')
        mr_data = []
        if exposure not in self.pheno_names:
            raise KeyError('{} not found in phenotypes'.format(exposure))
        if outcome not in self.pheno_names:
            raise KeyError('{} not found in phenotypes'.format(outcome))
        for c in self.data.keys():
            if p_cutoff is None:
                mr_data_ = self.data[c].copy()
            else:
                mr_data_ = self.data[c].loc[self.data[c]['p_{}'.format(exposure)] <= p_cutoff, :].copy()
            newcols = [['se', 'se'], ['b', 'b']]
            newcols2 = [[exposure, 'exposure'], [outcome, 'outcome']]
            renames = {'{}_{}'.format(old, x): '{}.{}'.format(x2, new) for x, x2 in newcols2 for old, new in newcols}
            mr_data_.rename(columns=renames, inplace=True)
            mr_data_.rename(columns={'ea': 'exposure.A1', 'oa': 'exposure.A2'}, inplace=True)
            mr_data_['outcome.A1'] = mr_data_['exposure.A1']
            mr_data_['outcome.A2'] = mr_data_['exposure.A2']
            mr_data.append(mr_data_[
                               ['rsid', 'chr', 'bp', 'exposure.A1', 'exposure.A2', 'outcome.A1', 'outcome.A2'] + list(
                                   renames.values())])
        mr_data = pd.concat(mr_data, axis=0)
        if filename is None:
            mr_data.to_csv('MR-prepped_{}_{}.csv'.format(exposure, outcome), **kwargs)
        else:
            if not bidirectional:
                mr_data.to_csv(filename, **kwargs)
            else:
                mr_data.to_csv(filename[0], **kwargs)
        if bidirectional:
            if filename is None:
                self.prep_for_mr(exposure=outcome, outcome=exposure, filename=None, p_cutoff=p_cutoff,
                                 bidirectional=False, **kwargs)
            else:
                self.prep_for_mr(exposure=outcome, outcome=exposure, filename=filename[1], p_cutoff=p_cutoff,
                                 bidirectional=False, **kwargs)

    def manhattan(self, filename=None, phenotypes=None, nrows=None, ncols=None, figsize=None, dpi=300, **kwargs):
        """Generates manhattan plots for each phenotype (or specified phenotypes) in merged GWAS data.

        :param filename: Target file to save the resulting figure to (if no name is specified, fig and axes are returned)
        :type filename: str
        :param phenotypes: List of phenotype names to plot manhattans for (defaults to plotting all phenotypes)
        :type phenotypes: list
        :param nrows: Specify number of rows in the figure ( defaults to int(ceil(len(phenotypes)/ncols)) )
        :type nrows: int
        :param ncols: Specify number of columns in the figure ( defaults to int(log2(len(phenotypes)/2)) )
        :type ncols: int
        :param figsize: Specify width and height of figure in inches ( defaults to (ncols*8, nrows*4) )
        :type figsize: (int, int)
        :param dpi: DPI setting to use when saving the figure.
        :type dpi: int
        :param kwargs: Other keyword arguments to be passed to :func:`pysumstats.plot.manhattan`
        :return: (fig, axes) or None.
        """
        assert isinstance(filename, str) or (filename is None), "filename should be a string or None"
        assert isinstance(phenotypes, list) or (phenotypes is None), "phenotypes should be a list of strings or None"
        assert isinstance(dpi, int), "dpi should be an integer"
        if phenotypes is None:
            phenotypes = self.pheno_names
        for p in phenotypes:
            assert p in self.pheno_names, "{} not found in phenotypes".format(p)
        plotwindow = _MultiWindowPlot(len(phenotypes), nrows=nrows, ncols=ncols, figsize=figsize, shape='rect')
        dat = self[['rsid', 'chr', 'bp']]
        for phenotype in phenotypes:
            fig, ax = plotwindow.get_next_ax()
            dat['p'] = self['p_{}'.format(phenotype)]
            if 'title' not in kwargs.keys():
                manhattan(dat, fig=fig, ax=ax, title=phenotype, **kwargs)
            else:
                manhattan(dat, fig=fig, ax=ax, **kwargs)
        return plotwindow.finish(filename=filename, dpi=dpi)

    def qqplot(self, filename=None, phenotypes=None, nrows=None, ncols=None, figsize=None, dpi=300, **kwargs):
        """Generates QQ-plots for each phenotype (or specified phenotypes) in merged GWAS data.

        :param filename: Target file to save the resulting figure to (if no name is specified, fig and axes are returned)
        :type filename: str
        :param phenotypes: List of phenotype names to plot QQ-plots for (defaults to plotting all phenotypes)
        :type phenotypes: list
        :param nrows: Specify number of rows in the figure ( defaults to int(sqrt(len(phenotypes))) )
        :type nrows: int
        :param ncols: Specify number of columns in the figure ( defaults to int(ceil(len(phenotypes)/nrows)) )
        :type ncols: int
        :param figsize: Specify width and height of figure in inches ( defaults to (ncols*5, nrows*5) )
        :type figsize: (int, int)
        :param dpi: DPI setting to use when saving the figure.
        :type dpi: int
        :param kwargs: Other keyword arguments to be passed to :func:`pysumstats.plot.qqplot`
        :return: (fig, axes) or None.

        """
        assert isinstance(filename, str) or (filename is None), "filename should be a string or None"
        assert isinstance(phenotypes, list) or (phenotypes is None), "phenotypes should be a list of strings or None"
        assert isinstance(dpi, int), "dpi should be an integer"
        if phenotypes is None:
            phenotypes = self.pheno_names
        for p in phenotypes:
            assert p in self.pheno_names, "{} not found in phenotypes".format(p)
        plotwindow = _MultiWindowPlot(len(phenotypes), nrows=nrows, ncols=ncols, figsize=figsize, shape='square')
        for phenotype in phenotypes:
            fig, ax = plotwindow.get_next_ax()
            if 'title' not in kwargs.keys():
                qqplot(self['p_{}'.format(phenotype)].values, fig=fig, ax=ax, title=phenotype, **kwargs)
            else:
                qqplot(self['p_{}'.format(phenotype)].values, fig=fig, ax=ax, **kwargs)
        return plotwindow.finish(filename=filename, dpi=dpi)

    def pzplot(self, filename=None, phenotypes=None, nrows=None, ncols=None, figsize=None, dpi=300, **kwargs):
        """Generates PZ-plots for each phenotype (or specified phenotypes) in merged GWAS data.

        :param filename: Target file to save the resulting figure to (if no name is specified, fig and axes are returned)
        :type filename: str
        :param phenotypes: List of phenotype names to plot PZ-plots for (defaults to plotting all phenotypes)
        :type phenotypes: list
        :param nrows: Specify number of rows in the figure ( defaults to int(sqrt(len(phenotypes))) )
        :type nrows: int
        :param ncols: Specify number of columns in the figure ( defaults to int(ceil(len(phenotypes)/nrows)) )
        :type ncols: int
        :param figsize: Specify width and height of figure in inches ( defaults to (ncols*5, nrows*5) )
        :type figsize: (int, int)
        :param dpi: DPI setting to use when saving the figure.
        :type dpi: int
        :param kwargs: Other keyword arguments to be passed to :func:`pysumstats.plot.pzplot`
        :return: (fig, axes) or None.

        """
        assert isinstance(filename, str) or (filename is None), "filename should be a string or None"
        assert isinstance(phenotypes, list) or (phenotypes is None), "phenotypes should be a list of strings or None"
        assert isinstance(dpi, int), "dpi should be an integer"
        if phenotypes is None:
            phenotypes = self.pheno_names
        for p in phenotypes:
            assert p in self.pheno_names, "{} not found in phenotypes".format(p)
        plotwindow = _MultiWindowPlot(len(phenotypes), nrows=nrows, ncols=ncols, figsize=figsize, shape='square')
        for phenotype in phenotypes:
            dat = self[['{}_{}'.format(v, phenotype) for v in ['b', 'se', 'p']]].copy()
            dat.rename(columns={'{}_{}'.format(v, phenotype): v for v in ['b', 'se', 'p']}, inplace=True)
            fig, ax = plotwindow.get_next_ax()
            if 'title' not in kwargs.keys():
                pzplot(dat, fig=fig, ax=ax, title='PZ_{}'.format(phenotype), **kwargs)
            else:
                pzplot(dat, fig=fig, ax=ax, **kwargs)
        return plotwindow.finish(filename=filename, dpi=dpi)

    def afplot(self, ref_phenotypes=None, other_phenotypes=None, filename=None, nrows=None, ncols=None, figsize=None,
               dpi=300, **kwargs):
        """Generates AF comparison plots for merged GWAS data.

        :param ref_phenotypes: List of phenotypes to use as reference (defaults to all phenotypes)
        :type ref_phenotypes: list
        :param other_phenotypes: List of phenotypes to compare reference to (defaults to all phenotypes, overlapping plots will be dropped)
        :type other_phenotypes: list
        :param filename: Target file to save the resulting figure to (if no name is specified, fig and axes are returned)
        :type filename: str
        :param nrows: Specify number of rows in the figure ( defaults to int(ceil(n_plots/ncols)) )
        :type nrows: int
        :param ncols: Specify number of columns in the figure ( defaults to int(sqrt(n_plots)))
        :type ncols: int
        :param figsize: Specify width and height of figure in inches ( defaults to (ncols*5, nrows*5) )
        :type figsize: (int, int)
        :param dpi: DPI setting to use when saving the figure.
        :type dpi: int
        :param kwargs: Other keyword arguments to be passed to :func:`pysumstats.plot.manhattan`
        :return: (fig, axes) or None.

        """
        assert isinstance(filename, str) or (filename is None), "filename should be a string or None"
        assert isinstance(ref_phenotypes, list) or (ref_phenotypes is None), "ref_phenotypes should be a string or None"
        assert isinstance(other_phenotypes, list) or (other_phenotypes is None), "other_phenotypes should be a string or None"
        assert isinstance(dpi, int), "dpi should be an integer"
        if ref_phenotypes is None:
            ref_phenotypes = self.pheno_names
        for p in ref_phenotypes:
            assert p in self.pheno_names, "{} not found in phenotypes".format(p)
        if other_phenotypes is None:
            other_phenotypes = self.pheno_names
        for p in other_phenotypes:
            assert p in self.pheno_names, "{} not found in phenotypes".format(p)
        n_plots = len(ref_phenotypes) * len(other_phenotypes)
        plotwindow = _MultiWindowPlot(n_plots, nrows=nrows, ncols=ncols, figsize=figsize, shape='square')
        for p1 in ref_phenotypes:
            eafp1 = self['eaf_{}'.format(p1)].values
            for p2 in other_phenotypes:
                fig, ax = plotwindow.get_next_ax()
                if p1 != p2:
                    afplot(eafp1, self['eaf_{}'.format(p2)].values,
                           refname=r'$EAF_{}$'.format('{' + p1.replace('_', r'\_') + '}'),
                           othername=r'$EAF_{}$'.format('{' + p2.replace('_', r'\_') + '}'), fig=fig, ax=ax,
                           **kwargs)
        return plotwindow.finish(filename=filename, dpi=dpi)

    def zzplot(self, ref_phenotypes=None, other_phenotypes=None, filename=None, nrows=None, ncols=None, figsize=None,
               dpi=300, **kwargs):
        """Generates ZZ comparison plots for merged GWAS data.

        :param ref_phenotypes: List of phenotypes to use as reference (defaults to all phenotypes)
        :type ref_phenotypes: list
        :param other_phenotypes: List of phenotypes to compare reference to (defaults to all phenotypes, overlapping plots will be dropped)
        :type other_phenotypes: list
        :param filename: Target file to save the resulting figure to (if no name is specified, fig and axes are returned)
        :type filename: str
        :param nrows: Specify number of rows in the figure ( defaults to int(ceil(n_plots/ncols)) )
        :type nrows: int
        :param ncols: Specify number of columns in the figure ( defaults to int(sqrt(n_plots)))
        :type ncols: int
        :param figsize: Specify width and height of figure in inches ( defaults to (ncols*5, nrows*5) )
        :type figsize: (int, int)
        :param dpi: DPI setting to use when saving the figure.
        :type dpi: int
        :param kwargs: Other keyword arguments to be passed to :func:`pysumstats.plot.zzplot`
        :return: (fig, axes) or None.

        """
        assert isinstance(filename, str) or (filename is None), "filename should be a string or None"
        assert isinstance(ref_phenotypes, list) or (ref_phenotypes is None), "ref_phenotypes should be a string or None"
        assert isinstance(other_phenotypes, list) or (other_phenotypes is None), "other_phenotypes should be a string or None"
        assert isinstance(dpi, int), "dpi should be an integer"
        if ref_phenotypes is None:
            ref_phenotypes = self.pheno_names
        for p in ref_phenotypes:
            assert p in self.pheno_names, "{} not found in phenotypes".format(p)
        if other_phenotypes is None:
            other_phenotypes = self.pheno_names
        for p in other_phenotypes:
            assert p in self.pheno_names, "{} not found in phenotypes".format(p)
        n_plots = len(ref_phenotypes) * len(other_phenotypes)
        plotwindow = _MultiWindowPlot(n_plots, nrows=nrows, ncols=ncols, figsize=figsize, shape='square')
        for p1 in ref_phenotypes:
            zz1 = self[['b_{}'.format(p1), 'se_{}'.format(p1)]].copy()
            zz1.rename(columns={'b_{}'.format(p1): 'b', 'se_{}'.format(p1): 'se'}, inplace=True)
            for p2 in other_phenotypes:
                fig, ax = plotwindow.get_next_ax()
                if p1 != p2:
                    zz2 = self[['b_{}'.format(p2), 'se_{}'.format(p2)]].copy()
                    zz2.rename(columns={'b_{}'.format(p2): 'b', 'se_{}'.format(p2): 'se'}, inplace=True)
                    zzplot(zz1, zz2, xname=r'$Z_{}$'.format('{' + p1.replace('_', r'\_') + '}'),
                           yname='r$Z_{}$'.format('{' + p2.replace('_', r'\_') + '}'),
                           fig=fig,
                           ax=ax, **kwargs)
        return plotwindow.finish(filename=filename, dpi=dpi)


import pickle
vocab = open("vocab","rb")
vocab = pickle.load(vocab)
d_vocab = {vocab["d_vocab"][i]:i for i in vocab["d_vocab"]}
num_layers = vocab["num_layers"]
import torch

def get_word(x):
	v,i=torch.max(x,0)
	return d_vocab[i.item()+1]

def get_tensor(x):
	return torch.LongTensor([vocab["d_vocab"][x]])

class Attention(torch.nn.Module):
	def __init__(self,embedding):
		super(Attention,self).__init__()
		self.embedding	= embedding
		self.emb1	= torch.nn.Embedding(len(vocab["e_vocab"])+1 , self.embedding)
		self.lstm1	= torch.nn.LSTM(num_layers = num_layers,input_size=self.embedding,hidden_size=self.embedding,batch_first=True)
		self.lin1	= torch.nn.Linear(vocab["e_max_len"],vocab["e_max_len"])
		self.soft	= torch.nn.Softmax(dim=1)

		self.emb2	= torch.nn.Embedding(len(vocab["d_vocab"])+1,self.embedding,padding_idx=0)
		self.lin2	= torch.nn.Linear(vocab["e_max_len"]+self.embedding,self.embedding)
		self.lstm2	= torch.nn.LSTM(num_layers = num_layers,input_size=self.embedding,hidden_size=self.embedding,batch_first=True)
		self.lin3	= torch.nn.Linear(self.embedding,len(vocab["d_vocab"]))

	def forward(self,e,d):
		pe=torch.nn.utils.rnn.pack_padded_sequence(e,[len(e[0])],batch_first=True)
		e = self.emb1(pe[0])
		#print(e)
		pe=torch.nn.utils.rnn.PackedSequence(e,pe[1])
		epo,(last_hid,last_state) = self.lstm1(pe)

		upo = torch.nn.utils.rnn.pad_packed_sequence(epo,batch_first=True,total_length=vocab["e_max_len"])[0]
		#print(upo)
		#print(last_hid)
		word = ""
		#++
		#d = d[:,None]
		#++
		#de = self.emb2(d)
		while word!="</s>":
		#------------------------------------------------------------------------------------------#
			#modified hidden for multi_layer purpose+++++++++
			last_hid_last = last_hid[-1][None]
			#last_hid_last = last_hid

			c = (last_hid_last[-1,:,None]*upo).sum(dim=2)
			c = self.lin1(c)
			#print(c)
			c = self.soft(c)
			#print(c)
			c = (c[:,:,None]*upo).sum(dim=2)
			#print(c)
			#--
			d = d[:,None]
			#print(d)
			#print(l)
			#--
			de = self.emb2(d)
			#print(de)
			c = c[:,None]
			de = torch.cat([c,de],dim=2)
			#print(de)
			de = self.lin2(de)
			#print(de)
			dout,(last_hid,last_state) = self.lstm2(de,(last_hid,last_state))
			#print(dout)
			#print(last_hid)
			#break
			if len(dout)>0:
				dout = self.lin3(dout)
				dout = dout[:,-1,:]
				word = get_word(dout[0])
				print(word)
				d    = get_tensor(word)
				#++d     = last_hid
		return
		#no problem-----------------------------------------------------------------------------------#

model = Attention(vocab["embedding"])
model.load_state_dict(torch.load('attention.pt'),strict=False)

sentence = "life </s>"
indexing = torch.LongTensor([[vocab["e_vocab"][s] for s in sentence.split()]])
d = model(indexing,get_tensor("<s>"))


####################
# NECESSARY IMPORTS#
####################

# core app function mechanics
import pandas as pd
import numpy as np
import cv2
from PIL import Image
import pickle

# importing custom functions
from softcopyOCR import decode_predictions, get_image_text, clean_string, clean_tesseract_output, display_ocr
from softcopy_textmatch import best_match
from softcopy_recommender import get_recommends_list, get_recommends, get_book_details

# app building
import streamlit as st
import time
import urllib.request
import io

# surpress streamlit warning
st.set_option('deprecation.showfileUploaderEncoding', False)
st.beta_set_page_config(page_title='SOFTCOPY',
                        page_icon=None,
                        layout='centered',
                        initial_sidebar_state='auto')

# loading data
with open('data/all_books_clean.pkl', 'rb') as f:
    books_all = pickle.load(f)

ISBNs = pd.read_csv('data/ISBNs.csv')

with open('data/vt_800.pkl', 'rb') as f:
    VT = pickle.load(f)

def show_book_images(recs):
    '''
    Displays images of recommendation ocr_results
    recs: list, ISBNs, of book recommendations
    '''
    # iterate through book recommendations
    for book in recs:
        try:
            # get url of large image for each recommended book
            rec_dict = get_book_details(book, books_all)
            url = rec_dict['image_l']
            with urllib.request.urlopen(url) as url:
                with open('temp.jpg', 'wb') as f:
                    f.write(url.read())

            # load and display image
            test_img = Image.open('temp.jpg')
            st.image(test_img)

        # error handling for image display error (rare)
        except OSError:
            pass

    # done!
    st.balloons()

def get_recs(matched_isbns, num_recs):
    '''
    Gets recommendations from matched ISBN_list
    Returns list of ISBNs for recommended books based on books in matched_isbns
    matched_isbns: list, book ISBNs matched from ORC output, produced by match_title
    '''

    st.header('**Step 3: Recommend books based on the ones on your bookshelf**')
    st.markdown('Getting recommendations...')

    # animates quick loading bar
    rec_loading_bar = st.progress(0)
    for percent_complete in range(100):
        time.sleep(0.02)
        rec_loading_bar.progress(percent_complete + 1)

    # gets recommendations
    global VT, ISBNs, books_all
    recs, basis = get_recommends_list(matched_isbns, VT, ISBNs, n=num_recs)

    # prints recommendation basis
    st.subheader('Based off of these books that you own : ')
    for book in basis:
        rec_dict = get_book_details(book, books_all)
        st.markdown("* {} by {} ({}, {})".format(rec_dict['title'],
                                                rec_dict['author'],
                                                rec_dict['publisher'],
                                                rec_dict['year']))

    # prints recommendations
    st.subheader('Your recommended reading includes: ')
    for book in recs:
        rec_dict = get_book_details(book, books_all)
        st.markdown("* {} by {} ({}, {})".format(rec_dict['title'],
                                                rec_dict['author'],
                                                rec_dict['publisher'],
                                                rec_dict['year']))

    # passing recs on to book cover displaying function
    show_book_images(recs)

def match_title(final_ocr, num_recs, confidence):
    '''
    Matches OCR text with book ISBNs
    Returns list of ISBNs of books that match OCR text above the confidence threshold
    final_ocr: list, strings for each grouping of text generated by OCR process
    '''

    st.header('**Step 2: Match OCR results to books in our database**')
    st.write("Matching titles...")
    match_loading_bar = st.progress(0)
    st.subheader('Text matching results:')

    matched_isbns = []
    global books_all

    # finding best match for each string detected by OCR
    for idx, title in enumerate(final_ocr):
        match_loading_bar.progress((idx+1)/len(final_ocr)) # increments loading bar
        match = best_match(title, books_all, min_confidence=confidence)
        matched_isbns.append(match)

    # w.i.p.: buttons to deselect books from rec process
    buttons = [get_book_details(i, books_all)['title'] for i in matched_isbns if get_book_details(i, books_all)]
    match_select = st.sidebar.multiselect('Recommendation basis: ', buttons, buttons)

    # passing matched titles on to recommendation engine
    get_recs(matched_isbns, num_recs)

def run_ocr(new_img, num_recs, confidence, verbose):
    '''
    Performs optical character recognition on specified image
    Returns list of strings
    new_img: image, supplied by main
    '''

    # parameters for OpenVC
    args = {'east': 'frozen_east_text_detection.pb',
            'min_confidence': 0.9,
            'width': 320,
            'height': 320,
            'padding': 0.2}

    st.header('**Step 1: Scan your image for text with OCR**')
    st.write('Scanning image...')

    # images preprocessing, text detection w/ open cv, and text recognition
    # with pytesseract
    ocr_results = get_image_text(args, new_img)

    # show boundary boxes during scan if sidebar box checked
    if verbose:
        display_ocr(ocr_results, new_img)

    # cleans up strings
    final_ocr = clean_tesseract_output(ocr_results, 100)

    # prints results
    st.subheader("Optical character recognition results: ")
    st.write(pd.DataFrame(final_ocr, columns = ['']))

    # passes final ocr results to text matching
    match_title(final_ocr, num_recs, confidence)

def main():
    '''
    Main app function. Features page selector and image upload.
    Press button once image uploaded to start matching/recommendation pipeline
    '''
    st.title('SoftCopy')
    activities = ['Home', 'About']

    # main site nav
    choice = st.sidebar.selectbox("", activities)

    st.sidebar.write('App parameters')

    # recs/text match for get_recs / get_recommends_list fcns
    num_recs = st.sidebar.slider("Recommendations per match", 1, 10, 4) #

    # text match confidence threshold for match_title / best_match fcns
    confidence = st.sidebar.slider('Text matching confidence % threshold',
    0, 100, 75)

    # toggle to show text detection/recognition images during OCR
    verbose = st.sidebar.checkbox('Show OCR Preprocessing', value=False)

    # main site nav choice
    if choice == 'Home':

        # image uploader
        st.header('**Upload an image of your books.**')
        st.header('**Get customized book recommendations.**')

        uploaded_file = st.file_uploader("Choose a image file")

        # displaying uploaded image
        if uploaded_file is not None:
            image_ = Image.open(uploaded_file) # reads image file
            st.image(image_, caption='Uploaded image', use_column_width=True) # displays on app
            new_img = np.array(image_.convert('RGB')) # makes it compatible with OpenCV

            # click to run program (pass image on to OCR)
            if st.button("Let's get started!"):
                run_ocr(new_img, num_recs, confidence, verbose)

    # main site nav choice
    elif choice == 'About':
        about()

def about():
    '''
    Helper function for the about page
    '''
    # overview
    st.subheader('About SoftCopy')
    st.markdown(
                '''
                SoftCopy uses an image of your bookshelf to recommend books for you to read.
                All you need to do is import a photo and the app does the rest!

                The app has three main phases:

                1. Optical character recognition (OCR)
                2. Text matching
                3. Recommendation
    	''')

    # select which phase to view details
    about_options = st.select_slider(
        label='Select a phase to learn more about how it works',
        options=['OCR', 'Text Matching', 'Recommendation'], )

    # OCR phase details
    if about_options == 'OCR':

        st.subheader('Optical character recognition')

        st.markdown('''SoftCopy uses Python, OpenCV and Pytesseract to detect and recognize text in your image.
        OpenCV is used to process images to prepare them for OCR''')

        st.image("https://nanonets.com/blog/content/images/2019/12/Tesseract.gif", width=500)

        st.markdown('''
        Step 1) Preprocessing
        * Image scaling, red/blue swap (for Pytesseract compatability)
        * Mean image subtraction (for clearer signal)

        Step 2) Text detection
        * Bounding box detection using EAST text detection model
        * Non-maxima suppression to eliminate weak, overlapping bounding boxes

        Step 3) Text recognition
        * Pytesseract - pretrained LSTM Neural Network model
        ''')

        # OCR example image
        st.image('Text detection pics/Text Detection_screenshot_09.09.2020 - 2.png', width=500, caption='OCR in action!')

    # text matching phase details
    if about_options == 'Text Matching':

        st.subheader('Text matching')

        st.markdown('''
        Next, SoftCopy takes the text that was detected during the OCR step and performs
        a **token set fuzzy match** the title and author against a list of **266,664 books**
        pulled from the [Book Crossing dataset](http://www2.informatik.uni-freiburg.de/~cziegler/BX/).
        ''')

        # books crossing logo
        st.image('images/bookcrossing.jpg')

        st.markdown('''
        Matching process:

        1. Combine title and author strings

        2. Tokenize into list of unigrams

        3. Sort tokens alphabetically

        4. Remove duplicate tokens

        5. Compare too each book in data set and assigne confidence scores

        Pairs with over 75% confidence are considered a match, but you can toggle
        this percentage in the sidebar to suit the needs of your image.

        Examples:
        * No match for "sil lack a sian sos oo tian m e a r ii lom aimles ba al".
        (Closest match was "smile like a plastic daisy sonia levitin" with 56% confidence.)

        * Closest match for "my ea t diaz junot d":
        **Drown** by **Junot Diaz** with 77% confidence.''')

        # publication year distribution viz
        st.image('viz/pub_year_dist.png')

        st.markdown('''It is important to note that book in this dataset were mostly published between 1980-2000,
        so the likelihood of matching goes up significantly for books from this time period.''')

    # recommendation phase details
    if about_options == 'Recommendation':

        st.subheader('Getting recommendations')

        st.markdown('''SoftCopy uses **collaborative filtering** to recommend books
        based off how your selections compare to other users book preferences.
        SoftCopy's recommendation matrix
        is comprised of:''')

        st.markdown('''
        * 134,299 ratings

        * 10,077 books

        * 12,854 unique reviewers
        ''')

        st.image(['viz/user_age_dist.png', 'viz/rating_dist.png'], width=325)

        st.markdown('''Under the hood, SoftCopy uses **singular value decomposition (SVD)**
        to generate 800 abstract features for each book in the Book Crossing data
        based off of how they have been rated by various users. This effectively groups
        the books into categories that can be throught of as something like *genres*.
        ''')

        st.markdown('Illustration of how SVD works:')

        st.image('images/svd.png', width=500)

        st.markdown('''
        The recommendation engine then uses these 800 properties to **compare the cosine similarity** each of your
        books to each book in the database and rank them in order of highest similarity.

        This process does not use any content-level data about your book such as language,
        length, or author to make a recommendation. **Your recommendations are strictly
        generated by collaborative filtering based on user ratings and SVD features!**
        ''')

# starts app
if __name__ == "__main__":
    main()


import dill as pickle
from datetime import datetime
import os
from constants import FUGENELEMENTE_FREQUENCY_ORDER

print(f'pid: {os.getpid()}')

def load_year(year):
    t1 = datetime.now()
    ycs = pickle.load(open(f'dumps/unfuge_candidates_added/{year}ycs', 'rb'))
    t2 = datetime.now()
    print(f'time to load {year}: {t2 - t1}')
    return ycs

years = {}

for year in range(1995, 2023):
    print(f'counting year: {year}')
    years[year] = {}
    ycs = load_year(year)

    for yc in ycs:
        matches = yc.lemma_matches
        for match in matches:
            for key in match.keys():
                if key not in years[year].keys():
                    years[year][key] = 0
                years[year][key] += 1

# for year, matchdict in years.items():
#     print(f'results for year: {year}')
#     for key, count in matchdict.items():
#         print(f'{key}: {count}')
#     print('\n')

# add the top row
table = [['year']]
for op in FUGENELEMENTE_FREQUENCY_ORDER:
    table[0].append(op)

for year, matchdict in years.items():
    entry = [year]
    for op in FUGENELEMENTE_FREQUENCY_ORDER:
        if op not in matchdict.keys():
            entry.append(0)
        else:
            entry.append(matchdict[op])
    table.append(entry)

csv_string = ''
for entry in table:
    csv_string += ','.join(str(s) for s in entry) + '\n'

outfile = open('lemma_strat_counts.csv', 'w')
outfile.write(csv_string)


#!/usr/bin/env python
from __future__ import print_function

import simulator.OfflineLogConverter as ConverterModule

def convert_log(class_name, result_file, output_path):

    Converter = getattr(ConverterModule, class_name)

    converter = Converter(result_file)

    with open(output_path, 'w') as output_file:
        for line in converter:
            print(line, file=output_file)

if __name__ == "__main__":
    import argparse
    import sys

    parser = argparse.ArgumentParser(description="Result Converter", add_help=True)
    parser.add_argument("name", type=str, help="The name of the converter clss")
    parser.add_argument("-in", "--result-file", type=str, required=True, help="The location of the result file.")
    parser.add_argument("-out", "--output-file", type=str, required=True, help="The location of the converted result file.")

    args = parser.parse_args(sys.argv[1:])

    convert_log(args.name, args.result_file, args.output_file)


import numpy as np
import tensorflow as tf
from PIL import Image

def anlysis(path):
    labels = open('labels.txt', "r").readlines()

    # Load the TFLite model
    interpreter = tf.lite.Interpreter(model_path=r"model1.tflite")
    interpreter.allocate_tensors()

    # Get input and output tensors
    input_details = interpreter.get_input_details()
    output_details = interpreter.get_output_details()

    # Load the image
    image = Image.open(r'%s'%path).convert("RGB")
    image = image.resize((input_details[0]['shape'][1], input_details[0]['shape'][2]))

    # Prepare input data
    input_data = np.expand_dims(image, axis=0)
    input_data = input_data.astype(np.float32)

    # Set the input tensor
    interpreter.set_tensor(input_details[0]['index'], input_data)

    # Run inference
    interpreter.invoke()

    # Get the output tensor
    output_data = interpreter.get_tensor(output_details[0]['index'])

    # Process the output
    prediction = np.argmax(output_data[0])
    return labels[prediction-1]


from collections import deque
def solution(cacheSize, cities):
    answer = 0
    queue=deque([])
    new_city=[]
    for i in cities:
        new_city.append(i.lower())

    for i in range(len(new_city)):
        if cacheSize==0:
            return len(new_city)*5

        if cacheSize==len(queue):
            if new_city[i] in queue:
                answer+=1
                temp=new_city[i]
                queue.remove(temp)
                queue.append(temp)
            else:
                answer+=5
                queue.popleft()
                queue.append(new_city[i])
        else:
            if new_city[i] in queue:
                answer+=1
                temp=new_city[i]
                queue.remove(temp)
                queue.append(temp)
            else:
                answer+=5
                queue.append(new_city[i])
    return answer

#모범풀이 deque의 maxlen을 지정해버리면 초과시 자동으로 맨 앞에꺼 삭제함
def solution(cacheSize, cities):
    cache = deque(maxlen=cacheSize)
    time = 0
    for i in cities:
        s = i.lower()
        if s in cache:
            cache.remove(s)
            cache.append(s)
            time += 1
        else:
            cache.append(s)
            time += 5
    return time


import logging

def get_logger(name):
    '''Sets logger format and level'''
    console_format = logging.Formatter(
        '%(asctime)s~%(levelname)s: %(message)s')
    console_handler = logging.StreamHandler()
    console_handler.setLevel(logging.DEBUG)
    console_handler.setFormatter(console_format)

    logger = logging.getLogger(name)
    logger.addHandler(console_handler)
    logger.setLevel(logging.DEBUG)
    return logger


from elasticsearch import Elasticsearch
from time import sleep
from bs4 import BeautifulSoup
import re, requests
class DataCollector:
    def __init__(self, name, host="192.168.1.3:9200"):
        self.es = Elasticsearch(hosts=host)
        self.name = name
        self.urls = []
        self.lasturl = ""
    def check_index(self, opt=True):
        if not opt and self.es.indices.exists(self.name):
            print("# 해당 인덱스 초기화를 위한 삭제")
            self.es.indices.delete(self.name)
        if not self.es.indices.exists(self.name):
            print("# 해당 인덱스 미존재")
            settings = {
                "index": {
                "analysis": {
                    "tokenizer": {
                    "nori_tokenizer_mixed": {
                        "type": "nori_tokenizer",
                        "decompound_mode": "mixed"
                    }
                    },
                    "analyzer": {
                    "korean": {
                        "type": "custom",
                        "tokenizer": "nori_tokenizer_mixed",
                        "filter": [
                        "nori_readingform",
                        "lowercase",
                        "nori_part_of_speech_basic"
                        ]
                    }
                    },
                    "filter": {
                    "nori_part_of_speech_basic": {
                        "type": "nori_part_of_speech",
                        "stoptags": ["E", "IC", "J",
                                    "MAG", "MAJ", "MM",
                                    "SP", "SSC", "SSO", "SC", "SE",
                                    "XPN", "XSA", "XSN", "XSV",
                                    "UNA", "NA", "VSV"]
                    }
                    }
                }
                }
            }
            body = {
            "settings": settings,
            "mappings" : {"properties" :
                            {"title" : {"type" : "text", "analyzer" : "korean"},
                                "body" : {"type" : "text", "analyzer" : "korean"},
                                "date" : {"type": "date", "format": "yyyy-MM-dd"}
                            }}
            }
            self.es.indices.create(index=self.name, body=body)#인덱스 생성
            print("# 해당 인덱스 생성 완료")
    def scrape_urls(self, url, pages=10):#page는 디폴트로 10
        print("# URL 수집 시작")
        self.lastnum = self.es.count(index=self.name, body={"query" : {"match_all" : {}}})['count']
        if not self.lastnum == 0:
            self.lasturl = self.es.get(index=self.name, id=self.lastnum)['_source']['url']
            print(self.lasturl)
        for i in range(1, pages+1):
            print("Page", i)
            sleep(0.1)
            for soup in BeautifulSoup(requests.get(url, params={"p" : i}).text, "lxml").select(".tr a"):
                url = re.search(r'\S+/\d+/\d+', soup['href']).group()
                if url == self.lasturl:
                    print("# 이미 수집된 URL => 종료")
                    return
                self.urls.append(url)
        print("# URL 수집 완료")
    def scrape_bodies(self):
        if len(self.urls) == 0 :
            print("# 우선 URL 수집을 진행해주세요!")
            return
        bulk_body = []
        for i, el in enumerate(reversed(self.urls)):
            print(i, el)
            sleep(0.1)
            bulk_body.append({"create" : {"_index" : self.name, "_id" : self.lastnum + i + 1}})
            soup = BeautifulSoup(requests.get(el).text, "lxml")
            bulk_body.append({"url" : el,
                        "community" : re.search(r'(.+) \:', soup.select_one("title").text.strip()).group(1),
                        "title" : soup.select_one(".articleTitle").text.strip(),
                        "date" : re.search(r'\d{4}-\d{2}-\d{2}', soup.select_one(".articleDate").text.strip()).group(),
                        "body" : re.sub(r'\xa0', "", soup.select_one("#powerbbsContent").text.strip()),
                        "hits" : int(re.search(r'[0-9,]+', soup.select_one(".articleHit").text).group().replace(",", "")),
                        "commments" : int(re.search(r'[0-9,]+', soup.select_one(".articleBottomMenu").text).group().replace(",", "")),
                        "recommand" : int(re.search(r'[0-9,]+', soup.select_one(".reqnum").text).group().replace(",", ""))
                        })
            if (i + 1) % 100 == 0:
                self.es.bulk(index=self.name, body=bulk_body)
                self.es.indices.refresh(index=self.name)
                print("# 중간 저장")
                bulk_body.clear()
        if len(bulk_body) != 0:
            self.es.bulk(index=self.name, body=bulk_body)
            self.es.indices.refresh(index=self.name)
        print("# 최종 저장")

collector = DataCollector(name="community_data")
collector.check_index(opt=False)
collector.scrape_urls("http://www.inven.co.kr/board/lineagem/5019", pages=60)
collector.scrape_bodies()


from src.load_data import load_data
from src.models.cnn import model_1
from keras.callbacks import ModelCheckpoint
import os
import pickle

continue_training = True

# Drive
drive_data_path = '/content/drive/My Drive/abdou/emotion_detection/fer2013.csv'

# Local
local_data_path = '../data/fer2013/fer2013.csv'
local_data_path2 = 'data/fer2013/fer2013.csv'

data_path_r = None
if os.path.isfile(local_data_path2):
    data_path_r = local_data_path2
elif os.path.isfile(local_data_path):
    data_path_r = local_data_path
else:
    data_path_r = drive_data_path

img_size = (48, 48)

x_train, y_train, x_val, y_val, x_test, y_test = load_data(data_path_r)

model = model_1()
model.summary()
save_path = 'save_model.ckpt'
save_model_callback = ModelCheckpoint(filepath=save_path, verbose=1)
if continue_training and os.path.isfile(save_path):
    model.load_weights(save_path)

history = model.fit(x_train, y_train, validation_data=(x_test, y_test), epochs=50,
                    batch_size=100, callbacks=[save_model_callback])

pickle.dump(history, open("history.bin", "wb"))

save_path_2 = "save_model_2.ckpt"
model.save(save_path_2)


import logging

import gevent

from volttron.platform.agent.known_identities import PLATFORM_ALERTER
from volttron.platform.agent import utils
from volttron.platform.messaging.health import Status, STATUS_BAD
from volttron.platform.vip.agent import Agent, Core, RPC

utils.setup_logging()
_log = logging.getLogger(__name__)

__version__ = '0.4'

class AlertAgent(Agent):
    def __init__(self, config_path, **kwargs):
        super(AlertAgent, self).__init__(**kwargs)
        self.config = utils.load_config(config_path)
        self.group_agent = {}

    @Core.receiver('onstart')
    def onstart(self, sender, **kwargs):
        for group_name, config in self.config.items():
            event = gevent.event.Event()
            group = AlertGroup(group_name, config,
                               address=self.core.address,
                               enable_store=False)
            gevent.spawn(group.core.run, event)
            event.wait()

            self.group_agent[group_name] = group

    @RPC.export
    def watch_topic(self, group, topic, timeout):
        """RPC method

        Listen for a topic to be published within a given
        number of seconds or send alerts.

        :pararm group: Group that should watch the topic.
        :type group: str
        :param topic: Topic expected to be published.
        :type topic: str
        :param timeout: Seconds before an alert is sent.
        :type timeout: int
        """
        group = self.group_agent[group]
        group.watch_topic(topic, timeout)

    @RPC.export
    def watch_device(self, group, topic, timeout, points):
        """RPC method

        Watch a device's ALL topic and expect points. This
        method calls the watch topic method so both methods
        don't need to be called.

        :pararm group: Group that should watch the device.
        :type group: str
        :param topic: Topic expected to be published.
        :type topic: str
        :param timeout: Seconds before an alert is sent.
        :type timeout: int
        :param points: Points to expect in the publish message.
        :type points: [str]
        """
        group = self.group_agent[group]
        group.watch_device(topic, timeout, points)

    @RPC.export
    def ignore_topic(self, group, topic):
        """RPC method

        Remove a topic from agent's watch list. Alerts will no
        longer be sent if a topic stops being published.

        :param group: Group that should ignore the topic.
        :type group: str
        :param topic: Topic to remove from the watch list.
        :type topic: str
        """
        group = self.group_agent[group]
        group.ignore_topic(topic)

class AlertGroup(Agent):
    def __init__(self, group_name, config, **kwargs):
        super(AlertGroup, self).__init__(**kwargs)
        self.group_name = group_name
        self.config = config
        self.wait_time = {}
        self.topic_ttl = {}
        self.point_ttl = {}

    @Core.receiver('onstart')
    def onstart(self, sender, **kwargs):
        _log.info("Listening for alert group {}".format(self.group_name))
        config = self.config
        for topic in config.iterkeys():

            # Optional config option with a list of points that
            # might not be published.
            if type(config[topic]) is dict:
                point_config = config[topic]
                self.watch_device(topic,
                                  point_config["seconds"],
                                  point_config["points"])

            # Default config option
            else:
                timeout = config[topic]
                self.watch_topic(topic, timeout)

    def watch_topic(self, topic, timeout):
        """Listen for a topic to be published within a given
        number of seconds or send alerts.

        :param topic: Topic expected to be published.
        :type topic: str
        :param timeout: Seconds before an alert is sent.
        :type timeout: int
        """
        self.wait_time[topic] = timeout
        self.topic_ttl[topic] = timeout
        self.vip.pubsub.subscribe(peer='pubsub',
                                  prefix=topic,
                                  callback=self.reset_time)
        _log.info("Expecting {} every {} seconds"
                   .format(topic, timeout))

    def watch_device(self, topic, timeout, points):
        """Watch a device's ALL topic and expect points. This
        method calls the watch topic method so both methods
        don't need to be called.

        :param topic: Topic expected to be published.
        :type topic: str
        :param timeout: Seconds before an alert is sent.
        :type timeout: int
        :param points: Points to expect in the publish message.
        :type points: [str]
        """
        self.point_ttl[topic] = {}

        for p in points:
            self.point_ttl[topic][p] = timeout

        self.watch_topic(topic, timeout)

    def ignore_topic(self, topic):
        """Remove a topic from the group watchlist

        :param topic: Topic to remove from the watch list.
        :type topic: str
        """
        _log.info("Removing topic {} from watchlist".format(topic))

        self.vip.pubsub.unsubscribe(peer='pubsub',
                                    prefix=topic,
                                    callback=self.reset_time)
        self.point_ttl.pop(topic, None)
        self.topic_ttl.pop(topic, None)
        self.wait_time.pop(topic, None)

    def reset_time(self, peer, sender, bus, topic, headers, message):
        """Callback for topic subscriptions

        Resets the timeout for topics and devices when publishes are received.
        """
        if topic not in self.wait_time:
            found = False
            # if topic isn't in wait time we need to figure out the
            # prefix topic so that we can determine the wait time
            for x in self.wait_time:
                # TODO: order the wait_time topics so furthest down the tree wins.
                if topic.startswith(x):
                    topic = x
                    found = True
                    break
            if not found:
                _log.debug("No configured topic prefix for topic {}".format(
                    topic)
                )
                return

        _log.debug("Resetting timeout for {}".format(topic))

        # Reset the standard topic timeout
        self.topic_ttl[topic] = self.wait_time[topic]

        # Reset timeouts on volatile points
        if topic in self.point_ttl:
            received_points = set(message[0].keys())
            expected_points = self.point_ttl[topic].keys()
            for point in expected_points:
                if point in received_points:
                    self.point_ttl[topic][point] = self.wait_time[topic]

    @Core.periodic(1)
    def decrement_ttl(self):
        """Periodic call

        Used to maintain the time since each topic's last publish.
        Sends an alert if any topics are missing.
        """
        unseen_topics = set()
        for topic in self.wait_time.iterkeys():

            # Send an alert if a topic hasn't been seen
            self.topic_ttl[topic] -= 1
            if self.topic_ttl[topic] <= 0:
                unseen_topics.add(topic)
                self.topic_ttl[topic] = self.wait_time[topic]

            # Send an alert if a point hasn't been seen
            try:
                points = self.point_ttl[topic].keys()
                for p in points:
                    self.point_ttl[topic][p] -= 1
                    if self.point_ttl[topic][p] <= 0:
                        unseen_topics.add((topic, p))
                        self.point_ttl[topic][p] = self.wait_time[topic]
            except KeyError:
                pass

        if unseen_topics:
            self.send_alert(list(unseen_topics))

    def send_alert(self, unseen_topics):
        """Send an alert for the group, summarizing missing topics.

        :param unseen_topics: List of topics that were expected but not received
        :type unseen_topics: list
        """
        alert_key = "AlertAgent Timeout for group {}".format(self.group_name)
        context = "Topic(s) not published within time limit: {}".format(
            sorted(unseen_topics))

        status = Status.build(STATUS_BAD, context=context)
        self.vip.health.send_alert(alert_key, status)

def main():
    utils.vip_main(AlertAgent, identity=PLATFORM_ALERTER, version=__version__)

if __name__ == '__main__':
    try:
        main()
    except KeyboardInterrupt:
        pass


import logging

from config import settings
from core.base_handler import BaseHandler
from generators.slack import SlackMessageGenerator
from targets.slack import Slack

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

class SlackHandler(BaseHandler):
    def scorecard_reminder(self):
        if not self.entities:
            logger.info("No entities found")
            return

        logger.info(
            f"Generating scorecards reminders for {len(self.entities)} entities"
        )
        blocks = SlackMessageGenerator().scorecard_reminder(
            settings.blueprint, self.scorecard, self.entities
        )
        logger.info("Sending scorecards reminders to slack channel")
        Slack().send_message(blocks)

    def scorecard_report(self):
        if not self.entities:
            logger.info("No entities found")
            return

        logger.info(f"Generating scorecard report for {len(self.entities)} entities")
        blocks = SlackMessageGenerator().scorecard_report(
            settings.blueprint, self.scorecard, self.entities
        )
        logger.info("Sending scorecard report to slack channel")
        Slack().send_message(blocks)


from core import weibo, weibo_v2
from django.core.management.base import BaseCommand

id_list = [
    2694295913,  # 热刺节操巷
    3065243713,  # 托特纳姆热刺足球俱乐部
    2747143811,  # 爆棚仓鼠
]

class Command(BaseCommand):
    def handle(self, *args, **options):
        for user_id in id_list:
            user_info = weibo_v2.get_user_info(user_id)
            author = user_info["screen_name"]
            avatar_url = user_info["avatar_hd"]

            weibo.get_or_create_user(user_id, author, avatar_url)
            self.stdout.write(f"{author} with weibo_id {user_id}")


import networkx as nx
import matplotlib.pyplot as plt

from animal import Animal

def draw_animal_genome(animal):
    G = nx.DiGraph()
    for (nid1, neuron) in animal.brain.neurons.items():
        G.add_node(nid1)
        for (neuron2, w) in neuron.inputs:
            G.add_weighted_edges_from([(neuron2.nid, nid1, w*1000)])
    pos = nx.spring_layout(G, pos=nx.random_layout(G), iterations=200)
    nx.draw_networkx(G, pos=pos, with_labels=True)
    plt.show()

def draw_genome(genome):
    animal = Animal(genome, 1)
    draw_animal_genome(animal)