import os
import pandas as pd
from datetime import date
from datetime import timedelta
import datetime
import requests
from requests.exceptions import MissingSchema
from flask import current_app
import numpy as np
from dashAndData import db
from dashAndData.models import Industrynames,Industryvalues, Commoditynames, Commodityvalues
from dashAndData.modelsCage import Cagecompanies
#added for BLS API call
from sqlalchemy import func
from dateutil.relativedelta import relativedelta
import json

def checkExt(filename):
    _, f_ext = os.path.splitext(filename)
    return f_ext

def allowedFileUtil(filename):
    return '.' in filename and filename.rsplit('.', 1)[1].lower() in ['xlsx', 'csv']

periodsDict = {'M01': 1,'M02': 2,'M03': 3,'M04': 4,'M05': 5,'M06': 6,'M07': 7,
    'M08': 8,'M09': 9,'M10': 10,'M11': 11,'M12': 12,'M13': 13}

def seriesIdTitleListIndustry():
    listOfSeriesIds=db.session.query(Industryvalues.series_id).distinct().all()
    listOfSeriesIds=[i[0] for i in list(listOfSeriesIds)]
    indexNameTableObj = Industrynames.query.filter(Industrynames.series_id.in_(listOfSeriesIds)).all()

    indexSeriesIdTitleList=[]
    for i in indexNameTableObj:
        indexIdTitleList=[i.series_id,i.series_title]
        indexSeriesIdTitleList.append(indexIdTitleList)
    return indexSeriesIdTitleList

def seriesIdTitleListCommodity():
    listOfSeriesIds=db.session.query(Commodityvalues.series_id).distinct().all()
    listOfSeriesIds=[i[0] for i in list(listOfSeriesIds)]
    indexNameTableObj = Commoditynames.query.filter(Commoditynames.series_id.in_(listOfSeriesIds)).all()

    indexSeriesIdTitleList=[]
    for i in indexNameTableObj:
        indexIdTitleList=[i.series_id,i.series_title]
        indexSeriesIdTitleList.append(indexIdTitleList)
    return indexSeriesIdTitleList

def priceIndicesToDf(seriesIdList,dfType):
    listDict={}

    for i in seriesIdList:
        if dfType=='Industry':
            series = Industrynames.query.filter_by(series_id=i).first()
        else:
            series = Commoditynames.query.filter_by(series_id=i).first()

        yearsList = [j.year for j in series.values]
        periodsList = [j.period for j in series.values]
        valuesList = [j.value for j in series.values]
        listDict[i] = [yearsList,periodsList, valuesList]
    dfDict={}
    for i in listDict:
        df=pd.DataFrame(list(zip(listDict[i][0],listDict[i][1],listDict[i][2])), columns=['years','periods','values'])
        df.replace({'periods':periodsDict}, inplace=True)
        df=df[df.periods!=13]
        df['date']=pd.to_datetime(dict(year=df.years,month=df.periods,day=1))

        df.drop(columns=['years','periods'], inplace=True)
        df.set_index(['date'], inplace=True)
        df.rename(columns={'values':i}, inplace=True)
        dfDict[i]=df

    dfExcel=list(dfDict.values())[0]
    if len(seriesIdList)>1:
        for i in dfDict:
            if i!=list(dfDict.keys())[0]:
                dfExcel=pd.merge(dfExcel,dfDict[i], how='outer',left_index=True, right_index=True)

    dfExcel.reset_index(inplace=True)
    dfExcel.sort_values(by=['date'],ascending = False, inplace = True)
    return dfExcel

def formatSeriesIdListUtil(seriesIdList):
    seriesIdList=seriesIdList.split(',')
    seriesIdList2=[]
    for i in seriesIdList:
        j=i.replace('\n','')
        seriesIdList2.append(j)
    return seriesIdList2

def buildMetaDfUtil(seriesIdListClean,metaDataItemsList,dfType):
    metaDict={}
    for j in seriesIdListClean:
        if dfType == 'Industry':
            subList= [Industrynames.query.with_entities(getattr(Industrynames, i)).filter_by(
                series_id=j).first()[0] for i in metaDataItemsList]
        else:
            subList= [Commoditynames.query.with_entities(getattr(Commoditynames, i)).filter_by(
                series_id=j).first()[0] for i in metaDataItemsList]
        metaDict[j] = subList

    metaDict['Meta Data Row Headings']=metaDataItemsList
    metaDf=pd.DataFrame.from_dict(metaDict)

    cols = list(metaDf.columns)
    cols = [cols[-1]] + cols[:-1]
    metaDf = metaDf[cols]

    return metaDf

def makeExcelObj_priceindices(filePathAndName,metaDf,indexValuesDf,seriesIdList,metaDataItemsList,sheetName,
        periodicity):
    excelObj=pd.ExcelWriter(filePathAndName,datetime_format='yyyy-mm-dd')
    metaDf.to_excel(excelObj, sheet_name=sheetName, header=False,index=False)

    #column count and row number for wrapping text and widening
    rowWrap=metaDataItemsList.index('series_title')
    columnCount=len(seriesIdList)+1
    print('periodicity:::', periodicity)
    #adjust column width and make certain cells wrap
    worksheet=excelObj.sheets[sheetName]
    workbook=excelObj.book
    if periodicity == 'annual':
        print('are we getting the right format?')
        indexValuesDf=annualizeDf(indexValuesDf)
        izquerda_date_format_wb = workbook.add_format({'align': 'left'})
        worksheet.set_column(0,0,15,izquerda_date_format_wb)
    elif periodicity == 'quarter':
        indexValuesDf=quarterizeDf(indexValuesDf)
        izquerda_date_format_wb = workbook.add_format({'align': 'left'})
        worksheet.set_column(0,0,15,izquerda_date_format_wb)
    else:
        izquerda_date_format_wb = workbook.add_format({'align': 'left', 'num_format' : 'mm/dd/yyyy'})
        worksheet.set_column(0,0,15,izquerda_date_format_wb)

    indexValuesDf.to_excel(
        excelObj, sheet_name=sheetName,startrow=len(metaDf), header=False, index=False)

    cell_format_wrap_values=workbook.add_format({'align':'center','text_wrap':True, 'num_format':'00.0'})
    cell_format_wrap_meta=workbook.add_format({'align':'center','text_wrap':True})
    wrapRows=[]

    for row in metaDataItemsList:
        if row == 'series_title' or row == 'series_id':
            wrapRows.append(metaDataItemsList.index(row))

    #format date column:
    for row in range(len(metaDf),len(indexValuesDf)+len(metaDf)):
        worksheet.write(row,0,indexValuesDf.iloc[row-len(metaDf),0], izquerda_date_format_wb)
        # worksheet.write(row,0,'year should go here', izquerda_date_format_wb)

    for col in range(1,columnCount):
        # for row in range(len(metaDf),len(indexValuesDf)+len(metaDf)):
        for row in range(len(metaDf),len(indexValuesDf)+len(metaDf)):
            try:
                worksheet.write(row,col,indexValuesDf.iloc[row-len(metaDf),col], cell_format_wrap_values)
            except TypeError:
                #TypeError is the error for trying to format na cells
                # worksheet.write(row,col,'Empty Cell', cell_format_center)
                pass

    # set wrap and width of column if length of cell is beyond an amount
    for row in wrapRows:
        for i in range(1,columnCount):
            if len(str(metaDf.iloc[row,i])) >10:
                worksheet.set_column(i,i,11, cell_format_wrap_meta)
            if len(str(metaDf.iloc[row,i])) >20:
                worksheet.set_column(i,i,15, cell_format_wrap_meta)
            if len(str(metaDf.iloc[row,i])) >83:
                worksheet.set_column(i,i,20, cell_format_wrap_meta)
            if len(str(metaDf.iloc[row,i])) >183:
                worksheet.set_column(i,i,30, cell_format_wrap_meta)

    worksheet.freeze_panes(len(metaDf),1)
    return excelObj

def checkStatusUtility(formDict):
    cs0=formDict.get('series_idCheckbox')
    cs1=formDict.get('industry_codeCheckbox')
    cs2=formDict.get('product_codeCheckbox')
    cs3=formDict.get('seasonalCheckbox')
    cs4=formDict.get('base_dateCheckbox')
    cs5=formDict.get('series_titleCheckbox')
    cs6=formDict.get('footnote_codesCheckbox')
    cs7=formDict.get('begin_yearCheckbox')
    cs8=formDict.get('begin_periodCheckbox')
    cs9=formDict.get('end_yearCheckbox')
    cs10=formDict.get('end_periodCheckbox')
    if formDict.get('periodicity')=="month":
        cs11='checked'
        cs12=None
        cs13=None
    elif formDict.get('periodicity')=="quarter":
        cs11=None
        cs12='checked'
        cs13=None
    else:
        cs11=None
        cs12=None
        cs13='checked'
    return (cs0,cs1,cs2,cs3,cs4,cs5,cs6,cs7,cs8,cs9,cs10,cs11,cs12,cs13)

def annualizeDf(df):
    df['year']=pd.DatetimeIndex(df['date']).year
    df=df.groupby(['year']).mean()
    df.reset_index(inplace=True)
    df.sort_values(by=['year'],ascending = False, inplace = True)
    print('dtypes for annualizedDF:::', df.dtypes)
    return df

def quarterizeDf(df):#probabaly need to add a column that has dates, sort on date, then remove date before return
    quarterDict = {1:'Q1',2:'Q1', 3:'Q1',4:'Q2',5:'Q2',6:'Q2',7:'Q3',8:'Q3',9:'Q3',10:'Q4',11:'Q4',12:'Q4'}
    monthDict= {'Q1':1,'Q2':4,'Q3':7,'Q4':10}
    # monthDict= {1:'Q1',4:'Q2',7:'Q3',10:'Q4'}
    df['quarter'] = pd.DatetimeIndex(df['date']).month
    df['year'] = pd.DatetimeIndex(df['date']).year.astype(str)
    df.replace({'quarter':quarterDict}, inplace=True)
    df['quarter-year'] = df[['quarter','year']].agg('-'.join,axis=1)
    df=df.groupby(['quarter-year']).mean()
    df.reset_index(inplace=True)
    df['year']= df['quarter-year'].str[3:].astype('int32')
    df['month'] = df['quarter-year'].str[0:2].astype('str')
    df['month'].replace(monthDict, inplace = True)
    df['day'] = 1
    df['date'] = pd.to_datetime(df[['year','month','day']])
    df.sort_values(by=['date'],ascending = False, inplace = True)
    df.drop(columns=['year','month','day','date'], inplace = True)
    return df

def makeSearchExactDict(formDict):
    searchStringDict={}
    exactDict={}
    for i,j in formDict.items():
        if i != 'searchCage' and i[-5:] != 'Exact':
            searchStringDict[i]=j
        elif i[-5:] == 'Exact':
            exactDict[i[:-5]]=j
    return (searchStringDict,exactDict)

def searchQueryCageToDf(formDict):
    searchDict={'companyName':'ADRS_NM_C_TXT1', 'companyNameSub':'ADRS_NM_C_TXT2','cageCode': 'CAGE_code',
        'address': ['ST_ADDRESS1','ST_ADDRESS2'], 'city': 'CITY', 'state': 'ST_US_POSN'}

    searchStringDict, exactDict=makeSearchExactDict(formDict)
    searchQuery = Cagecompanies.query
    for i,j in searchStringDict.items():
        if i == 'address' and exactDict.get(i)=='checked':
            searchQuery=searchQuery.filter(getattr(Cagecompanies, 'ST_ADDRESS1')==j)
        elif i == 'address':
            for n in searchDict[i]:
                searchQuery=searchQuery.filter(getattr(Cagecompanies, n).contains(j))
        elif i != 'searchCage':
            if exactDict.get(i):
                searchQuery=searchQuery.filter(getattr(Cagecompanies, searchDict[i])==j)
            else:
                searchQuery=searchQuery.filter(getattr(Cagecompanies, searchDict[i]).contains(j))

    columnNames=['CAGE','Company Name', 'Company Sub Name', 'Address1', 'Address2','PO Box',
        'City', 'State', 'Zip', 'Country', 'Phone']
    searchQueryDf=pd.DataFrame([(i.CAGE_code,i.ADRS_NM_C_TXT1,i.ADRS_NM_C_TXT2,i.ST_ADDRESS1,i.ST_ADDRESS2,
                            i.PO_BOX,i.CITY,i.ST_US_POSN, i.ZIP_CODE,i.COUNTRY,i.TELEPHONE_NUMBER) for i in searchQuery],
                           columns=columnNames)

    return searchQueryDf

def cageExcelObjUtil(filePathAndName,df):
    excelObj=pd.ExcelWriter(filePathAndName,datetime_format='yyyy-mm-dd')
    sheetName='CAGE Search'
    dfHeader = pd.DataFrame([list(df.columns)])
    dfHeader.to_excel(excelObj,sheet_name=sheetName,header=False,index=False)
    df.to_excel(excelObj, sheet_name=sheetName,startrow=len(dfHeader), header=False,index=False)
    #adjust column width
    worksheet=excelObj.sheets[sheetName]
    worksheet.set_column(1,1,20)
    worksheet.set_column(3,3,20)
    worksheet.set_column(6,6,15)
    worksheet.set_column(7,7,5)
    worksheet.set_column(10,10,12)
    workbook=excelObj.book
    worksheet.freeze_panes(1,0)
    return excelObj

def checkDbForExistingData(seriesIdListClean, table_name):
    for table in db.Model.__subclasses__():
        if table.__tablename__ == table_name:
            database_to_search = table

    api_search_list =[]
    for series in seriesIdListClean:
        max_year = db.session.query(func.max(database_to_search.year)).filter(
            database_to_search.series_id == seriesIdListClean[0]).first()[0]
        id_list = db.session.query(database_to_search.id, database_to_search.period).filter(
            database_to_search.series_id == seriesIdListClean[0],database_to_search.year == max_year).all()

        x=0
        for i in id_list:
            if int(i[1][1:3]) > x:
                x = int(i[1][1:3])
                id_of_max_period_from_series_id = i[0]#<----id_of_max_period_from_series_id is the
        #database_to_search.id for the most current entry of theseries_id in the database.

        month_value, year_value = db.session.query(database_to_search).filter(
            database_to_search.id == id_of_max_period_from_series_id).with_entities(
            database_to_search.period, database_to_search.year).first()
        month_value = int(month_value[1:])
        db_date = datetime.datetime.strptime(f'{year_value}-{month_value}-1','%Y-%m-%d').date()

        # BLS release dates vary between the 9th and 17th of the month. Here i'm assuming
        #all db is current up to 2 months and 13 days behind the current date. Or
        #up to the 14th of the two months following the month in the db.
        #***idea: put release schedule in a dictionary and update db_date_is_good_until based on schedule.
        db_date_is_good_until = db_date +relativedelta(months=2)+relativedelta(days=13)

        current_date = datetime.datetime.now().date()
        if current_date >db_date_is_good_until:
            api_search_list.append(series)
    return (db_date_is_good_until, api_search_list)

def blsApiCall(db_date_is_good_until, api_search_list):
    headers = {'Content-type': 'application/json'}
    startYear=datetime.datetime.now().date() + relativedelta(years=-3)
    startYear=startYear.year
    endYear=db_date_is_good_until.year
    data = json.dumps({"seriesid": api_search_list,"startyear":startYear,
                       "endyear":endYear,
                       'registrationkey':current_app.config['REGISTRATION_KEY']
                      })
    bls_pull = requests.post(current_app.config['BLS_API_URL'], data=data, headers=headers)
    print('API Status code:',bls_pull.status_code)
    if bls_pull.status_code != 200:
        return print('API call did not go through. Status is something other than 200.')
    json_data=json.loads(bls_pull.text)
    return json_data

def makeDfDictionary(api_search_list, json_data):
    df_dict={}
    for series in api_search_list:
        series_id_list = []
        year_list =[]
        period_list =[]
        value_list =[]
        footnote_codes_list = []
        series_count = api_search_list.index(series)
        print(series)
        print('makeDfDictionary -- json_data:::',json_data)
        try:
            for record in json_data['Results']['series'][series_count]['data']:
                series_id_list.append(json_data['Results']['series'][series_count]['seriesID'])
                year_list.append(record['year'])
                period_list.append(record['period'])
                value_list.append(record['value'])
                footnote_codes_list.append(record['footnotes'][0].get('code'))
            df_dict[series]=pd.DataFrame(list(zip(series_id_list,year_list,period_list,value_list,
                                                 footnote_codes_list)), columns=['series_id','year','period','value','footnote_codes'])
        except:
            print(series, 'has current data to update')

    for df in df_dict.values():
        df['id']=df.index
        df.set_index('id', inplace = True)

    return df_dict

def deleteOldRows(table_name, df_dict):
    import sqlalchemy as sa
    meta = sa.MetaData()
    # table_name = 'commodityvalues'
    for df in df_dict.values():
        sa_table = sa.Table(table_name, meta, autoload=True, autoload_with=db.engine)
        cond = df.apply(lambda row: sa.and_(sa_table.c['series_id'] == row['series_id'],
                                                sa_table.c['year'] == row['year'],
                                                sa_table.c['period'] == row['period']), axis=1)
        cond = sa.or_(*cond)

        # Define and execute the DELETE
        delete = sa_table.delete().where(cond)
        with db.engine.connect() as conn:
            conn.execute(delete)

        # Now you can insert the new data
        df.to_sql(table_name, db.engine, if_exists='append', index=False)

def updateDbWithApi(seriesIdListClean, table_name):

    db_date_is_good_until, api_search_list = checkDbForExistingData(seriesIdListClean, table_name)
    print('successfully checked Db for Existing Data. The follwing indicies need updating: ',api_search_list)
    if len(api_search_list)>0:
        json_data = blsApiCall(db_date_is_good_until, api_search_list)
        df_dict = makeDfDictionary(api_search_list, json_data)
        print('succesfully created df_dict')
        deleteOldRows(table_name, df_dict)
        print('Succesfully update! Appeneded ', table_name,' for :',api_search_list)
    else:
        print('All requested series_ids already up to date. No BLS API call necessary.')


import typing  # noqa: F401

from kubernetes import client  # noqa: F401
from kuber import kube_api as _kube_api  # noqa: F401

from kuber import definitions as _kuber_definitions  # noqa: F401
from kuber import _types  # noqa: F401
from kuber.v1_17.meta_v1 import ListMeta  # noqa: F401
from kuber.v1_17.meta_v1 import ObjectMeta  # noqa: F401

class AuditSink(_kuber_definitions.Resource):
    """
    AuditSink represents a cluster level audit sink
    """

    def __init__(
        self,
        metadata: "ObjectMeta" = None,
        spec: "AuditSinkSpec" = None,
    ):
        """Create AuditSink instance."""
        super(AuditSink, self).__init__(
            api_version="auditregistration/v1alpha1", kind="AuditSink"
        )
        self._properties = {
            "metadata": metadata if metadata is not None else ObjectMeta(),
            "spec": spec if spec is not None else AuditSinkSpec(),
        }
        self._types = {
            "apiVersion": (str, None),
            "kind": (str, None),
            "metadata": (ObjectMeta, None),
            "spec": (AuditSinkSpec, None),
        }

    @property
    def metadata(self) -> "ObjectMeta":
        """ """
        return typing.cast(
            "ObjectMeta",
            self._properties.get("metadata"),
        )

    @metadata.setter
    def metadata(self, value: typing.Union["ObjectMeta", dict]):
        """ """
        if isinstance(value, dict):
            value = typing.cast(
                ObjectMeta,
                ObjectMeta().from_dict(value),
            )
        self._properties["metadata"] = value

    @property
    def spec(self) -> "AuditSinkSpec":
        """
        Spec defines the audit configuration spec
        """
        return typing.cast(
            "AuditSinkSpec",
            self._properties.get("spec"),
        )

    @spec.setter
    def spec(self, value: typing.Union["AuditSinkSpec", dict]):
        """
        Spec defines the audit configuration spec
        """
        if isinstance(value, dict):
            value = typing.cast(
                AuditSinkSpec,
                AuditSinkSpec().from_dict(value),
            )
        self._properties["spec"] = value

    def create_resource(self, namespace: "str" = None):
        """
        Creates the AuditSink in the currently
        configured Kubernetes cluster.
        """
        names = ["create_namespaced_audit_sink", "create_audit_sink"]

        _kube_api.execute(
            action="create",
            resource=self,
            names=names,
            namespace=namespace,
            api_client=None,
            api_args={"body": self.to_dict()},
        )

    def replace_resource(self, namespace: "str" = None):
        """
        Replaces the AuditSink in the currently
        configured Kubernetes cluster.
        """
        names = ["replace_namespaced_audit_sink", "replace_audit_sink"]

        _kube_api.execute(
            action="replace",
            resource=self,
            names=names,
            namespace=namespace,
            api_client=None,
            api_args={"body": self.to_dict(), "name": self.metadata.name},
        )

    def patch_resource(self, namespace: "str" = None):
        """
        Patches the AuditSink in the currently
        configured Kubernetes cluster.
        """
        names = ["patch_namespaced_audit_sink", "patch_audit_sink"]

        _kube_api.execute(
            action="patch",
            resource=self,
            names=names,
            namespace=namespace,
            api_client=None,
            api_args={"body": self.to_dict(), "name": self.metadata.name},
        )

    def get_resource_status(self, namespace: "str" = None):
        """This resource does not have a status."""
        pass

    def read_resource(self, namespace: str = None):
        """
        Reads the AuditSink from the currently configured
        Kubernetes cluster and returns the low-level definition object.
        """
        names = [
            "read_namespaced_audit_sink",
            "read_audit_sink",
        ]
        return _kube_api.execute(
            action="read",
            resource=self,
            names=names,
            namespace=namespace,
            api_client=None,
            api_args={"name": self.metadata.name},
        )

    def delete_resource(
        self,
        namespace: str = None,
        propagation_policy: str = "Foreground",
        grace_period_seconds: int = 10,
    ):
        """
        Deletes the AuditSink from the currently configured
        Kubernetes cluster.
        """
        names = [
            "delete_namespaced_audit_sink",
            "delete_audit_sink",
        ]

        body = client.V1DeleteOptions(
            propagation_policy=propagation_policy,
            grace_period_seconds=grace_period_seconds,
        )

        _kube_api.execute(
            action="delete",
            resource=self,
            names=names,
            namespace=namespace,
            api_client=None,
            api_args={"name": self.metadata.name, "body": body},
        )

    @staticmethod
    def get_resource_api(
        api_client: client.ApiClient = None, **kwargs
    ) -> "client.AuditregistrationV1alpha1Api":
        """
        Returns an instance of the kubernetes API client associated with
        this object.
        """
        if api_client:
            kwargs["apl_client"] = api_client
        return client.AuditregistrationV1alpha1Api(**kwargs)

    def __enter__(self) -> "AuditSink":
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        return False

class AuditSinkList(_kuber_definitions.Collection):
    """
    AuditSinkList is a list of AuditSink items.
    """

    def __init__(
        self,
        items: typing.List["AuditSink"] = None,
        metadata: "ListMeta" = None,
    ):
        """Create AuditSinkList instance."""
        super(AuditSinkList, self).__init__(
            api_version="auditregistration/v1alpha1", kind="AuditSinkList"
        )
        self._properties = {
            "items": items if items is not None else [],
            "metadata": metadata if metadata is not None else ListMeta(),
        }
        self._types = {
            "apiVersion": (str, None),
            "items": (list, AuditSink),
            "kind": (str, None),
            "metadata": (ListMeta, None),
        }

    @property
    def items(self) -> typing.List["AuditSink"]:
        """
        List of audit configurations.
        """
        return typing.cast(
            typing.List["AuditSink"],
            self._properties.get("items"),
        )

    @items.setter
    def items(self, value: typing.Union[typing.List["AuditSink"], typing.List[dict]]):
        """
        List of audit configurations.
        """
        cleaned: typing.List[AuditSink] = []
        for item in value:
            if isinstance(item, dict):
                item = typing.cast(
                    AuditSink,
                    AuditSink().from_dict(item),
                )
            cleaned.append(typing.cast(AuditSink, item))
        self._properties["items"] = cleaned

    @property
    def metadata(self) -> "ListMeta":
        """ """
        return typing.cast(
            "ListMeta",
            self._properties.get("metadata"),
        )

    @metadata.setter
    def metadata(self, value: typing.Union["ListMeta", dict]):
        """ """
        if isinstance(value, dict):
            value = typing.cast(
                ListMeta,
                ListMeta().from_dict(value),
            )
        self._properties["metadata"] = value

    @staticmethod
    def get_resource_api(
        api_client: client.ApiClient = None, **kwargs
    ) -> "client.AuditregistrationV1alpha1Api":
        """
        Returns an instance of the kubernetes API client associated with
        this object.
        """
        if api_client:
            kwargs["apl_client"] = api_client
        return client.AuditregistrationV1alpha1Api(**kwargs)

    def __enter__(self) -> "AuditSinkList":
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        return False

class AuditSinkSpec(_kuber_definitions.Definition):
    """
    AuditSinkSpec holds the spec for the audit sink
    """

    def __init__(
        self,
        policy: "Policy" = None,
        webhook: "Webhook" = None,
    ):
        """Create AuditSinkSpec instance."""
        super(AuditSinkSpec, self).__init__(
            api_version="auditregistration/v1alpha1", kind="AuditSinkSpec"
        )
        self._properties = {
            "policy": policy if policy is not None else Policy(),
            "webhook": webhook if webhook is not None else Webhook(),
        }
        self._types = {
            "policy": (Policy, None),
            "webhook": (Webhook, None),
        }

    @property
    def policy(self) -> "Policy":
        """
        Policy defines the policy for selecting which events should
        be sent to the webhook required
        """
        return typing.cast(
            "Policy",
            self._properties.get("policy"),
        )

    @policy.setter
    def policy(self, value: typing.Union["Policy", dict]):
        """
        Policy defines the policy for selecting which events should
        be sent to the webhook required
        """
        if isinstance(value, dict):
            value = typing.cast(
                Policy,
                Policy().from_dict(value),
            )
        self._properties["policy"] = value

    @property
    def webhook(self) -> "Webhook":
        """
        Webhook to send events required
        """
        return typing.cast(
            "Webhook",
            self._properties.get("webhook"),
        )

    @webhook.setter
    def webhook(self, value: typing.Union["Webhook", dict]):
        """
        Webhook to send events required
        """
        if isinstance(value, dict):
            value = typing.cast(
                Webhook,
                Webhook().from_dict(value),
            )
        self._properties["webhook"] = value

    def __enter__(self) -> "AuditSinkSpec":
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        return False

class Policy(_kuber_definitions.Definition):
    """
    Policy defines the configuration of how audit events are
    logged
    """

    def __init__(
        self,
        level: str = None,
        stages: typing.List[str] = None,
    ):
        """Create Policy instance."""
        super(Policy, self).__init__(
            api_version="auditregistration/v1alpha1", kind="Policy"
        )
        self._properties = {
            "level": level if level is not None else "",
            "stages": stages if stages is not None else [],
        }
        self._types = {
            "level": (str, None),
            "stages": (list, str),
        }

    @property
    def level(self) -> str:
        """
        The Level that all requests are recorded at. available
        options: None, Metadata, Request, RequestResponse required
        """
        return typing.cast(
            str,
            self._properties.get("level"),
        )

    @level.setter
    def level(self, value: str):
        """
        The Level that all requests are recorded at. available
        options: None, Metadata, Request, RequestResponse required
        """
        self._properties["level"] = value

    @property
    def stages(self) -> typing.List[str]:
        """
        Stages is a list of stages for which events are created.
        """
        return typing.cast(
            typing.List[str],
            self._properties.get("stages"),
        )

    @stages.setter
    def stages(self, value: typing.List[str]):
        """
        Stages is a list of stages for which events are created.
        """
        self._properties["stages"] = value

    def __enter__(self) -> "Policy":
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        return False

class ServiceReference(_kuber_definitions.Definition):
    """
    ServiceReference holds a reference to Service.legacy.k8s.io
    """

    def __init__(
        self,
        name: str = None,
        namespace: str = None,
        path: str = None,
        port: int = None,
    ):
        """Create ServiceReference instance."""
        super(ServiceReference, self).__init__(
            api_version="auditregistration/v1alpha1", kind="ServiceReference"
        )
        self._properties = {
            "name": name if name is not None else "",
            "namespace": namespace if namespace is not None else "",
            "path": path if path is not None else "",
            "port": port if port is not None else None,
        }
        self._types = {
            "name": (str, None),
            "namespace": (str, None),
            "path": (str, None),
            "port": (int, None),
        }

    @property
    def name(self) -> str:
        """
        `name` is the name of the service. Required
        """
        return typing.cast(
            str,
            self._properties.get("name"),
        )

    @name.setter
    def name(self, value: str):
        """
        `name` is the name of the service. Required
        """
        self._properties["name"] = value

    @property
    def namespace(self) -> str:
        """
        `namespace` is the namespace of the service. Required
        """
        return typing.cast(
            str,
            self._properties.get("namespace"),
        )

    @namespace.setter
    def namespace(self, value: str):
        """
        `namespace` is the namespace of the service. Required
        """
        self._properties["namespace"] = value

    @property
    def path(self) -> str:
        """
        `path` is an optional URL path which will be sent in any
        request to this service.
        """
        return typing.cast(
            str,
            self._properties.get("path"),
        )

    @path.setter
    def path(self, value: str):
        """
        `path` is an optional URL path which will be sent in any
        request to this service.
        """
        self._properties["path"] = value

    @property
    def port(self) -> int:
        """
        If specified, the port on the service that hosting webhook.
        Default to 443 for backward compatibility. `port` should be
        a valid port number (1-65535, inclusive).
        """
        return typing.cast(
            int,
            self._properties.get("port"),
        )

    @port.setter
    def port(self, value: int):
        """
        If specified, the port on the service that hosting webhook.
        Default to 443 for backward compatibility. `port` should be
        a valid port number (1-65535, inclusive).
        """
        self._properties["port"] = value

    def __enter__(self) -> "ServiceReference":
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        return False

class Webhook(_kuber_definitions.Definition):
    """
    Webhook holds the configuration of the webhook
    """

    def __init__(
        self,
        client_config: "WebhookClientConfig" = None,
        throttle: "WebhookThrottleConfig" = None,
    ):
        """Create Webhook instance."""
        super(Webhook, self).__init__(
            api_version="auditregistration/v1alpha1", kind="Webhook"
        )
        self._properties = {
            "clientConfig": client_config
            if client_config is not None
            else WebhookClientConfig(),
            "throttle": throttle if throttle is not None else WebhookThrottleConfig(),
        }
        self._types = {
            "clientConfig": (WebhookClientConfig, None),
            "throttle": (WebhookThrottleConfig, None),
        }

    @property
    def client_config(self) -> "WebhookClientConfig":
        """
        ClientConfig holds the connection parameters for the webhook
        required
        """
        return typing.cast(
            "WebhookClientConfig",
            self._properties.get("clientConfig"),
        )

    @client_config.setter
    def client_config(self, value: typing.Union["WebhookClientConfig", dict]):
        """
        ClientConfig holds the connection parameters for the webhook
        required
        """
        if isinstance(value, dict):
            value = typing.cast(
                WebhookClientConfig,
                WebhookClientConfig().from_dict(value),
            )
        self._properties["clientConfig"] = value

    @property
    def throttle(self) -> "WebhookThrottleConfig":
        """
        Throttle holds the options for throttling the webhook
        """
        return typing.cast(
            "WebhookThrottleConfig",
            self._properties.get("throttle"),
        )

    @throttle.setter
    def throttle(self, value: typing.Union["WebhookThrottleConfig", dict]):
        """
        Throttle holds the options for throttling the webhook
        """
        if isinstance(value, dict):
            value = typing.cast(
                WebhookThrottleConfig,
                WebhookThrottleConfig().from_dict(value),
            )
        self._properties["throttle"] = value

    def __enter__(self) -> "Webhook":
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        return False

class WebhookClientConfig(_kuber_definitions.Definition):
    """
    WebhookClientConfig contains the information to make a
    connection with the webhook
    """

    def __init__(
        self,
        ca_bundle: str = None,
        service: "ServiceReference" = None,
        url: str = None,
    ):
        """Create WebhookClientConfig instance."""
        super(WebhookClientConfig, self).__init__(
            api_version="auditregistration/v1alpha1", kind="WebhookClientConfig"
        )
        self._properties = {
            "caBundle": ca_bundle if ca_bundle is not None else "",
            "service": service if service is not None else ServiceReference(),
            "url": url if url is not None else "",
        }
        self._types = {
            "caBundle": (str, None),
            "service": (ServiceReference, None),
            "url": (str, None),
        }

    @property
    def ca_bundle(self) -> str:
        """
        `caBundle` is a PEM encoded CA bundle which will be used to
        validate the webhook's server certificate. If unspecified,
        system trust roots on the apiserver are used.
        """
        return typing.cast(
            str,
            self._properties.get("caBundle"),
        )

    @ca_bundle.setter
    def ca_bundle(self, value: str):
        """
        `caBundle` is a PEM encoded CA bundle which will be used to
        validate the webhook's server certificate. If unspecified,
        system trust roots on the apiserver are used.
        """
        self._properties["caBundle"] = value

    @property
    def service(self) -> "ServiceReference":
        """
        `service` is a reference to the service for this webhook.
        Either `service` or `url` must be specified.

        If the webhook is running within the cluster, then you
        should use `service`.
        """
        return typing.cast(
            "ServiceReference",
            self._properties.get("service"),
        )

    @service.setter
    def service(self, value: typing.Union["ServiceReference", dict]):
        """
        `service` is a reference to the service for this webhook.
        Either `service` or `url` must be specified.

        If the webhook is running within the cluster, then you
        should use `service`.
        """
        if isinstance(value, dict):
            value = typing.cast(
                ServiceReference,
                ServiceReference().from_dict(value),
            )
        self._properties["service"] = value

    @property
    def url(self) -> str:
        """
        `url` gives the location of the webhook, in standard URL
        form (`scheme://host:port/path`). Exactly one of `url` or
        `service` must be specified.

        The `host` should not refer to a service running in the
        cluster; use the `service` field instead. The host might be
        resolved via external DNS in some apiservers (e.g., `kube-
        apiserver` cannot resolve in-cluster DNS as that would be a
        layering violation). `host` may also be an IP address.

        Please note that using `localhost` or `127.0.0.1` as a
        `host` is risky unless you take great care to run this
        webhook on all hosts which run an apiserver which might need
        to make calls to this webhook. Such installs are likely to
        be non-portable, i.e., not easy to turn up in a new cluster.

        The scheme must be "https"; the URL must begin with
        "https://".

        A path is optional, and if present may be any string
        permissible in a URL. You may use the path to pass an
        arbitrary string to the webhook, for example, a cluster
        identifier.

        Attempting to use a user or basic auth e.g. "user:password@"
        is not allowed. Fragments ("#...") and query parameters
        ("?...") are not allowed, either.
        """
        return typing.cast(
            str,
            self._properties.get("url"),
        )

    @url.setter
    def url(self, value: str):
        """
        `url` gives the location of the webhook, in standard URL
        form (`scheme://host:port/path`). Exactly one of `url` or
        `service` must be specified.

        The `host` should not refer to a service running in the
        cluster; use the `service` field instead. The host might be
        resolved via external DNS in some apiservers (e.g., `kube-
        apiserver` cannot resolve in-cluster DNS as that would be a
        layering violation). `host` may also be an IP address.

        Please note that using `localhost` or `127.0.0.1` as a
        `host` is risky unless you take great care to run this
        webhook on all hosts which run an apiserver which might need
        to make calls to this webhook. Such installs are likely to
        be non-portable, i.e., not easy to turn up in a new cluster.

        The scheme must be "https"; the URL must begin with
        "https://".

        A path is optional, and if present may be any string
        permissible in a URL. You may use the path to pass an
        arbitrary string to the webhook, for example, a cluster
        identifier.

        Attempting to use a user or basic auth e.g. "user:password@"
        is not allowed. Fragments ("#...") and query parameters
        ("?...") are not allowed, either.
        """
        self._properties["url"] = value

    def __enter__(self) -> "WebhookClientConfig":
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        return False

class WebhookThrottleConfig(_kuber_definitions.Definition):
    """
    WebhookThrottleConfig holds the configuration for throttling
    events
    """

    def __init__(
        self,
        burst: int = None,
        qps: int = None,
    ):
        """Create WebhookThrottleConfig instance."""
        super(WebhookThrottleConfig, self).__init__(
            api_version="auditregistration/v1alpha1", kind="WebhookThrottleConfig"
        )
        self._properties = {
            "burst": burst if burst is not None else None,
            "qps": qps if qps is not None else None,
        }
        self._types = {
            "burst": (int, None),
            "qps": (int, None),
        }

    @property
    def burst(self) -> int:
        """
        ThrottleBurst is the maximum number of events sent at the
        same moment default 15 QPS
        """
        return typing.cast(
            int,
            self._properties.get("burst"),
        )

    @burst.setter
    def burst(self, value: int):
        """
        ThrottleBurst is the maximum number of events sent at the
        same moment default 15 QPS
        """
        self._properties["burst"] = value

    @property
    def qps(self) -> int:
        """
        ThrottleQPS maximum number of batches per second default 10
        QPS
        """
        return typing.cast(
            int,
            self._properties.get("qps"),
        )

    @qps.setter
    def qps(self, value: int):
        """
        ThrottleQPS maximum number of batches per second default 10
        QPS
        """
        self._properties["qps"] = value

    def __enter__(self) -> "WebhookThrottleConfig":
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        return False


import argparse
from collections import defaultdict
import pickle
# from ipdb import set_trace
import numpy as np
import os
from sklearn.linear_model import SGDClassifier
from sklearn.naive_bayes import BernoulliNB
from sklearn.neural_network import MLPClassifier
from sklearn.metrics import f1_score, accuracy_score
from sklearn.metrics import precision_recall_fscore_support as score
import sys

from tadat.core import helpers

def get_features(data_path, features):
	with open(data_path,"rb") as fid:
		train_data = pickle.load(fid)
		X = None
		Y = np.array(train_data[1])
		label_dict = train_data[3]
		#remove extension from filename
		data_path = os.path.splitext(data_path)[0]
		#get features
		for ft in features:
			feat_suffix = "-"+ft+".npy"
			x = np.load(data_path+feat_suffix, allow_pickle=True)
			#check if this is a sparse matrix (which have no 'shape')
			try:
				_ = x.shape[0]
			except IndexError:
				x = x.item()
			if X is None:
				X = x
			else:
				X = np.concatenate((X,x),axis=1)
	return X, Y, label_dict

def hypertune(train, dev, features, obj, hyperparams, res_path=None):
	X_train, Y_train, _ = get_features(train, features)
	X_dev,  Y_dev, _  = get_features(dev, features)
	best_hp = None
	best_score = 0
	for hp in hyperparams:
		#initialize model with the hyperparameters
		model = SGDClassifier(random_state=1234,**hp)
		model.fit(X_train,Y_train)
		Y_hat = model.predict(X_dev)
		score = obj(Y_dev, Y_hat)
		if score > best_score:
			best_score = score
			best_hp = hp
		res = {"score":round(score,3), "hyper":repr(hp)}
		if res_path is not None:
			helpers.save_results(res,res_path)
		helpers.print_results(res)
	print("\n[best conf: {} | score: {}]".format(repr(best_hp),round(best_score,3)))
	return best_hp, best_score

def main(train, test, run_id, features, hyperparameters=None, res_path=None):
	if not hyperparameters: hyperparameters = {}
	#train and evalute model
	if features[0].lower() == "naive_bayes":
		X_train, Y_train, labels = get_features(train, ["bow-bin"])
		X_test,  Y_test, labels  = get_features(test, ["bow-bin"])
		model = BernoulliNB()
		model_name = "NaiveBayes"
	elif features[0].lower() == "mlp":
		X_train, Y_train, labels = get_features(train, ["bow-bin"])
		X_test,  Y_test, labels  = get_features(test, ["bow-bin"])
		model = MLPClassifier(solver='lbfgs', activation="logistic", hidden_layer_sizes=[400])
		model_name = "MLP"
	elif features[0].lower() == "mlp-2":
		X_train, Y_train, labels = get_features(train, ["bow-bin"])
		X_test,  Y_test, labels  = get_features(test, ["bow-bin"])
		model = MLPClassifier(solver='lbfgs', activation="logistic", hidden_layer_sizes=[400,100])
		model_name = "MLP-2"
	else:
		X_train, Y_train, labels = get_features(train, features)
		X_test,  Y_test, labels  = get_features(test, features)
		#initialize model with the hyperparameters
		model = SGDClassifier(random_state=1234,**hyperparameters)
		model_name = "+".join(features)
	model.fit(X_train,Y_train)
	Y_hat = model.predict(X_test)
	avgF1 = f1_score(Y_test, Y_hat,average="macro")
	acc = accuracy_score(Y_test, Y_hat)
	res = {"acc":round(acc,3), \
			"avgF1":round(avgF1,3),	\
			"model":model_name, \
			"dataset":os.path.basename(test), \
			"run_id":run_id, \
			"train_size":X_train.shape[0], \
			"test_size":X_test.shape[0], \
			"hyper":repr(hyperparameters)}
	cols = ["dataset", "run_id", "acc", "avgF1","hyper"]
	helpers.print_results(res, columns=cols)
	precision, recall, fscore, support = score(Y_test, Y_hat)
	for l, i in labels.items():
		print("\n== {} ==".format(l))
		print('precision: {}'.format(round(precision[i],3)))
		print('recall: {}'.format(round(recall[i],3)))
		print('fscore: {}'.format(round(fscore[i],3)))
		print('support: {}'.format(round(support[i],3)))
		print("")
	if res_path is not None:
		cols = ["dataset", "model", "run_id", "acc", "avgF1"]
		helpers.save_results(res, res_path, sep="\t", columns=cols)
	return res

def get_parser():
	par = argparse.ArgumentParser(description="Document Classifier")
	par.add_argument('-train', type=str, required=True, help='train data')
	par.add_argument('-dev', type=str, help='dev data')
	par.add_argument('-test', type=str, required=True, help='test data')
	par.add_argument('-features', type=str, required=True, nargs='+', help='features')
	par.add_argument('-run_id', type=str, help='run id')
	par.add_argument('-res_path', type=str, help='results file')
	par.add_argument('-silent', action="store_true",help='silent')
	par.add_argument('-pos_label', type=str, default="positive", \
					help='label for the positive class')
	par.add_argument('-neg_label', type=str, default="negative", \
					help='label for the negative class')
	par.add_argument('-cv', type=int, help='crossfold')
	par.add_argument('-hyperparams_path', type=str, default="", help='path to a dictionary of hyperparameters')

	return par

if __name__=="__main__":
	parser = get_parser()
	args = parser.parse_args()
	#open datasets
	#train
	print("[features: {}]".format("+".join(args.features)))
	if args.run_id is None: args.run_id = "+".join(args.features)
	hyper_results_path = None

	hyperparams_grid = []
	if os.path.isfile(args.hyperparams_path):
		assert args.dev is not None, "Need a dev set for hyperparameter search"
		hyperparams_grid = helpers.get_hyperparams(args.hyperparams_path, {})
		print("[tuning hyperparameters from @ {}]".format(args.hyperparams_path))
		if args.res_path is not None:
			fname, _ = os.path.splitext(args.res_path)
			hyper_results_path = fname+"_"+os.path.basename(args.test)+"_hyper.txt"
		else:
			hyper_results_path = None
		scorer = lambda y_true,y_hat: f1_score(y_true, y_hat,average="macro")

	if args.cv is None:
		if len(hyperparams_grid) > 0:
			best_hyper, _ = hypertune(args.train, args.dev, args.features, \
									scorer, hyperparams_grid, res_path=hyper_results_path)
		else:
			best_hyper = {}
		#run model with the best hyperparams
		main(args.train, args.test, args.run_id, args.features, best_hyper, args.res_path)
	else:
		assert args.cv > 2, "need at leat 2 folds for cross-validation"
		results = []
		cv_results_path = None
		if args.res_path is not None:
			#in CV experiments save the results of each fold in an external file
			fname, _ = os.path.splitext(args.res_path)
			cv_results_path = fname+"_"+os.path.basename(args.test)+"_CV.txt"
		#loop through cross-validation folds
		for cv_fold in range(1, args.cv+1):
			if len(hyperparams_grid) > 0:
				best_hyper, _ = hypertune(args.train, args.dev, args.features, \
										scorer, hyperparams_grid, res_path=hyper_results_path)
			else:
				best_hyper = {}
			print(cv_results_path)
			#run model with the best hyperparams
			res = main(args.train+"_"+str(cv_fold), args.test+"_"+str(cv_fold), \
				args.run_id+"_"+str(cv_fold), args.features, best_hyper, cv_results_path)
			results.append(res)

		accs = [res["acc"] for res in results ]
		f1s = [res["avgF1"] for res in results ]

		cv_res = {"acc_mean":round(np.mean(accs),3), \
				"acc_std":round(np.std(accs),3), \
				"avgF1_mean":round(np.mean(f1s),3), \
				"avgF1_std":round(np.std(f1s),3), \
				"model":"+".join(args.features), \
				"dataset":os.path.basename(args.test), \
				"run_id":args.run_id}
		helpers.print_results(cv_res)
		#save the results of each run
		if args.res_path is not None:
			colz = ["dataset", "run_id", "model", "acc_mean","acc_std","avgF1_mean","avgF1_std"]
			helpers.save_results(cv_res, args.res_path, columns=colz)


# Copyright (c) 2016-2018, University of Idaho
# All rights reserved.
#
# Roger Lew (rogerlew@gmail.com)
#
# The project described was supported by NSF award number IIA-1301792
# from the NSF Idaho EPSCoR Program and by the National Science Foundation.

from typing import Tuple, List, Dict, Union
import os
from os.path import join as _join
from os.path import exists as _exists
import math
import json
from collections import Counter
from math import pi, atan2
import warnings

import numpy as np
from scipy.stats import circmean

from wepppy.all_your_base import (
    isfloat,
    read_arc,
    wgs84_proj4,
    centroid_px,
    GeoTransformer
)
from .wepp_top_translator import WeppTopTranslator

_thisdir = os.path.dirname(__file__)
_template_dir = _join(_thisdir, "templates")

def ischannel(topaz_id: int) -> bool:
    return str(topaz_id).endswith('4')

def garbrecht_length(distances: List[List[float]]) -> float:
    """
    calculates the length of a subcatchment from the flowpaths
    contained within the subcatchment. The length is weighted_flowpaths
    by the flowpaths length relative to its area.

    distances should be an array of distances between cells along the
    flowpath (not cumulative distance)

    eq. 3.4 in Thomas Cochrane's Dissertation
    # """
    x = np.array([np.sum(d) for d in distances])
    a = np.array([len(d) for d in distances], dtype=np.float64)
    return float(np.sum(x * a) / np.sum(a))

def cummnorm_distance(distance: List[float]) -> np.array:
    """
    builds and returns cumulative normalized distance array from an array
    of cell-to-cell distances
    """
    assert len(distance) > 0

    if len(distance) == 1:
        assert distance[0] > 0.0
        return np.array([0, 1])

    distance_p = np.cumsum(np.array(distance, np.float64))
    distance_p -= distance_p[0]
    distance_p /= distance_p[-1]
    return distance_p

def representative_normalized_elevations(x: List[float], dy: List[float]) -> List[float]:
    """
    x should be a normed distance array between 0 and 1
    dy is an array of slopes

    returns normalized elevations (relative to the length of x)
    """
    assert len(x) == len(dy), (x, dy)
    assert x[0] == 0.0
    assert x[-1] == 1.0

    # calculate the positions, assume top of hillslope is 0 y
    y = [0.0]
    for i in range(len(dy) - 1):
        step = x[i+1] - x[i]
        y.append(y[-1] - step * dy[i])

    return y

def read_geojson(fname):
    data = json.loads(open(fname).read())

    d = {}
    for feature in data['features']:
        top = feature['properties']['TopazID']
        d[top] = np.array(feature['geometry']['coordinates'][0])

    return d

def interpolate_slp(distances, slopes, max_points):
    _s = np.array(slopes)
    _d = np.array(distances)

    if _s.shape == (1,) and _d.shape == (2,):  # slope is a single cell
        _s = np.array([slopes[0], slopes[0]])

    assert _s.shape == _d.shape, str([_s.shape, _d.shape])

    for i in range(len(_d)-1):
        assert _d[i] < _d[i+1], distances

    assert _d[0] == 0.0
    assert _d[-1] == 1.0

    npts = len(_d)

    # interpolate if there are too many slope points
    if npts > max_points:
        _d2 = np.linspace(0.0, 1.0, max_points)
        _s2 = np.interp(_d2, _d, _s)
        _d, _s = _d2, _s2

    return _d, _s

def write_slp(aspect, width, cellsize, length, slope, distance_p, fp, version=97.3, max_points=19):
    """
    writes a slope file in the 97.3 format for WEPP
    """
    assert isfloat(aspect)
    assert isfloat(width)
    assert isfloat(length)
    assert isfloat(cellsize)

    _d, _s = distance_p, slope

    nofes = 1
    npts = len(_d)
    if npts > max_points:
        _d, _s = interpolate_slp(distance_p, slope, max_points)
        npts = len(_d)

    if version == 97.3:
        _slp = '97.3\n{nofes}\n{aspect} {width}\n{npts} {length}\n{defs} '
        defs = ' '.join(['%0.4f, %0.5f' % ds for ds in zip(_d, _s)])

        fp.write(_slp.format(nofes=nofes, aspect=aspect, width=width,
                             npts=npts, length=length, defs=defs))

    else:
        _slp = '{aspect} {cellsize}\n{npts} {length}\n{defs} \n'
        defs = ' '.join(['%0.4f %0.5f' % ds for ds in zip(_d, _s)])

        fp.write(_slp.format(nofes=nofes, aspect=aspect, cellsize=float(cellsize),
                             npts=npts, length=length, defs=defs))

def _identify_subflows(flowpaths: List[np.array]) -> List[List[int]]:
    """
    given an ndarray of flowpaths flowpath (n, 2) for a subcatchment
    identify which flowpaths travel over the same path down the
    hillslope. These are grouped into subflows of flowpath indices
    """
    # if there is only 1 flowpath then there are no subflows
    if len(flowpaths) == 1:
        return [[0]]

    # create a dictionary of px coord tuples so we can point back to the
    # index of the flowpaths list. This is aligned
    # with slopes and distance.
    fps_d = {}
    for k, fp in enumerate(flowpaths):
        fps_d[k] = list(set((fp[j, 0], fp[j, 1]) for j in range(fp.shape[0])))

    # here we create a list of tuples containing
    # the flow indices and the length of the flows
    # we want to iterate through the flows in
    # descending flow length
    lns = [(k, len(v)) for k, v in fps_d.items()]

    # build an empty list to populate subflows
    subflows = []

    # iterate over the flowpaths. If a subflow is identified as
    # a subflow then add it to the subflows index list
    # otherwise create a new subflow list
    for i, n in sorted(lns, key=lambda y: y[1], reverse=True):
        # unpack the set of pixel coords for the subflow
        fp0 = fps_d[i]

        # this gets set to 1 if fp0 is identified as a subflow
        issub = 0

        for j, fp1_indx_arr in enumerate(subflows):
            # because we are iterating in descending order the
            # first index of each subflow index list will
            # be the longest
            fp1 = fps_d[fp1_indx_arr[0]]

            # is fp0 a subset of fp1
            if fp0 <= fp1:
                subflows[j].append(i)
                issub = 1
                break

        # fp0 is not a subset so create a new subflow list
        if not issub:
            subflows.append([i])

    return subflows

def _weighted_slope_average(flowpaths, slopes, distances, max_points=19):
    """
    calculates weighted slopes based on the flowpaths contained on the hillslope
    """
    # determine longest flowpath
    lengths = [float(np.sum(d)) for d in distances]
    i = int(np.argmax(lengths))
    longest = float(lengths[i])

    # determine number of points to define slope
    num_points = len(distances[i])
    if num_points > max_points:
        num_points = max_points

    if num_points == 1:
        slope = float(slopes[i])
        return [slope, slope], [0.0, 1.0]

    # for each flowpath determine the distance from channel
    # this requires reversing the elements in the distance
    # array and calculating the cumulative sum
    rev_cum_distances = [np.cumsum(d[::-1]) for d in distances]

    # if we did this right, then the distance should equally the longest
    # length
    assert round(longest, 5) == round(rev_cum_distances[i][-1], 5)

    # determine weights for each flowpath
    areas = [f.shape[0] for f in flowpaths]
    kps = np.array([L * a for L, a in zip(lengths, areas)])

    # build an array with equally spaced points to interpolate on
    distance_p = np.linspace(0, longest, num_points)

    # this will hold the weighted slope estimates
    eps = []

    # we will weight the slope at each distance away from the channel
    for d_p in distance_p:
        num = 0  # to hold numerator value
        kpsum = 0  # to hold k_p sum

        for slp, rcd, kp in zip(slopes, rev_cum_distances, kps):
            # reverse slopes
            slp = slp[::-1]

            # we only want to interpolate where the slope is defined
            if d_p - 1e-6 > rcd[-1]:
                continue

            slp_p = np.interp(d_p, rcd, slp)
            num += slp_p * kp
            kpsum += kp

        # store the weighted slope estimate
        eps.append(num / kpsum)

    # normalize distance_p array
    distance_p /= longest

    # reverse weighted slopes and return
    w_slopes = np.array(eps[::-1])

    return w_slopes.flatten().tolist(), distance_p.tolist()

def compute_direction(head: List, tail: List) -> float:
    a = atan2(tail[1] - head[1],
              head[0] - tail[0]) * (180.0 / pi) - 180.0

    if a < 0:
        return a + 360.0

    return a

slope_template = """\
{aspect} {profile_width}
{num_points} {length}
{profile}"""

colordict = {
  0:     "#787878",
  "n":  ["#9afb0c", "#9ddd5e", "#9fc085"],
  "ne": ["#00ad43", "#3dab71", "#72a890"],
  "e":  ["#0068c0", "#5078b6", "#7c8ead"],
  "se": ["#6c00a3", "#77479d", "#8c75a0"],
  "s":  ["#ca009c", "#c04d9c", "#b47ba1"],
  "sw": ["#ff5568", "#e76f7a", "#cb8b8f"],
  "w":  ["#ffab47", "#e2a66c", "#c5a58a"],
  "nw": ["#f4fa00", "#d6db5e", "#bdbf89"]
}

c_slope_breaks = [0.05, 0.15, 0.30]

def slp_asp_color(slope: float, aspect: float) -> str:
    aspect %= 360.0

    i = 0
    for j, brk in enumerate(c_slope_breaks):
        if slope > brk:
            i = j + 1

    if i == 0:
        return colordict[0]

    cat_asp = "n"
    if 22.5 < aspect < 67.5:
        cat_asp = "ne"
    elif aspect < 112.5:
        cat_asp = "e"
    elif aspect < 157.5:
        cat_asp = "se"
    elif aspect < 202.5:
        cat_asp = "s"
    elif aspect < 247.5:
        cat_asp = "sw"
    elif aspect < 292.5:
        cat_asp = "w"
    elif aspect < 337.5:
        cat_asp = "nw"

    return colordict[cat_asp][i-1]

class ChannelRoutingError(Exception):
    """
    THE NETWORK STRUCTURE DELINEATED BY TOPAZ CONTAINS MORE THAN 3 CHANNELS
    AS INPUT TO A SINGLE CHANNEL. WEPP CAN ONLY HANDLE 3 OR LESS CHANNELS
    DRAINING INTO A SINGLE CHANNEL.

    THE CHANNELS SHOULD BE REDELINEATED AFTER ADJUSTING THE MINIMUM CHANNEL
    LENGTH (MCL) AND/OR CRITICAL SOURCE AREA (CSA)
    """

    __name__ = 'ChannelRoutingError'

    def __init__(self):
        pass

class CentroidSummary(object):
    def __init__(self, **kwds):
        self.px = tuple(int(v) for v in kwds['px'])
        self.lnglat = tuple(float(v) for v in kwds['lnglat'])

class SummaryBase(object):
    def __init__(self, **kwds):
        topaz_id = kwds['topaz_id']
        if topaz_id is not None:
            topaz_id = int(topaz_id)
        self.topaz_id = topaz_id

        wepp_id = kwds['wepp_id']
        if wepp_id is not None:
            wepp_id = int(wepp_id)
        self.wepp_id = wepp_id

        self.length = float(kwds['length'])
        self.width = float(kwds['width'])
        self.area = float(kwds['area'])
        self.aspect = float(kwds['aspect'])
        self.direction = float(kwds['direction'])
        self.slope_scalar = float(kwds['slope_scalar'])
        self.color = str(kwds['color'])
        self.centroid = kwds['centroid']
        self.distance_p = tuple(kwds['distance_p'])

        self._max_points = 19

        self.w_slopes = None
        self.slopes = None

    @property
    def num_points(self) -> int:
        npts = len(self.distance_p)
        max_points = self.max_points
        return (npts, max_points)[npts > max_points]

    @property
    def max_points(self) -> int:
        return self._max_points

    @property
    def profile(self) -> str:
        max_points = self.max_points
        distance_p = self.distance_p

        try:
            slopes = self.w_slopes
            if slopes is None:
                slopes = self.slopes
        except AttributeError:
            slopes = self.slopes

        _d, _s = distance_p, slopes

        npts = len(_d)
        if npts > max_points:
            _d, _s = interpolate_slp(_d, _s, max_points)

        return ' '.join(["%0.6f, %0.3f" % (d, s) for d, s in zip(_d, _s)])

    def as_dict(self):
        d = dict(
            fname=self.fname,
            topaz_id=self.topaz_id,
            wepp_id=self.wepp_id,
            length=self.length,
            width=self.width,
            area=self.area,
            aspect=self.aspect,
            direction=self.direction,
            slope_scalar=self.slope_scalar,
            color=self.color,
            centroid=self.centroid.lnglat
        )

        try:
            d['order'] = self.order
            d['channel_type'] = self.channel_type
            d['cell_width'] = self.cell_width
        except:
            pass

        try:
            d['slopes'] = self.slopes
            d['coords'] = self.coords
        except:
            pass

        return d

class HillSummary(SummaryBase):
    def __init__(self, **kwds):
        super(HillSummary, self).__init__(**kwds)
        self.w_slopes = tuple(kwds['w_slopes'])
        self.pourpoint = tuple(kwds['pourpoint'])

    @property
    def fname(self) -> str:
        return 'hill_%i.slp' % self.topaz_id

    @property
    def pourpoint_coord(self):
        return str(tuple([int(v) for v in self.pourpoint])).replace(' ', '')

class ChannelSummary(SummaryBase):
    def __init__(self, **kwds):
        super(ChannelSummary, self).__init__(**kwds)
        self.slopes = tuple(kwds['slopes'])
        self._order = kwds.get('order', None)
        self.isoutlet = kwds['isoutlet']
        self.head = kwds['head']
        self.tail = kwds['tail']
        self.chn_enum = int(kwds['chn_enum'])
        self._chn_type = kwds['chn_type']

    @property
    def head_coord(self):
        return str(tuple([int(v) for v in self.head])).replace(' ', '')

    @property
    def tail_coord(self):
        return str(tuple([int(v) for v in self.tail])).replace(' ', '')

    @property
    def fname(self) -> str:
        return 'chn_%i.slp' % self.topaz_id

    @property
    def order(self) -> int:
        return self._order

    @order.setter
    def order(self, value):
        self._order = int(value)

    @property
    def channel_type(self) -> str:
        return getattr(self, '_chn_type', 'Default')

        # return 'Default'
        # return ('OnEarth 1', 'OnGravel 1', 'OnRock 2')[int([None, 0, 0, 1, 1, 1, 2, 2][self.order])]

    @channel_type.setter
    def channel_type(self, value):
        self._chn_type = value

    @property
    def cell_width(self) -> int:
        return [None, 1, 2, 2, 3, 3, 3, 4][self.order]

class FlowpathSummary(SummaryBase):
    def __init__(self, **kwds):
        super(FlowpathSummary, self).__init__(**kwds)
        self.slopes = tuple(kwds['slopes'])
        self.coords = kwds['coords']

    @property
    def fname(self) -> str:
        return 'flow_%i.slp' % self.topaz_id

class WatershedAbstraction:
    def __init__(self, wd, wat_dir):
        if not _exists(wd):
            raise Exception('Specified working directory does not exist')

        if not _exists(wat_dir):
            raise Exception('Specified wat_dir does not exist')

        self.wd = wd
        self.wat_dir = wat_dir

        # lookup table for the flowpath directions specified in flovec
        #
        # 1     2     3
        #   \   |   /
        #     \ | /
        # 4 <-- o --> 6
        #     / | \
        #   /   |   \
        # 7     8     9
        #
        # the origin for the maps is the top right corner
        # we read in the maps with gdal and transpose so the
        # data is in row major order. This is less consistent
        # with maps but more consistent with numpy
        sqrt2 = 2.0**0.5
        self.paths = {1: ([-1, -1], sqrt2),
                      2: ([0,  -1], 1),
                      3: ([1,  -1], sqrt2),
                      4: ([-1,  0], 1),
                      6: ([0,   1], 1),
                      7: ([-1,  1], sqrt2),
                      8: ([0,   1], 1),
                      9: ([1,   1], 1)}

        # initialize datastructure containing json representation
        # of the abstracted watershed
        self.watershed = dict(hillslopes={}, channels={}, flowpaths={})
        self.hillslope_n = 0
        self.channel_n = 0
        self.impoundment_n = 0
        self._structure = None
        self._linkdata = None
        self._network = None

        bound_fn = _join(wd, 'BOUND.ARC')
        flopat_fn = _join(wd, 'FLOPAT.ARC')
        flovec_fn = _join(wd, 'FLOVEC.ARC')
        fvslop_fn = _join(wd, 'FVSLOP.ARC')
        subwta_fn = _join(wd, 'SUBWTA.ARC')
        relief_fn = _join(wd, 'RELIEF.ARC')
        taspec_fn = _join(wd, 'TASPEC.ARC')
        dnmcnt_inp = _join(wd, 'DNMCNT.INP')

        if not _exists(bound_fn) or \
           not _exists(flopat_fn) or \
           not _exists(flovec_fn) or \
           not _exists(fvslop_fn) or \
           not _exists(subwta_fn) or \
           not _exists(relief_fn) or \
           not _exists(taspec_fn):
            raise Exception('Missing required input file')

        bound, _transform, _proj = read_arc(bound_fn, dtype=np.uint8)
        flopat, _transform, _proj = read_arc(flopat_fn, dtype=np.uint8)
        flovec, _transform, _proj = read_arc(flovec_fn, dtype=np.uint8)
        fvslop, _transform, _proj = read_arc(fvslop_fn, dtype=np.float64)
        subwta, _transform, _proj = read_arc(subwta_fn, dtype=np.int64)
        relief, _transform, _proj = read_arc(relief_fn, dtype=np.float64)
        taspec, _transform, _proj = read_arc(taspec_fn, dtype=np.float64)

        # make sure the datasets are aligned
        assert flopat.shape == flovec.shape
        assert flopat.shape == fvslop.shape
        assert flopat.shape == subwta.shape
        assert flopat.shape == relief.shape
        assert flopat.shape == taspec.shape

        # clamp slope values at 0.001 to prevent internal
        # errors in wepp
        fvslop[np.where(fvslop < 0.001)] = 0.001

        # pull cellsize from transform
        self.transform = _transform
        self.cellsize = _transform[1]
        self.cellsize2 = self.cellsize * self.cellsize
        self.totalarea = float(np.sum(bound)) * self.cellsize2

        # store in class
        self.flopat = flopat
        self.flovec = flovec
        self.fvslop = fvslop
        self.subwta = subwta
        self.relief = relief
        self.taspec = taspec

        # find outlet
        outlet = self._read_outlet(dnmcnt_inp)
        outlet_top_id = subwta[outlet[0], outlet[1]]
        assert str(outlet_top_id).endswith('4'), outlet_top_id

        self.watershed['outlet'] = outlet
        self.outlet_top_id = outlet_top_id
        self.watershed['srs'] = dict(transform=_transform,
                                     projection=_proj)

        self.utmProj = _proj
        self.proj2wgs_transformer = GeoTransformer(src_proj4=self.utmProj, dst_proj4=wgs84_proj4)

        # find the centroid_px for the watershed
        indx, indy = np.where(bound == 1)
        _centroid_px = centroid_px(indx, indy)
        centroid_lnglat = self.px_to_lnglat(*_centroid_px)

        self._centroid = CentroidSummary(px=_centroid_px,
                                         lnglat=centroid_lnglat)

        top_ids = list(set(subwta.flatten().tolist()))
        top_ids.remove(0)
        top_sub_ids = [v for v in top_ids if str(v)[-1] in '123']
        top_chn_ids = [v for v in top_ids if str(v).endswith('4')]

        self.translator = WeppTopTranslator(top_sub_ids, top_chn_ids)

    @property
    def structure(self) -> List[List[int]]:
        return self._structure

    @property
    def linkdata(self) -> Dict[int, Dict[str, List[Union[int, float]]]]:
        return self._linkdata

    @property
    def network(self) -> Dict[int, List[int]]:
        return self._network

    @property
    def centroid(self) -> CentroidSummary:
        return self._centroid

    def abstract(self, wepp_chn_type='Default', verbose=False, warn=False):
        self.abstract_channels(wepp_chn_type=wepp_chn_type, verbose=verbose)
        self.abstract_subcatchments(verbose=verbose, warn=warn)
        self.abstract_structure(verbose=verbose)

    def write_slps(self, channels=1, subcatchments=1, flowpaths=1, cell_width=None):
        """
        Writes slope files to the specified wat_dir. The channels,
        subcatchments, and flowpaths args specify what slope files
        should be written.
        """
        out_dir = self.wat_dir
        if channels:
            self._make_channel_slps(out_dir, cell_width=cell_width)

        if subcatchments:
            self._write_subcatchment_slps(out_dir)

        if flowpaths:
            self._write_flowpath_slps(out_dir)

    def _make_channel_slps(self, out_dir, cell_width=None):
        ws = self.watershed
        translator = self.translator

        chn_enums = ws["channels"].keys()
        chn_enums = sorted([translator.chn_enum(chn_id=v) for v in chn_enums])

        # watershed run requires a slope file defining all of the channels in the
        # 99.1 format. Here we write a combined channel slope file and a slope
        # file for each individual channel
        fp2 = open(_join(out_dir, 'channels.slp'), 'w')
        fp2.write('99.1\n')
        fp2.write('%i\n' % len(chn_enums))

        for chn_enum in chn_enums:
            top = translator.top(chn_enum=chn_enum)
            chn_id = 'chn_%i' % top
            d = ws['channels'][chn_id]

            if cell_width is None:
                _cell_width = d.cell_width
            else:
                _cell_width = cell_width

            slp_fn = _join(out_dir, '%s.slp' % chn_id)
            fp = open(slp_fn, 'w')
            write_slp(d.aspect, d.width, _cell_width, d.length,
                      d.slopes, d.distance_p, fp, 97.3)
            fp.close()

            write_slp(d.aspect, d.width, _cell_width, d.length,
                      d.slopes, d.distance_p, fp2, 99.1)

        fp2.close()

    def _write_subcatchment_slps(self, out_dir):
        ws = self.watershed
        cellsize = self.cellsize

        for sub_id, d in ws['hillslopes'].items():
            slp_fn = _join(out_dir, '%s.slp' % sub_id)
            fp = open(slp_fn, 'w')
            write_slp(d.aspect, d.width, cellsize, d.length,
                      d.w_slopes, d.distance_p, fp, 97.3)
            fp.close()

    def _write_flowpath_slps(self, out_dir):
        ws = self.watershed
        cellsize = self.cellsize

        for sub_id in ws['hillslopes']:
            for fp_id, d in ws['flowpaths'][sub_id].items():
                fp = open(_join(out_dir, fp_id + '.slp'), 'w')
                write_slp(d.aspect, d.width, cellsize, d.length,
                          d.slopes, d.distance_p, fp, 97.3)
                fp.close()

    def px_to_utm(self, x: int, y: int) -> Tuple[float, float]:
        _transform = self.transform
        e = _transform[0] + _transform[1] * x
        n = _transform[3] + _transform[5] * y
        return e, n

    def px_to_lnglat(self, x: int, y: int) -> Tuple[float, float]:
        proj2wgs_transformer = self.proj2wgs_transformer

        e, n = self.px_to_utm(x, y)
        lng, lat = proj2wgs_transformer.transform(e, n)
        assert not np.isinf(lng), (self.transform, x, y, e, n)
        assert not np.isinf(lat)
        return lng, lat

    def _read_netw_tab(self):
        translator = self.translator

        keys = "chnum order row col row1 col1 outr outc chnlen elevvup "\
               "elevdn areaup areadn1 areadn dda node1 node2 node3 node4 "\
               "node5 node6 node7 slopedirect slopesmoothed".split()

        types = [int, int, int, int, int, int, int, int,
                 float, float, float, float, float, float,
                 float, int, int, int, int, int, int, int,
                 float, float]

        wd = self.wd
        subwta = self.subwta

        netw_tab = _join(wd, "NETW.TAB")

        with open(netw_tab) as fid:
            lines = fid.readlines()

        last = [i for i, v in enumerate(lines) if v.strip().startswith("-1")][0]
        lines = lines[27:last]
        lines = [[_t(v) for _t, v in zip(types, l.split())] for l in lines]
        assert all([len(l) == 24 for l in lines])
        data = {l[0]: dict(zip(keys, l)) for l in lines}
        # -> Dict[int, Dict[str, List[Union[int, float]]]]
        network = {}

        for chnum in data:
            head = data[chnum]["col"]-1, data[chnum]["row"]-1
            center = data[chnum]["col1"]-1, data[chnum]["row1"]-1
            tail = data[chnum]["outc"]-1, data[chnum]["outr"]-1

            chn_id0 = subwta[head]
            chn_id1 = subwta[center]
            chnout_id = subwta[tail]

            assert chn_id0 != 0, chn_id0
            assert chn_id1 != 0, chn_id1
            assert chnout_id != 0, chnout_id
            assert chn_id0 == chn_id1
            assert str(chn_id0).endswith("4"), chn_id0
            assert str(chnout_id).endswith("4"), chnout_id
            assert chn_id0 >= chnout_id, "%i %i" % (chnout_id, chn_id0)

            data[chnum]["chn_id"] = chn_id0
            data[chnum]["chnout_id"] = chnout_id

            node_indexes = [data[chnum]["node%i" % i] for i in range(1, 8)]
            node_indexes = [indx for indx in node_indexes if indx != 0]

            for chn_enum in node_indexes:
                if chn_enum > translator.channel_n:
                    continue

                chn_id = "chn_%i" % translator.top(chn_enum=chn_enum)
                if chn_id not in self.watershed["channels"]:
                    self.watershed["channels"][chn_id] = {}

                order = data[chnum]["order"]
                self.watershed["channels"][chn_id].order = order
                # self.watershed["channels"][chn_id].width = self.cellsize / order

            if chnout_id == chn_id0:
                continue

            if chnout_id not in network:
                network[chnout_id] = [chn_id0]
            else:
                network[chnout_id].append(chn_id0)

        if not all([len(L) <= 3 for L in network.values()]):
            raise ChannelRoutingError()

        self._linkdata = data
        self._network = network

    def _read_outlet(self, dnmcnt_inp) -> Tuple[int, int]:
        subwta = self.subwta

        with open(dnmcnt_inp) as fid:
            lines = fid.readlines()

            row = int(lines[98])-1
            col = int(lines[107])-1

            if not str(subwta[col, row]).endswith('4'):
                raise Exception('Identified outlet location is not a channel')

            return col, row

    def _determine_aspect(self, indx, indy) -> float:
        taspec = self.taspec
        rads = np.array(taspec[(indx, indy)]) * pi / 180.0
        return float(circmean(rads) * 180.0 / pi)

    def abstract_channels(self, wepp_chn_type='Default', verbose=False):
        subwta = self.subwta

        # extract the subcatchment and channel ids from the subwta map
        # subwta contains 0 values outside the watershed
        _ids = sorted(list(set(subwta.flatten())))

        # channels end in 4
        chn_ids = [i for i in _ids if str(i).endswith('4') and i > 0]

        n = len(chn_ids)
        if n == 0:
            raise Exception('SUBWTA contains no channels')

        for i, chn_id in enumerate(chn_ids):
            if verbose:
                print('abstracting channel %s (%i of %i)...' % (chn_id, i+1, n))
            self.abstract_channel(chn_id, wepp_chn_type=wepp_chn_type)

        self.channel_n = len(chn_ids)

    def _walk_channel(self, chn_id: int) -> \
            Tuple[np.array, np.array, np.array, np.array, np.array]:
        """
        for a channel specified by chn_id identifies
        the flowpath of the channel from top to bottom,
        the slope along the flowpath and the distance
        between points in the flowpath
        """
        subwta = self.subwta
        relief = self.relief
        flovec = self.flovec
        fvslop = self.fvslop
        paths = self.paths

        # use the subwta map to identify the coordinates the channel
        indx, indy = np.where(subwta == chn_id)

        # Relief contains the elevation values of the terrain.
        # this creates a list of tuples containing indices and elevations
        # the indices are aligned to the indx and indy arrays and are sorted
        # in descending order by elevation. We can't walk down the flovec map
        # because the length of the channel gets exaggerated due to the path
        # wandering in and out of the channel.
        indices = sorted([t for t in enumerate(relief[indx, indy])],
                         key=lambda tup: tup[1], reverse=True)

        # build the flowpath
        flowpath = [[indx[t[0]], indy[t[0]]] for t in indices]

        # follow flovec down hill an additional step to connect
        # to next channel
        c, r = flowpath[-1]
        vec = flovec[c, r]
        (x, y), dis = paths[vec]
        c += x
        r += y
        flowpath.append([c, r])
        flowpath = np.array(flowpath)

        # calculate slope and distance
        slope = np.array([fvslop[indx[t[0]], indy[t[0]]] for t in indices])
        _distance = flowpath[:-1, :] - flowpath[1:, :]
        distance = np.sqrt(np.power(_distance[:, 0], 2.0) +
                           np.power(_distance[:, 1], 2.0))

        assert distance.shape == slope.shape

        # return to _abstract_channel
        return flowpath, slope, distance, indx, indy

    def abstract_channel(self, chn_id: int, wepp_chn_type='Default'):
        """
        define channel abstraction for the purposes of running WEPP
        """
        watershed = self.watershed
        translator = self.translator
        cellsize = self.cellsize

        flowpath, slope, distance, indx, indy = self._walk_channel(chn_id)

        # need normalized distance_p to define slope
        distance_p = cummnorm_distance(distance)
        if len(slope) == 1:
            slope = np.array([float(slope), float(slope)])

        # calculate the length from the distance array
        length = float(np.sum(distance) * cellsize)
        width = float(cellsize)
        aspect = float(self._determine_aspect(indx, indy))
        isoutlet = bool(chn_id == self.outlet_top_id)

        head = [v * cellsize for v in flowpath[-1]]
        head = [float(v) for v in head]
        tail = [v * cellsize for v in flowpath[0]]
        tail = [float(v) for v in tail]

        direction = compute_direction(head, tail)

        _centroid_px = centroid_px(indx, indy)
        centroid_lnglat = self.px_to_lnglat(*_centroid_px)

        elevs = representative_normalized_elevations(distance_p, slope)
        slope_scalar = float(abs(elevs[-1]))

        chn_summary = ChannelSummary(
            topaz_id=chn_id,
            wepp_id=translator.wepp(top=chn_id),
            chn_enum=translator.chn_enum(top=chn_id),
            chn_type=wepp_chn_type,
            isoutlet=isoutlet,
            length=length,
            width=width,
            aspect=aspect,
            head=head,
            tail=tail,
            direction=direction,
            elevs=elevs,
            slope_scalar=slope_scalar,
            color=slp_asp_color(slope_scalar, aspect),
            area=float(length) * float(width),
            distance_p=distance_p.tolist(),
            slopes=slope.tolist(),
            centroid=CentroidSummary(
                px=_centroid_px,
                lnglat=centroid_lnglat
            )
        )

        # save channel abstraction to instance
        watershed['channels']['chn_%i' % chn_id] = chn_summary

    def abstract_subcatchments(self, verbose=False, warn=False):
        subwta = self.subwta

        # extract the subcatchment and channel ids from the subwta map
        # subwta contains 0 values outside the watershed
        _ids = sorted(list(set(subwta.flatten())))

        # subcatchments end in 1, 2, and 3, subcatchments that end in
        # are source subcatchments, 2 are left and 3 are right
        sub_ids = [i for i in _ids if not str(i).endswith('4') and i > 0]

        n = len(sub_ids)
        if n == 0:
            raise Exception('SUBWTA contains no channels')

        for i, sub_id in enumerate(sub_ids):
            if verbose:
                print('abstracting subtatchment %s (%i of %i)' % (sub_id, i+1, n))
            self.abstract_subcatchment(sub_id, verbose=verbose, warn=warn)

        self.hillslope_n = len(sub_ids)

    def _walk_flowpath(self, sub_id: int, c: int, r: int, warn=False) -> Tuple[np.array, np.array, np.array]:
        """
        walk down the gradient until we reach a channel
        to find the flowpath in pixel coords, the slope
        along the flowpath and the distance between the
        cells
        """
        subwta = self.subwta
        flopat = self.flopat
        flovec = self.flovec
        fvslop = self.fvslop
        paths = self.paths

        # we are providing the starting point of the flowpath
        flowpath = [(c, r)]
        distance = []

        n, m = subwta.shape

        # time to walk down the path
        i = 0
        while 1:
            vec = flovec[c, r]

            # break if we hit a depression, not sure what
            # happens if we hit this
            if vec == 5:
                break

            # x and y specify the direction we need to move
            (x, y), dis = paths[vec]
            c += x
            r += y

            # make sure we aren't in a loop
            if (c, r) in flowpath:
                if warn:
                    warnings.warn('Flowpath c=%i, r=%i for %i went in a circle' % (c, r, sub_id))
                break

            # store the new values
            flowpath.append((c, r))
            distance.append(dis)

            # determine whether we have reached the end of the flowpath
            _id = subwta[c, r]
            pat = flopat[c, r]
            if pat in [0, 1] or _id != sub_id:
                break

            i += 1

            if i > n + m:
                if warn:
                    warnings.warn('Flowpath c=%i, r=%i for %i is too long (>N)' % (c, r, sub_id))
                break

        # cast flowpath and distance as ndarrays
        flowpath = np.array(flowpath)
        distance = np.array(distance)

        # extract the slope values for the flowpath. We don't take the
        # last flowpath so it is aligned with the distance array
        slope = fvslop[(flowpath[:-1, 0], flowpath[:-1, 1])]

        # double check...
        assert distance.shape == slope.shape

        # return to _walk_flowpaths
        return flowpath, slope, distance

    def _walk_flowpaths(self, sub_id: int, verbose=False, warn=False) -> \
            Tuple[List[np.array], List[np.array], List[np.array], np.array, np.array]:
        """
        considers each cell of the subcatchment as a starting
        point for a flowpath. It walks down each flowpath to
        determine the slopes and distance for each.
        """
        subwta = self.subwta

        flowpaths = []
        slopes = []
        distances = []

        indx, indy = np.where(subwta == sub_id)

        n = len(indx)
        for i, (c, r) in enumerate(zip(indx, indy)):
            if verbose:
                print('walking flowpath %i of %i (%s)' % (i+1, n, sub_id))

            flowpath, slope, distance = self._walk_flowpath(sub_id, c, r, warn=warn)
            flowpaths.append(flowpath)
            slopes.append(slope)
            distances.append(distance)

        # return to abstract_subcatchment
        return flowpaths, slopes, distances, indx, indy

    def abstract_flowpath(self, flowpath, slope, distance) -> FlowpathSummary:
        cellsize = self.cellsize
        cellsize2 = self.cellsize2

        indx, indy = flowpath[:, 0], flowpath[:, 1]

        # we really only care about the longest flowpath
        # which is the first because of how we identify the
        # subflows
        distance_p = cummnorm_distance(distance)
        if len(slope) == 1:
            slope = np.array([float(slope), float(slope)])
        length = float(np.sum(distance) * cellsize)
        area = float(np.sum(distance) * cellsize2)
        width = float(area / length)
        aspect = float(self._determine_aspect(indx, indy))

        _centroid_px = centroid_px(indx, indy)
        centroid_lnglat = self.px_to_lnglat(*_centroid_px)

        head = [v * cellsize for v in [indx[-1], indy[-1]]]
        head = [float(v) for v in head]
        tail = [v * cellsize for v in [indx[0], indy[0]]]
        tail = [float(v) for v in tail]

        direction = compute_direction(head, tail)

        elevs = representative_normalized_elevations(distance_p, slope)
        slope_scalar = float(abs(elevs[-1]))
        color = slp_asp_color(slope_scalar, aspect)

        return FlowpathSummary(
            topaz_id=None,
            wepp_id=None,
            length=float(length),
            width=float(width),
            area=float(area),
            aspect=float(aspect),
            direction=float(direction),
            slope_scalar=slope_scalar,
            color=color,
            coords=[(int(x), int(y)) for (x, y) in zip(indx, indy)],
            distance_p=distance_p.tolist(),
            slopes=slope.tolist(),
            centroid=CentroidSummary(
                px=_centroid_px,
                lnglat=centroid_lnglat
            )
        )

    def abstract_flowpaths(self, sub_id: int, flowpaths: List[np.array],
                           slopes: List[np.array], distances: List[np.array]) -> \
            Tuple[Dict[str, FlowpathSummary], List[List[int]]]:

        # returns a list of lists
        # each list contains indices corresponding to the flowpaths that
        # follow the same route. The index of the longest flowpath is
        # listed first
        subflows = _identify_subflows(flowpaths)

        # initialize a dictionary to hold the flowpath data
        fp_d = {}
        for j, fp_indxs in enumerate(subflows):
            # build the flowpath id
            # start at 1 to make sure wepp is happy
            fp_id = 'flow_%i_%i' % (sub_id, j+1)

            fp_d[fp_id] = self.abstract_flowpath(flowpaths[fp_indxs[0]],
                                                 slopes[fp_indxs[0]],
                                                 distances[fp_indxs[0]])

        return fp_d, subflows

    def abstract_subcatchment(self, sub_id, verbose=False, warn=False):
        """
        define subcatchment abstraction for the purposes of running WEPP
        """
        watershed = self.watershed
        translator = self.translator
        cellsize2 = self.cellsize2
        cellsize = self.cellsize

        # determine flowpaths
        # Tuple[List[np.array], List[np.array], List[np.array], np.array, np.array]
        flowpaths, slopes, distances, indx, indy = \
            self._walk_flowpaths(sub_id, verbose=verbose, warn=warn)

        # determine area, width, and length
        area = float(len(indx)) * cellsize2

        # find corresponding chn_id
        chn_id = 'chn_%i' % (int(math.floor(sub_id / 10.0) * 10) + 4)
        chn_summary = watershed["channels"][chn_id]

        # If subcatchment is a source type then we calculate the distance
        # by taking a weighted average based on the length of the flowpaths
        # contained in the subcatchment
        if str(sub_id).endswith('1'):
            length = cellsize * garbrecht_length(distances)
            width = area / length

        # Otherwise the  width of the subcatchment is determined by the
        # channel that the subcatchment drains into. The length is
        # then determined by the area / width
        else:
            width = chn_summary.length
            length = area / width

        direction = chn_summary.direction
        if str(sub_id).endswith('2'):
            direction += 90
        if str(sub_id).endswith('3'):
            direction -= 90

        # determine aspect
        aspect = self._determine_aspect(indx, indy)

        # calculate weighted slope from flowpaths
        w_slopes, distance_p = \
            _weighted_slope_average(flowpaths, slopes, distances)

        # abstract the flowpaths
        fp_d, subflows = \
            self.abstract_flowpaths(sub_id, flowpaths, slopes, distances)

        watershed['flowpaths']['hill_%i' % sub_id] = fp_d

        # cast lists of ndarrays to list of lists so we can dump to json
        flowpaths = [x.tolist() for x in flowpaths]
#        distances = [x.tolist() for x in distances]
#        slopes = [x.tolist() for x in slopes]

        # determine pourpoint from flowpaths
        pourpoint = Counter([tuple(fp[-1]) for fp in flowpaths]).most_common()[0][0]
        pourpoint = list([v * cellsize for v in pourpoint])

        _centroid_px = centroid_px(indx, indy)
        centroid_lnglat = self.px_to_lnglat(*_centroid_px)

        elevs = representative_normalized_elevations(distance_p, w_slopes)
        slope_scalar = float(abs(elevs[-1]))

        sub_summary = HillSummary(
            topaz_id=sub_id,
            wepp_id=translator.wepp(top=sub_id),
            w_slopes=list(w_slopes),
            pourpoint=list(pourpoint),
            length=float(length),
            width=float(width),
            area=float(area),
            aspect=float(aspect),
            direction=direction,
            slope_scalar=slope_scalar,
            color=slp_asp_color(slope_scalar, aspect),
            distance_p=list(distance_p),
            centroid=CentroidSummary(
                px=_centroid_px,
                lnglat=centroid_lnglat
            )
        )
        watershed['hillslopes']['hill_%i' % sub_id] = sub_summary

    def abstract_structure(self, verbose=False):
        self._read_netw_tab()
        network = self.network
        translator = self.translator

        # now we are going to define the lines of the structure file
        # this doesn't handle impoundments

        structure = []
        for chn_id in translator.iter_chn_ids():
            if verbose:
                print('abstracting structure for channel %s...' % chn_id)
            top = translator.top(chn_id=chn_id)
            chn_enum = translator.chn_enum(chn_id=chn_id)

            # right subcatchments end in 2
            hright = top - 2
            if not translator.has_top(hright):
                hright = 0

            # left subcatchments end in 3
            hleft = top - 1
            if not translator.has_top(hleft):
                hleft = 0

            # center subcatchments end in 1
            hcenter = top - 3
            if not translator.has_top(hcenter):
                hcenter = 0

            # define structure for channel
            # the first item defines the channel
            _structure = [chn_enum]

            # network is defined from the NETW.TAB file that has
            # already been read into {network}
            # the 0s are appended to make sure it has a length of
            # at least 3
            chns = network.get(top, []) + [0, 0, 0]

            # structure line with top ids
            _structure += [hright, hleft, hcenter] + chns[:3]

            # this is where we would handle impoundments
            # for now no impoundments are assumed
            _structure += [0, 0, 0]

            # and translate topaz to wepp
            structure.append([int(v) for v in _structure])

        self._structure = structure

if __name__ == '__main__':
    watershed_a = WatershedAbstraction('/geodata/weppcloud_runs/419e3b7f-ae62-4bc1-af10-bba44266f494/dem/topaz',
                                       '/geodata/weppcloud_runs/419e3b7f-ae62-4bc1-af10-bba44266f494/watershed')
    watershed_a.abstract(verbose=False)
#    watershed_a.write_slps(channels=1, subcatchments=1, flowpaths=1)
#    js = watershed_a.json()

#    watershed_b = WatershedAbstraction()
#    watershed_b.load(js)
#    watershed_b.dumpjson('wb.json')
#    pprint(watershed_b.watershed["channels"])