import backtrader as bt
import sqlite3
import pandas as pd
import datetime

# 투자전략 만들기
class TestStrategy(bt.Strategy):

    def log(self, txt, dt=None):
        '''전략의 로그 남기기'''
        dt = dt or self.datas[0].datetime.date(0)
        print('%s, %s' % (dt.isoformat(), txt))
        # if self.params.printlog or doprint:
        #     dt = dt or self.datas[0].datetime.date(0)
        #     print('%s, %s' % (dt.isoformat(), txt))

    def __init__(self):
        # 변수 모음 ==============================================================================
        self.dataclose = self.datas[0].close
        self.dataopen = self.datas[0].open

        self.order = None
        self.buyprice = None
        self.buycomm = None
        # ======================================================================================

        # 지표 추가 ========================================================================================
        self.sma = bt.indicators.MovingAverageSimple(self.datas[0], period=20)
        # =================================================================================================

    def notify_order(self, order):
        if order.status in [order.Submitted, order.Accepted]:
            return

        # 주문 완료됐는지 체크
        if order.status in [order.Completed]:
            if order.isbuy():
                self.log('매수 체결| 매수가: %.2f, 비용: %.2f, 수수료: %.2f' %
                         (order.executed.price, order.executed.value, order.executed.comm))
            elif order.issell():
                self.log('매도 체결| 매도가: %.2f, 비용: %.2f, 수수료: %.2f' %
                         (order.executed.price, order.executed.value, order.executed.comm))

            self.bar_executed = len(self)

        elif order.status in [order.Canceled, order.Margin, order.Rejected]:
            self.log('주문 취소/거절')

        self.order = None

    def notify_trade(self, trade):
        if not trade.isclosed:
            return

        self.log('총손익: %.2f, 순손익: %.2f' %
                 (trade.pnl, trade.pnlcomm))

    def next(self):
        self.log('시가: %.2f 종가: %.2f' % (self.dataopen[0], self.dataclose[0]))

        # 진행 중인 주문이 있는지 확인
        if self.order:
            return

        # 포지션 보유 중인지 확인
        if not self.position:
            # 3일 연속 종가 하락하면 Buy 시그널 =======================================================================================
            ## 직전 일자 데이터 없으면 가장 최근데이터 가져옴 >> data length 확인 필요
            if len(self) > 3:
                if self.dataclose[0] < self.dataclose[-1]:
                    if self.dataclose[-1] < self.dataclose[-2]:
                        self.log('매수 시그널: %.2f (전일종가: %.2f | 2일전종가: %.2f)' % (self.dataclose[0], self.dataclose[-1], self.dataclose[-2]))
                        self.order = self.buy(coc=True)

        else:
            # 보유 후 N일 경과하면 매도
            if len(self) >= (self.bar_executed + 5):
                self.log('매도 시그널: %.2f' % self.dataclose[0])

                self.order = self.sell(coc=False)

if __name__ == '__main__':
    cerebro = bt.Cerebro()

    # 전략 추가 ===================================================================================
    cerebro.addstrategy(TestStrategy)

    # 데이터 가져오기 ===============================================================================
    from_date = "20200801"
    to_date = ''
    data_path = 'E:/DB/cmp_ohlcv.db'
    ## 시총 상위 200개 종목
    sql = "SELECT * FROM cmp_master WHERE Cmp_type=1 AND List_date < " + from_date + " ORDER BY Mkt_cap DESC LIMIT 200"
    con = sqlite3.connect(data_path)
    code_list = pd.read_sql(sql, con)
    code_list = code_list['Code']
    # ============================================================================================

    # 데이터 Cerebro에 입력 =========================================================================
    sql = "SELECT DAY, OPEN, HIGH, LOW, CLOSE, VOLUME  FROM T_STK_DATA WHERE CMP_CD = '{0}' AND DAY > " + from_date + " ORDER BY DAY ASC"

    code = 'A069500'
    data = pd.read_sql(sql.format(code), con, index_col='DAY', parse_dates=['DAY'])
    data = bt.feeds.PandasData(dataname=data, name=code)
    cerebro.adddata(data)
    # ============================================================================================

    # 초기현금, 수수료, Sizer 세팅 ===================================================================
    cerebro.broker.set_cash(10000000)
    cerebro.broker.setcommission(0.015)
    cerebro.addsizer(bt.sizers.FixedSize, stake=10)
    cerebro.broker.set_coc(True)
    # ============================================================================================

    print('기초 포트폴리오 가치: %.2f' % cerebro.broker.getvalue())

    cerebro.run()

    print('기말 포트폴리오 가치: %.2f' % cerebro.broker.getvalue())

    cerebro.plot()


import pandas
from sklearn import linear_model
#reading the file
df = pandas.read_csv("cars.csv")

#make list of independent values, these are the x values
#the dependent values we will store these in the y value
X = df[['Weight', 'Volume']]
y  = df['CO2']

regr = linear_model.LinearRegression()
regr.fit(X,y)
#Now we have a regression object that are ready to predict CO2 values based on a car's weight and volume:
#predict the CO2 emission of a car where the weight is 2300kg, and the volume is 1300cm3:
predictedCo2 = regr.predict([[2300,1300]])
print(predictedCo2)

#this pritns the coeficient of the regression object
print(regr.coef_)

#We have predicted that a car with 1.3 liter engine, and a weight of 2300 kg,
#will release approximately 107 grams of CO2 for every kilometer it drives.


# -*- coding:utf-8 -*-
import json
import csv
import pandas as pd
import config
from requests_oauthlib import OAuth1Session

CK = config.CONSUMER_KEY
CS = config.CONSUMER_SECRET
AT = config.ACCESS_TOKEN
ATS = config.ACCESS_TOKEN_SECRET
twitter = OAuth1Session(CK, CS, AT, ATS)

# search
def search(url, keyword, count):
    params = {"q": keyword, "count": count}
    response = twitter.get(url, params=params)
    if response.status_code == 200:
        search_result = json.loads(response.text)
        fields = ["created_at", "id", "text"]
        df = pd.DataFrame(search_result["statuses"], columns=fields)
        df.to_csv("result.csv", index=False, quoting=csv.QUOTE_ALL, encoding="utf-8-sig")
    else:
        print("ERROR: %d" % response.status_code)

if __name__ == '__main__':
    search(config.URL_SEARCH, config.SEARCH_KEYWORD, 100)


import logging as lg
import os
import re
from aztest.common import to_list
from aztest.errors import AzTestError
logger = lg.getLogger(__name__)

DEFAULT_WHITELIST_FILE = "lmbr_test_whitelist.txt"
DEFAULT_BLACKLIST_FILE = "lmbr_test_blacklist.txt"

def get_default_whitelist():
    """Returns the default whitelist file path if it exists, otherwise empty string

    :return: whitelist path or empty string
    :rtype: str
    """
    if os.path.exists(os.path.abspath(DEFAULT_WHITELIST_FILE)):
        return os.path.abspath(DEFAULT_WHITELIST_FILE)
    else:
        return ""

def get_default_blacklist():
    """Returns the default blacklist file path if it exists, otherwise empty string

    :return: blacklist path or empty string
    :rtype: str
    """
    if os.path.exists(os.path.abspath(DEFAULT_BLACKLIST_FILE)):
        return os.path.abspath(DEFAULT_BLACKLIST_FILE)
    else:
        return ""

class FileApprover(object):
    """Class for compiling and validating the set of files that are allowed to be tested"""
    whitelist = None
    blacklist = None

    def __init__(self, whitelist_files=None, blacklist_files=None):
        self.make_whitelist(whitelist_files)
        self.make_blacklist(blacklist_files)

    def make_whitelist(self, whitelist_files):
        """Make the whitelist from a file or list of files

        It is assumed that if no whitelist is provided, then we will scan everything not in a blacklist.

        :param whitelist_files: path to a whitelist file or list of paths
        """
        if whitelist_files:
            for whitelist_file in to_list(whitelist_files):
                self.add_whitelist(whitelist_file)
        else:
            self.whitelist = None

    def make_blacklist(self, blacklist_files):
        """Make the blacklist from a file or list of files

        It is assumed that is no blacklist is provided, then we will scan everything in a directory or what is
        whitelisted.

        :param blacklist_files: path to a blacklist file or list of paths
        """
        if blacklist_files:
            for blacklist_file in to_list(blacklist_files):
                self.add_blacklist(blacklist_file)
        else:
            self.blacklist = None

    def add_whitelist(self, whitelist_file):
        """Add the file of patterns to the whitelist

        :param whitelist_file: path to a whitelist file
        """
        if whitelist_file:
            patterns = self.get_patterns_from_file(whitelist_file)
            if patterns:
                self.whitelist = patterns | (self.whitelist or set())

    def add_blacklist(self, blacklist_file):
        """Add the file of patterns to the blacklist

        :param blacklist_file: path to a blacklist file
        """
        if blacklist_file:
            patterns = self.get_patterns_from_file(blacklist_file)
            if patterns:
                self.blacklist = patterns | (self.blacklist or set())

    def add_whitelist_patterns(self, patterns):
        """Add patterns to the whitelist

        :param patterns: regular expression pattern or list of patterns to add to whitelist
        """
        self.whitelist = set(to_list(patterns)) | (self.whitelist or set())

    def add_blacklist_patterns(self, patterns):
        """Add patterns to the blacklist

        :param patterns: regular expression pattern or list of patterns to add to blacklist
        """
        self.blacklist = set(to_list(patterns)) | (self.blacklist or set())

    def is_approved(self, file_name):
        """Checks to see if file_name is in the whitelist and not the blacklist

        :param file_name: name or path of the file to check
        :return: true if file_name in whitelist or whitelist is None and not in blacklist or blacklist is None,
        else false
        """
        return self.is_whitelisted(file_name) and not self.is_blacklisted(file_name)

    def is_whitelisted(self, file_name):
        """Checks to see if file_name is in the whitelist

        :param file_name: name or path of the file to check
        :return: true if file_name in whitelist or whitelist is None, else false
        """
        return True if not self.whitelist else self.is_in_list(file_name, self.whitelist, 'whitelist')

    def is_blacklisted(self, file_name):
        """Checks to see if file_name is in the blacklist

        :param file_name: name or path of the file to check
        :return: true if file_name not in blacklist or blacklist is None, else false
        """
        return False if not self.blacklist else self.is_in_list(file_name, self.blacklist, 'blacklist')

    @staticmethod
    def get_patterns_from_file(pattern_file):
        """Returns set of patterns from pattern_file if pattern_file is valid, otherwise returns None

        :param pattern_file: path of the whitelist or blacklist file to get patterns from
        :return: the set of patterns from the file, or None if pattern_file is invalid
        :rtype: set
        """
        if not pattern_file or not os.path.exists(pattern_file):
            raise AzTestError("Invalid module/pattern file path: {}".format(pattern_file))
        logger.info("Using module/pattern file: {}".format(pattern_file))
        with open(pattern_file, 'r') as f:
            patterns = set((line.strip() for line in f))
            return patterns

    @staticmethod
    def is_in_list(file_name, patterns, list_name='CMD'):
        """Checks file_name against the set of patterns using regex

        For the curious, we do not compile the patterns ahead of time because the 're' module uses an internal cache
        that stores compiled patterns, and it is large enough (100) that we shouldn't need to worry about patterns being
        recompiled during a scan.

        :param file_name: name or path of the file to check
        :param patterns: the set of patterns to use as regular expressions
        :param list_name: the name of the list being checked (if applicable, defaults to CMD)
        :return: true if there is a match, otherwise false
        """
        if not file_name or not patterns:
            return False
        logger.debug("Checking file: {}".format(file_name))
        for pattern in patterns:
            full_pattern = r"^.*[/\\]{}(\.dll|\.exe|\.dylib)?$".format(pattern)
            match = re.match(full_pattern, file_name, re.IGNORECASE)
            if match:
                logger.debug("Found file {} in {} patterns.".format(file_name, list_name))
                return True
        return False


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Dec  5 16:05:02 2019

@author: diaa
"""

z_dim = 2
X_dim = 784
N = 1000

from torch import nn
import torch.nn.functional as F
class Encoder(nn.Module):
    def __init__(self):
        super(Encoder, self).__init__()
        self.lin1 = nn.Linear(X_dim, N)
        self.lin2 = nn.Linear(N, N)
        # Gaussian code (z)
        self.lin3gauss = nn.Linear(N, z_dim)

    def forward(self, x):
        x = F.dropout(self.lin1(x), p=0.2, training=self.training)
        x = F.relu(x)
        x = F.dropout(self.lin2(x), p=0.2, training=self.training)
        x = F.relu(x)
        xgauss = self.lin3gauss(x)

        return xgauss

class Decoder(nn.Module):
    def __init__(self):
        super(Decoder, self).__init__()
        self.lin1 = nn.Linear(z_dim, N)
        self.lin2 = nn.Linear(N, N)
        self.lin3 = nn.Linear(N, X_dim)

    def forward(self, x):
        x = self.lin1(x)
        x = F.dropout(x, p=0.2, training=self.training)
        x = F.relu(x)
        x = self.lin2(x)
        x = F.dropout(x, p=0.2, training=self.training)
        x = self.lin3(x)
        return F.sigmoid(x)


from django.http import HttpResponse
from django.template import RequestContext, loader

from polls.models import Products, TDD_LTE_Phases, WMP_Phases, DCM_Phases

def index_validator(table):
    try:
        name = table.objects.order_by('-product_id')[0]
    except IndexError:
        name = 'null'
    return name

def index(request):
    #Get latest data from each Product/Subcomponent
    latest_tdlte = TDD_LTE_Phases.objects.order_by('-product_id')[0]
    latest_wmp = WMP_Phases.objects.order_by('-product_id')[0]
    latest_dcm = index_validator(DCM_Phases)
    template = loader.get_template('polls/index.html')
    #Add latest data into context, and pass it to overview page.
    context = RequestContext(request, {
        'latest_tdlte': latest_tdlte,
        'latest_wmp':latest_wmp,
        'latest_dcm':latest_dcm,
    })
    return HttpResponse(template.render(context))

def detail(request, product_id):
    return HttpResponse("You're looking at Products %s." % product_id)

def results(request, products_id):
    response = "You're looking at the results of Products %s."
    return HttpResponse(response % products_id)

def vote(request, products_id):
    return HttpResponse("You're voting on Products %s." % products_id)

# Chart generation
from chartit import DataPool, Chart

def chart_view_with_parameter(request, chart_name):

    template = loader.get_template('polls/chart.html')

    #Step 1: Create a DataPool with the data we want to retrieve.
    tdlte = \
        DataPool(
           series=
            [{'options': {
               'source': eval(chart_name).objects.all()},
              'terms': [
                'id',
                'entity_build',
                'majorRegression',
                'neve']}
             ])

    #Step 2: Create the Chart object
    cht = Chart(
            datasource = tdlte,
            series_options =
              [{'options':{
                  'type': 'line',
                  'stacking': False},
                'terms':{
                  'id': [
                    'entity_build',
                    'majorRegression',
                    'neve']
                  }}],
            chart_options =
              {'title': {
                   'text': 'Entity build history for ' + chart_name},
               'xAxis': {
                    'title': {
                       'text': 'Product ID'}}})

    #Step 3: Send the chart object to the template.
    context = RequestContext(request, {
        'charts': cht,
    })

    return HttpResponse(template.render(context))

def chart_view(request):

    template = loader.get_template('polls/chart_selector.html')

    #Step 1: Create a DataPool with the data we want to retrieve.
    tdlte = \
        DataPool(
           series=
            [{'options': {
               'source': WMP_Phases.objects.all()},
              'terms': [
                'id',
                'entity_build',
                'majorRegression',
                'neve']}
             ])

    #Step 2: Create the Chart object
    cht = Chart(
            datasource = tdlte,
            series_options =
              [{'options':{
                  'type': 'line',
                  'stacking': False},
                'terms':{
                  'id': [
                    'entity_build',
                    'majorRegression',
                    'neve']
                  }}],
            chart_options =
              {'title': {
                   'text': 'Entity build history for WMP_Phases'},
               'xAxis': {
                    'title': {
                       'text': 'Product ID'}}})

    #Step 3: Send the chart object to the template.
    context = RequestContext(request, {
        'charts': cht,
    })

    return HttpResponse(template.render(context))


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('api', '0004_housecode'),
    ]

    operations = [
        migrations.AddField(
            model_name='housecode',
            name='temperature_opentrv',
            field=models.CharField(default=None, max_length=6),
        ),
    ]


import logging
import struct
from rdflib import BNode, Literal, URIRef
from aiohttp_rdf4j.utils import Statement

logger = logging.getLogger(__name__)

# Constants
MAGIC_NUMBER = b'BRDF'

FORMAT_VERSION = 1

#record types

NAMESPACE_DECL = 0
STATEMENT = 1
COMMENT = 2
VALUE_DECL = 3
ERROR = 126
END_OF_DATA = 127

#value types

NULL_VALUE = 0
URI_VALUE = 1
BNODE_VALUE = 2
PLAIN_LITERAL_VALUE = 3
LANG_LITERAL_VALUE = 4
DATATYPE_LITERAL_VALUE = 5
VALUE_REF = 6

class BinaryRDFParser:

    def __init__(self, stream):
        self.declaredValues = dict()
        self.namespaces = dict()
        #Utf8Decoder = codecs.getincrementaldecoder('utf-16be')
        #self.decoder = Utf8Decoder()
        self.stream = stream
        self.buffer = b""
        #self.vars = None #Dummy attribute so that query can ask for vars in construct
        #self.cache = collections.deque()

    async def read_(self,n, t=None):
        while len(self.buffer)<n:
            chunk, eos = await self.stream.readchunk()
            self.buffer+=chunk
        res, self.buffer = self.buffer[:n], self.buffer[n:]
        return struct.unpack(t, res) if t else res

    async def parse(self):
        mn= await self.read_(4)
        if mn != MAGIC_NUMBER:
            ##print ("No Magic")
            raise IOError("Bad Magic Number")
            #return

        fv, = await self.read_(4,"!i")
        #print("fv", fv)
        if  fv != FORMAT_VERSION:
            raise IOError("Wrong Version")
            #return

        while True:
            recordType, = await self.read_(1,"!b")
            #print("in while", recordType)
            if recordType == END_OF_DATA:
                break
            elif recordType == STATEMENT:
                yield await self.readStatement()

            elif recordType == VALUE_DECL:
                await self.readValueDecl()
            elif recordType == NAMESPACE_DECL:
                #print("ns")
                await self.readNamespaceDecl()
            elif recordType == COMMENT:
                await self.readComment()
            else:
                raise IOError("Parse Error for recordType %s", recordType)
                break

    async def readNamespaceDecl(self):
        #print("in namespace")
        prefix = await self.readString()
        #print("prefix", prefix)
        namespace = await self.readString()
        #print("namespace", namespace)
        self.namespaces[prefix] = namespace

    async def readComment(self):
        comment = await self.readString()

    async def readValueDecl(self):
        #self.stream_caching(4)
        id, = await self.read_(4, "!i")
        v = await self.readValue()
        self.declaredValues[id] = v

    async def readStatement(self):

        subj = await self.readValue()
        #print("subj", subj)
        pred = await self.readValue()
        #print("pred", pred)
        obj = await self.readValue()
        #print("obj", obj)
        ctx = await self.readValue()
        #print("ctx", ctx)

        #print ("ii", subj,pred,obj, ctx)
        return Statement(subj,pred,obj,ctx)

        #return ((subj, pred, obj), ctx)

    async def readValue(self):

        valueType, = await self.read_(1, "!b")
        ##print(valueType)
        if valueType == NULL_VALUE:
            return None
        elif valueType == VALUE_REF:
            return await self.readValueRef()
        elif valueType == URI_VALUE:
            return await self.readUri()
        elif valueType == BNODE_VALUE:
            return await self.readBNode()
        elif valueType == PLAIN_LITERAL_VALUE:
            return await self.readPlainLiteral()
        elif valueType == LANG_LITERAL_VALUE:
            return await self.readLangLiteral()
        elif valueType == DATATYPE_LITERAL_VALUE:
            return await self.readDatatypeLiteral()
        else:
            return None

    async def readValueRef(self):
        id, = await self.read_(4, "!i")
        return self.declaredValues[id]

    async def readUri(self):
        return URIRef(await self.readString())

    async def readBNode(self):
        return BNode(await self.readString())

    async def readPlainLiteral(self):
        return Literal(await self.readString())

    async def readLangLiteral(self):
        label = await self.readString()
        language = await self.readString()
        return Literal(label, lang=language)

    async def readDatatypeLiteral(self):
        label = await self.readString()
        datatype = await self.readString()
        dt = URIRef(datatype)
        return Literal(label, datatype=dt)

    async def readString(self):
        ##print(x)
        length, = await self.read_(4, "!i")
        stringBytes = length << 1
        ##print(self.stream.getbuffer().nbytes, stringBytes, self.stream.tell())
        #self.stream_caching(length)
        _t = await self.read_(stringBytes)
        string = _t.decode("utf-16be")
        #string = self.decoder.decode(_t)
        ##print (string)
        return string

    def __aiter__(self):
        return self.parse()


import random
import numpy as np
import matplotlib.pyplot as plt
import math
from scipy import stats

plt.style.use('ggplot')

def direct_method(probs, samples=10000, plot=False):
    """
    Function for direct sampling method
    """

    sample_values = []

    for i in range(samples):
        sample = random.random()

        for j in range(len(probs)):
            if sample > sum(probs[0:j+1]):
                continue
            else:
                sample_values.append(j+1)
                break

    if plot:
        plt.hist(sample_values, bins=len(probs), rwidth=0.75)
        plt.title('Direct sample method')
        plt.show()

    return sample_values

def rejection_method(probs, sample_values=10000, plot=False):
    """
    Function for rejection sampling method
    """

    # Compute c
    k = len(probs)
    c = max(k * probs)

    num_reject = 0
    samples = []

    while len(samples) < sample_values:
        # get U1, U2
        rand1 = random.random()
        rand2 = random.random()

        # Get indicator variable
        indicator = math.floor(k * rand1) + 1

        # see if we should reject
        if rand2 <= probs[indicator-1]/c:
            samples.append(indicator)

        # Reject and try once more
        else:
            num_reject += 1
            continue

    if plot:
        plt.hist(samples, bins=k, rwidth=0.75)
        plt.title(
            f'Rejection method, {num_reject} rejected')
        plt.show()

    return samples

def alias_method(probs, num_samples=10000, plot=False):
    """
    Function for alias sampling method
    """

    # Generate F and L
    k = len(probs)

    L = list(range(1, k+1))
    F = k * probs

    # Generate g and s
    g = list(np.array(L)[(F >= 1)])
    s = list(np.array(L)[(F <= 1)])

    while len(s) > 0:

        i, j = g[0], s[0]

        L[j-1] = i
        F[i-1] = F[i-1] - (1-F[j-1])

        if F[i-1] < 1 - 1e-5:

            # Remove first element
            g.pop(0)

            # Add i to S
            s.append(i)

        s.pop(0)

    samples = []

    while len(samples) < num_samples:
        rand1 = random.random()
        rand2 = random.random()

        indicator = math.floor(k * rand1) + 1

        if rand2 <= F[indicator-1]:
            samples.append(indicator)
        else:
            samples.append(L[indicator-1])

    if plot:
        plt.hist(samples, bins=len(probs), rwidth=0.75)
        plt.title('Alias Method')
    plt.show()

    return samples

# Probabilities that will be used for simulation
probs = np.array([7/48, 5/48, 1/8, 1/16, 1/4, 5/16])

plt.bar(list(range(1, len(probs) + 1)), probs)
plt.title('Probabilites')
plt.show()

num_samples = 10000

# Direct method
samples_direct = direct_method(probs, samples=num_samples, plot=True)
frequencies_direct = np.bincount(samples_direct)[1:]

chisq_direct, pval_direct = stats.chisquare(
    frequencies_direct, probs*num_samples)
print(f'Direct Test statistic {chisq_direct}')
print(f'Direct pval {pval_direct}')

# Rejection method
samples_reject = rejection_method(probs, sample_values=num_samples, plot=True)
frequencies_reject = np.bincount(samples_reject)[1:]

chisq_reject, pval_reject = stats.chisquare(
    frequencies_reject, probs*num_samples)
print(f'Reject Test statistic {chisq_reject}')
print(f'Reject pval {pval_reject}')

# Alias method
samples_alias = alias_method(probs, num_samples=num_samples, plot=True)
frequencies_alias = np.bincount(samples_alias)[1:]

chisq_alias, pval_alias = stats.chisquare(frequencies_alias, probs*num_samples)
print(f'Alias Test statistic {chisq_alias}')
print(f'Alias pval {pval_alias}')


"""Collection of functions to operate on data from University of Oregon Solar
Radiation Monitoring Laboratory (SRML) data.
"""
import numpy as np
import pandas as pd
import urllib
import warnings

from pvlib._deprecation import deprecated

# VARIABLE_MAP is a dictionary mapping SRML data element numbers to their
# pvlib names. For most variables, only the first three digits are used,
# the fourth indicating the instrument. Spectral data (7xxx) uses all
# four digits to indicate the variable. See a full list of data element
# numbers `here. <http://solardat.uoregon.edu/DataElementNumbers.html>`_

VARIABLE_MAP = {
    '100': 'ghi',
    '201': 'dni',
    '300': 'dhi',
    '920': 'wind_direction',
    '921': 'wind_speed',
    '930': 'temp_air',
    '931': 'temp_dew',
    '933': 'relative_humidity',
    '937': 'temp_cell',
}

def read_srml(filename, map_variables=True):
    """
    Read University of Oregon SRML 1min .tsv file into pandas dataframe.

    The SRML is described in [1]_.

    Parameters
    ----------
    filename: str
        filepath or url to read for the tsv file.
    map_variables: bool, default: True
        When true, renames columns of the DataFrame to pvlib variable names
        where applicable. See variable :const:`VARIABLE_MAP`.

    Returns
    -------
    data: Dataframe
        A dataframe with datetime index

    Notes
    -----
    The time index is shifted back by one interval to account for the
    daily endtime of 2400, and to avoid time parsing errors on leap
    years. The returned data values are labeled by the left endpoint of
    interval, and should be understood to occur during the interval from
    the time of the row until the time of the next row. This is consistent
    with pandas' default labeling behavior.

    See [2]_ for more information concerning the file format.

    References
    ----------
    .. [1] University of Oregon Solar Radiation Monitoring Laboratory
       `http://solardat.uoregon.edu/ <http://solardat.uoregon.edu/>`_
    .. [2] `Archival (short interval) data files
       <http://solardat.uoregon.edu/ArchivalFiles.html>`_
    """
    tsv_data = pd.read_csv(filename, delimiter='\t')
    data = _format_index(tsv_data)
    # Drop day of year and time columns
    data = data[data.columns[2:]]

    if map_variables:
        data = data.rename(columns=_map_columns)

    # Quality flag columns are all labeled 0 in the original data. They
    # appear immediately after their associated variable and are suffixed
    # with an integer value when read from the file. So we map flags to
    # the preceding variable with a '_flag' suffix.
    #
    # Example:
    #   Columns ['ghi_0', '0.1', 'temp_air_2', '0.2']
    #
    #   Yields a flag_label_map of:
    #       { '0.1': 'ghi_0_flag',
    #         '0.2': 'temp_air_2'}
    #
    columns = data.columns
    flag_label_map = {flag: columns[columns.get_loc(flag) - 1] + '_flag'
                      for flag in columns[1::2]}
    data = data.rename(columns=flag_label_map)

    # Mask data marked with quality flag 99 (bad or missing data)
    for col in columns[::2]:
        missing = data[col + '_flag'] == 99
        data[col] = data[col].where(~(missing), np.NaN)
    return data

def _map_columns(col):
    """Map data element numbers to pvlib names.

    Parameters
    ----------
    col: str
        Column label to be mapped.

    Returns
    -------
    str
        The pvlib label if it was found in the mapping,
        else the original label.
    """
    if col.startswith('7'):
        # spectral data
        try:
            return VARIABLE_MAP[col]
        except KeyError:
            return col
    try:
        variable_name = VARIABLE_MAP[col[:3]]
        variable_number = col[3:]
        return variable_name + '_' + variable_number
    except KeyError:
        return col

def _format_index(df):
    """Create a datetime index from day of year, and time columns.

    Parameters
    ----------
    df: pd.Dataframe
        The srml data to reindex.

    Returns
    -------
    df: pd.Dataframe
        The Dataframe with a DatetimeIndex localized to 'Etc/GMT+8'.
    """
    # Name of the second column indicates the year of the file, but
    # the column contains times.
    year = int(df.columns[1])
    df_doy = df[df.columns[0]]
    # Times are expressed as integers from 1-2400, we convert to 0-2359 by
    # subracting the length of one interval and then correcting the times
    # at each former hour. interval_length is determined by taking the
    # difference of the first two rows of the time column.
    # e.g. The first two rows of hourly data are 100 and 200
    #      so interval_length is 100.
    interval_length = df[df.columns[1]][1] - df[df.columns[1]][0]
    df_time = df[df.columns[1]] - interval_length
    if interval_length == 100:
        # Hourly files do not require fixing the former hour timestamps.
        times = df_time
    else:
        # Because hours are represented by some multiple of 100, shifting
        # results in invalid values.
        #
        # e.g. 200 (for 02:00) shifted by 15 minutes becomes 185, the
        #      desired result is 145 (for 01:45)
        #
        # So we find all times with minutes greater than 60 and remove 40
        # to correct to valid times.
        old_hours = df_time % 100 > 60
        times = df_time.where(~old_hours, df_time - 40)
    times = times.apply(lambda x: '{:04.0f}'.format(x))
    doy = df_doy.apply(lambda x: '{:03.0f}'.format(x))
    dts = pd.to_datetime(str(year) + '-' + doy + '-' + times,
                         format='%Y-%j-%H%M')
    df.index = dts
    df = df.tz_localize('Etc/GMT+8')
    return df

@deprecated('0.10.0', alternative='pvlib.iotools.get_srml', removal='0.11.0')
def read_srml_month_from_solardat(station, year, month, filetype='PO',
                                  map_variables=True):
    """Request a month of SRML data and read it into a Dataframe.

    The SRML is described in [1]_.

    Parameters
    ----------
    station: str
        The name of the SRML station to request.
    year: int
        Year to request data for
    month: int
        Month to request data for.
    filetype: string
        SRML file type to gather. See notes for explanation.
    map_variables: bool, default: True
        When true, renames columns of the DataFrame to pvlib variable names
        where applicable. See variable :const:`VARIABLE_MAP`.

    Returns
    -------
    data: pd.DataFrame
        One month of data from SRML.

    Notes
    -----
    File types designate the time interval of a file and if it contains
    raw or processed data. For instance, `RO` designates raw, one minute
    data and `PO` designates processed one minute data. The availability
    of file types varies between sites. Below is a table of file types
    and their time intervals. See [1] for site information.

    ============= ============ ==================
    time interval raw filetype processed filetype
    ============= ============ ==================
    1 minute      RO           PO
    5 minute      RF           PF
    15 minute     RQ           PQ
    hourly        RH           PH
    ============= ============ ==================

    References
    ----------
    .. [1] University of Oregon Solar Radiation Measurement Laboratory
       `http://solardat.uoregon.edu/ <http://solardat.uoregon.edu/>`_
    """
    file_name = "{station}{filetype}{year:02d}{month:02d}.txt".format(
        station=station,
        filetype=filetype,
        year=year % 100,
        month=month)
    url = "http://solardat.uoregon.edu/download/Archive/"
    data = read_srml(url + file_name, map_variables=map_variables)
    return data

def get_srml(station, start, end, filetype='PO', map_variables=True,
             url="http://solardat.uoregon.edu/download/Archive/"):
    """Request data from UoO SRML and read it into a Dataframe.

    The University of Oregon Solar Radiation Monitoring Laboratory (SRML) is
    described in [1]_. A list of stations can be found in [2]_.

    Data is returned for the entire months between and including start and end.

    Parameters
    ----------
    station : str
        Two letter station abbreviation.
    start : datetime-like
        First day of the requested period
    end : datetime-like
        Last day of the requested period
    filetype : string, default: 'PO'
        SRML file type to gather. See notes for explanation.
    map_variables : bool, default: True
        When true, renames columns of the DataFrame to pvlib variable names
        where applicable. See variable :const:`VARIABLE_MAP`.
    url : str, default: 'http://solardat.uoregon.edu/download/Archive/'
        API endpoint URL

    Returns
    -------
    data : pd.DataFrame
        Dataframe with data from SRML.
    meta : dict
        Metadata.

    Notes
    -----
    File types designate the time interval of a file and if it contains
    raw or processed data. For instance, `RO` designates raw, one minute
    data and `PO` designates processed one minute data. The availability
    of file types varies between sites. Below is a table of file types
    and their time intervals. See [1] for site information.

    ============= ============ ==================
    time interval raw filetype processed filetype
    ============= ============ ==================
    1 minute      RO           PO
    5 minute      RF           PF
    15 minute     RQ           PQ
    hourly        RH           PH
    ============= ============ ==================

    Warning
    -------
    SRML data has nighttime data prefilled with 0s through the end of the
    current month (i.e., values are provided for data in the future).

    References
    ----------
    .. [1] University of Oregon Solar Radiation Measurement Laboratory
       `http://solardat.uoregon.edu/ <http://solardat.uoregon.edu/>`_
    .. [2] Station ID codes - Solar Radiation Measurement Laboratory
       `http://solardat.uoregon.edu/StationIDCodes.html
       <http://solardat.uoregon.edu/StationIDCodes.html>`_
    """
    # Use pd.to_datetime so that strings (e.g. '2021-01-01') are accepted
    start = pd.to_datetime(start)
    end = pd.to_datetime(end)

    # Generate list of months
    months = pd.date_range(
        start, end.replace(day=1) + pd.DateOffset(months=1), freq='1M')
    months_str = months.strftime('%y%m')

    # Generate list of filenames
    filenames = [f"{station}{filetype}{m}.txt" for m in months_str]

    dfs = []  # Initialize list of monthly dataframes
    for f in filenames:
        try:
            dfi = read_srml(url + f, map_variables=map_variables)
            dfs.append(dfi)
        except urllib.error.HTTPError:
            warnings.warn(f"The following file was not found: {f}")

    data = pd.concat(dfs, axis='rows')

    meta = {'filetype': filetype,
            'station': station,
            'filenames': filenames}

    return data, meta


import datetime

import pandas as pd
import numpy as np
import bs4
import re
import requests
from helpers.date_time_handler import DateTimeHandler
from helpers.progress_handler import ProgressHandler
from models.club import Club
from models.league import League

class SportsScraper:
    """
    Static methods which perform the scraping functionality.

    Attributes
    ----------
        __leagues      Acts as a cache for storing league url/name
        __clubs        Acts as a cache for storing club   ids/names

    Methods
    -------
        __scrap_leagues():
            Scraps data containing a list of leagues.
        scrap_leagues():
            Calls __get_leagues if __leagues is None, otherwise, it retrieves __leagues immediately.

        __get_cached_clubs():
            Retrieves the club's snapshot.
        cache_clubs():
            Collects a snapshot of the clubs for faster fetch in the future.
        __get_clubs(leagues, tolerate_too_many_requests=False):
            Calls http://site.api.espn.com/apis/site/v2/sports/soccer/{league}/teams iteratively to fetch all clubs ids.
        get_clubs(leagues, tolerate_too_many_requests=False):
            Calls __get_clubs if __clubs is None, otherwise, it retrieves __clubs immediately.

        __get_cached_players():
            Retrieves the player's snapshot.
        cache_players():
            Collects a snapshot of the players for faster fetch in the future.
        scrap_players(season_years=None, leagues=None, clubs=None, fast_fetch_clubs=False, fast_fetch=False)
            Scraps data containing information about club's players.
        __scrap_players(season_years=None, leagues=None, clubs=None, fast_fetch_clubs=False):
            Scraps data containing information about club's players.

        __get_cached_matches():
            Retrieves the match's snapshot.
        cache_matches():
            Collects a snapshot of the matches for faster fetch in the future.
        scrap_matches(start_date=None, end_date=None, fast_fetch=False):
            Scraps data containing information about the results of the matches.
        __scrap_matches(start_date=datetime.date.today() - datetime.timedelta(days=7), end_date=datetime.date.today()):
            Scraps data containing information about the results of the matches.
    """

    __leagues = None
    __clubs = None

    @staticmethod
    def __scrap_leagues():
        """
        Scraps data containing a list of leagues.

        :return: A list of leagues object
        """

        leagues = []

        # Partially prevents scraping detection
        headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.0; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0'}
        res = requests.get('https://www.espn.com/soccer/teams', headers=headers)
        soup = bs4.BeautifulSoup(res.text, 'html.parser')
        ddl = soup.find('select', attrs={'class': 'dropdown__select'})

        if not ddl:
            return leagues

        options = ddl.find_all('option')
        for option in options:
            leagues.append(League(option['value'], option.text))

        return leagues

    @staticmethod
    def scrap_leagues():
        """
        Calls __get_leagues if __leagues is None, otherwise, it retrieves __leagues immediately.

        :return: A list of leagues object
        """

        if SportsScraper.__leagues is None:
            print('Fetching leagues, this is a one time process...')
            SportsScraper.__leagues = SportsScraper.__scrap_leagues()
            print('Received leagues\n')

        return SportsScraper.__leagues.copy()

    @staticmethod
    def __get_cached_clubs():
        """
        Retrieves the club's snapshot.
        """

        clubs = []
        df = pd.read_csv('cached_clubs.csv', index_col='club_id', skiprows=1)

        for index, row in df.iterrows():
            clubs.append(Club(index, row['club_name'], League(row['league_url'], row['league_name'])))

        return clubs

    @staticmethod
    def cache_clubs():
        """
        Collects a snapshot of the clubs for faster fetch in the future.
        """
        clubs = SportsScraper.get_clubs()

        data = {
            'club_id': [x.club_id for x in clubs],
            'club_name': [x.name for x in clubs],
            'league_url': [x.league.url for x in clubs],
            'league_name': [x.league.name for x in clubs],
        }

        df = pd.DataFrame(data)

        df.set_index('club_id', inplace=True)

        f = open('cached_clubs.csv', "w+")
        f.write(f'# Timestamp: {datetime.datetime.utcnow()}\n')
        f.close()

        # noinspection PyTypeChecker
        df.to_csv('cached_clubs.csv', mode='a')

    @staticmethod
    def __get_clubs(tolerate_too_many_requests=False, fast_fetch=False):
        """
        Calls http://site.api.espn.com/apis/site/v2/sports/soccer/{league}/teams iteratively to fetch all clubs ids.

        :param bool tolerate_too_many_requests: Specify to whether throw an exception if the status code is not 200
        :param bool fast_fetch: Retrieves clubs from a saved snapshot instantly
        :return: A list of clubs object
        """

        if fast_fetch:
            return SportsScraper.__get_cached_clubs()

        scraped_leagues = SportsScraper.scrap_leagues()
        clubs = []

        processed = 0
        for league in scraped_leagues:
            print(ProgressHandler.show_progress(processed, len(scraped_leagues)))
            processed += 1
            # Partially prevents scraping detection
            headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.0; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0'}
            response = requests.get(f'http://site.api.espn.com/apis/site/v2/sports/soccer/{league.url}/teams',
                                    headers=headers)
            if response.status_code != 200 and tolerate_too_many_requests:
                continue
            for club in response.json()['sports'][0]['leagues'][0]['teams']:
                clubs.append(Club(club['team']['id'], club['team']['name'], league))

        ProgressHandler.reset_progress()

        return clubs

    @staticmethod
    def get_clubs(fast_fetch=False):
        """
        Calls __get_clubs if __clubs is None, otherwise, it retrieves __clubs immediately.

        :param bool fast_fetch: Retrieves clubs from a saved snapshot instantly
        :return: A list of clubs object
        """

        if SportsScraper.__clubs is None:
            print('Fetching clubs, this is a one time process...')
            SportsScraper.__clubs = SportsScraper.__get_clubs(tolerate_too_many_requests=True, fast_fetch=fast_fetch)
            print('Received clubs\n')

        return SportsScraper.__clubs.copy()

    @staticmethod
    def __get_cached_players():
        """
        Retrieves the player's snapshot.
        """

        players = pd.read_csv('cached_players.csv', skiprows=1)

        return players

    @staticmethod
    def cache_players():
        """
        Collects a snapshot of the players for faster fetch in the future.
        """
        players = SportsScraper.scrap_players(fast_fetch_clubs=True)

        f = open('cached_clubs.csv', "w+")
        f.write(f'# Timestamp: {datetime.datetime.utcnow()}\n')
        f.close()

        # noinspection PyTypeChecker
        players.to_csv('cached_players.csv', mode='a')

    @staticmethod
    def scrap_players(season_years=None, leagues=None, clubs=None, fast_fetch_clubs=False, fast_fetch=False):
        """
        Scraps data containing information about club's players.

        :param list[int] season_years: Collect the data from the provided year(s)
        :param list[str] leagues: Specify the desired league(s)
        :param list[str] clubs: Specify the desired club(s)
        :param bool fast_fetch_clubs: Retrieves clubs from a saved snapshot instantly
        :param bool fast_fetch: Retrieves players from a saved snapshot instantly
        :return: A dataframe containing club players
        """

        if fast_fetch:
            df = SportsScraper.__get_cached_players()
            if season_years is not None:
                df = df[df.YEAR.isin(season_years)]
            if leagues is not None:
                df = df[df.LEAGUE.isin(leagues)]
            if clubs is not None:
                df = df[df.CLUB.isin(clubs)]
            return df

        else:
            return SportsScraper.__scrap_players(season_years, leagues, clubs, fast_fetch_clubs)

    @staticmethod
    def __scrap_players(season_years=None, leagues=None, clubs=None, fast_fetch_clubs=False):
        """
        Scraps data containing information about club's players.

        :param list[int] season_years: Collect the data from the provided year(s)
        :param list[str] leagues: Specify the desired league(s)
        :param list[str] clubs: Specify the desired club(s)
        :param bool fast_fetch_clubs: Retrieves clubs from a saved snapshot instantly
        :return: A dataframe containing club players
        """

        if (season_years is None) or (len(season_years) == 0):
            season_years = np.arange(2000, 2022)

        scraped_leagues = SportsScraper.scrap_leagues()
        if (leagues is not None) and (len(leagues) != 0):
            scraped_leagues = list(filter(lambda x: x.name in leagues, scraped_leagues))

        if fast_fetch_clubs:
            scraped_clubs = SportsScraper.__get_cached_clubs()
        else:
            scraped_clubs = SportsScraper.get_clubs()

        if (clubs is not None) and (len(clubs) != 0):
            scraped_clubs = list(
                filter(lambda x: x.name in clubs and x.league.name in [x.name for x in scraped_leagues], scraped_clubs))

        if not all(isinstance(x, np.int32) or isinstance(x, int) for x in season_years):
            raise ValueError('season_year must be a list of integer')
        if clubs is not None and not all(isinstance(x, str) for x in clubs):
            raise ValueError('clubs must be a list of string')
        if leagues is not None and not all(isinstance(x, str) for x in leagues):
            raise ValueError('leagues must be a list of string')

        players_df_goalkeeper = pd.DataFrame()

        players_df_player = pd.DataFrame()

        processed = 0
        for season_year in season_years:
            for club in scraped_clubs:
                print(ProgressHandler.show_progress(processed,
                                                    len(season_years) * len(scraped_clubs)))
                processed += 1
                # Partially prevents scraping detection
                headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.0; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0'}
                res = requests.get(
                    f'https://www.espn.com/soccer/team/squad/_/'
                    f'id/{club.club_id}/'
                    f'league/{club.league.url}/'
                    f'season/{season_year}',
                    headers=headers)

                soup = bs4.BeautifulSoup(res.text, 'html.parser')
                tables = soup.find_all('table', attrs={'class': 'Table'})

                if not tables or len(tables) != 2:
                    continue

                for x in np.arange(0, 2):
                    data = []
                    rows = tables[x].find_all('tr')
                    for row in rows:
                        cols = row.find_all('td')
                        cols = [ele.text.strip() for ele in cols]  # Strips elements
                        if cols:  # If column is not empty
                            buff = [club.league.name, club.name, str(season_year)] + \
                                   [ele for ele in cols if ele]  # If element is not empty
                            number_list = re.findall(r'\d+$', buff[3])
                            if number_list:  # Checks if there is a number for the player
                                buff.insert(4, number_list[0])  # Extract player's number
                                buff[3] = re.findall(r'^([^0-9]*)', buff[3])[0]  # Remove player's name number
                            else:
                                buff.insert(4, np.nan)
                            data.append(buff)

                    if x == 0:
                        players_df_goalkeeper = players_df_goalkeeper.append(data)
                    else:
                        players_df_player = players_df_player.append(data)

        if players_df_goalkeeper.empty:
            players_df_goalkeeper = pd.DataFrame(np.empty((0, 19)))

        if players_df_player.empty:
            players_df_player = pd.DataFrame(np.empty((0, 20)))

        players_df_goalkeeper.columns = ['YEAR', 'LEAGUE', 'CLUB', 'NAME', 'NUM', 'POS', 'AGE', 'HT', 'WT', 'NAT',
                                         'APP', 'SUB',
                                         'SV', 'GA', 'A',
                                         'FC', 'FA', 'YC', 'RC']
        players_df_player.columns = ['YEAR', 'LEAGUE', 'CLUB', 'NAME', 'NUM', 'POS', 'AGE', 'HT', 'WT', 'NAT', 'APP',
                                     'SUB',
                                     'G', 'A', 'SH', 'ST',
                                     'FC', 'FA', 'YC', 'RC']

        df = players_df_goalkeeper.append(players_df_player)
        df = df.replace(r'^\s*$', np.nan, regex=True) \
            .replace('--', np.nan) \
            .reset_index(drop=True)

        df['WT'] = df['WT'].apply(lambda x: x if (pd.isnull(x)) else int(x.split(" ")[0]) / 2.205)
        df['HT'] = df['HT'].apply(
            lambda x: x
            if (pd.isnull(x)) else
            int(x.split("'")[0]) * 30.48 +
            int(x.split("'")[1].split("\"")[0]) * 2.54
        )

        cols = df.columns.drop(['LEAGUE', 'CLUB', 'NAME', 'POS', 'NAT'])
        df[cols] = df[cols].apply(pd.to_numeric)

        df.set_index(['LEAGUE', 'CLUB', 'YEAR', 'NAME'], inplace=True)

        ProgressHandler.reset_progress()
        return df

    @staticmethod
    def __get_cached_matches():
        """
        Retrieves the match's snapshot.
        """

        matches = pd.read_csv('cached_matches.csv', skiprows=1, dtype={'date': object,
                                                                       'club1': str,
                                                                       'SCORE': str,
                                                                       'club2': str,
                                                                       'DURATION': str,
                                                                       'LOCATION': str,
                                                                       'ATTENDANCE': str,
                                                                       'TIME': str,
                                                                       'TV': np.float})

        matches['ATTENDANCE'] = pd.to_numeric(matches['ATTENDANCE'].str.replace(',', ''), downcast='integer')
        matches['date'] = pd.to_datetime(matches['date']).dt.date

        return matches

    @staticmethod
    def cache_matches():
        """
        Collects a snapshot of the matches for faster fetch in the future.
        """

        matches = SportsScraper.scrap_matches(start_date=datetime.date(2002, 10, 1),
                                              end_date=datetime.date(2022, 5, 29))

        f = open('cached_matches.csv', "w+")
        f.write(f'# Timestamp: {datetime.datetime.utcnow()}\n')
        f.close()

        # noinspection PyTypeChecker
        matches.to_csv('cached_matches.csv', index=False, mode='a')

    @staticmethod
    def scrap_matches(start_date=None, end_date=None, fast_fetch=False):
        """
        Scraps data containing information about the results of the matches.

        :param datetime.date start_date: Specify the start date of the search
        :param datetime.date end_date: Specify the end date of the search
        :param bool fast_fetch: Retrieves matches from a saved snapshot instantly
        :return: An array of two dataframe containing match results (0: Elapsed, 1: Fixtures)
        """

        if fast_fetch:
            df = SportsScraper.__get_cached_matches()
            if start_date is not None:
                df = df[df.date >= start_date]
            if end_date is not None:
                df = df[df.date <= end_date]
            return df

        else:
            return SportsScraper.__scrap_matches(start_date, end_date)

    @staticmethod
    def __scrap_matches(start_date=datetime.date.today() - datetime.timedelta(days=7),
                        end_date=datetime.date.today(),
                        request_tries=8):
        """
        Scraps data containing information about the results of the matches.

        :param datetime.date start_date: Specify the start date of the search
        :param datetime.date end_date: Specify the end date of the search
        :param int request_tries: Determine to number of tries for each webpage request whenever it fails
        :return: An array of two dataframe containing match results (0: Elapsed, 1: Fixtures)
        """

        if start_date > end_date:
            raise ValueError('start_date cannot be less than end_date')

        elapsed_matches_df = pd.DataFrame()
        fixtures_list_df = pd.DataFrame()

        processed = 0

        days_between = DateTimeHandler.get_dates_between(start_date, end_date)

        for singleDay in days_between:
            data = []
            print(ProgressHandler.show_progress(processed, len(days_between)))
            processed += 1
            # Partially prevents scraping detection
            headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.0; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0'}
            res = requests.get(
                f'https://www.espn.in/football/fixtures/_/'
                f'date/{singleDay}',
                headers=headers)
            soup = bs4.BeautifulSoup(res.text, 'html.parser')
            tables = soup.find_all('tbody')

            tries = 0
            print(singleDay)
            while soup.find('h1', {'class': 'Error404__Title'}) is not None and tries < request_tries:
                print(f'try {tries + 1}')
                tries += 1
                headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.0; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0'}
                res = requests.get(
                    f'https://www.espn.in/football/fixtures/_/'
                    f'date/{singleDay}',
                    headers=headers)
                soup = bs4.BeautifulSoup(res.text, 'html.parser')
                tables = soup.find_all('tbody')

            if tries == request_tries:
                print('giving up...')

            if not tables:
                continue

            for table in tables:
                rows = table.find_all('tr')
                for row in rows:
                    cols = row.find_all('td')
                    if cols:  # If column is not empty
                        arr = [DateTimeHandler.year_month_day_to_date(singleDay)]
                        for col in np.arange(0, len(cols)):
                            if cols[col].find('small'):
                                continue
                            if col == 0:
                                club1 = cols[col].find('span').text
                                result = cols[col].find_all('a')[-1].text
                                arr.append(club1)
                                arr.append(result)
                            elif col == 1:
                                club2 = cols[col].find_all('span')[-1].text
                                arr.append(club2)
                            elif col == 2:
                                if cols[col].get('data-date'):
                                    date = datetime.datetime.strptime(cols[col].get('data-date'), '%Y-%m-%dT%H:%MZ')
                                    arr.append('{:d}:{:02d}'.format(date.hour, date.minute))
                                else:
                                    arr.append(cols[col].find('a').text)
                            else:
                                arr.append(cols[col].text)
                        data.append(arr)

            data = list(filter(lambda x: len(x) != 1, data))

            elapsed_matches_list = list(filter(lambda x: x[4] != 'LIVE' or ':' not in x[4], data))
            fixtures_list = list(filter(lambda x: x[4] == 'LIVE' or ':' in x[4], data))

            elapsed_matches_df = elapsed_matches_df.append(elapsed_matches_list)
            fixtures_list_df = fixtures_list_df.append(fixtures_list)

        if elapsed_matches_df.empty:
            elapsed_matches_df = pd.DataFrame(np.empty((0, 7)))
        if fixtures_list_df.empty:
            fixtures_list_df = pd.DataFrame(np.empty((0, 6)))

        elapsed_matches_df.columns = ['date', 'club1', 'SCORE', 'club2', 'DURATION', 'LOCATION', 'ATTENDANCE']
        fixtures_list_df.columns = ['date', 'club1', 'SCORE', 'club2', 'TIME', 'TV']

        df = elapsed_matches_df.append(fixtures_list_df)

        df = df.replace(r'^\s*$', np.nan, regex=True) \
            .replace('--', np.nan) \
            .fillna(value=np.nan) \
            .dropna(thresh=3) \
            .reset_index(drop=True)

        df = df.replace(r'^\s*$', np.nan, regex=True) \
            .replace('--', np.nan) \
            .fillna(value=np.nan) \
            .dropna(thresh=3) \
            .reset_index(drop=True)

        ProgressHandler.reset_progress()
        return df


import subprocess
import os
import sys
import re
from workflow import Workflow

adb_path = os.getenv('adb_path')
serial = os.getenv('serial')
api = os.getenv('device_api')
ip = os.getenv("ip")

def wordMatch(arg, sentence):
    words = arg.lower().split(" ")
    sentenceComponents = sentence.lower().split(" ")
    for word in words:
        included = False
        for sentenceComponent in sentenceComponents:
            if word in sentenceComponent:
                included = True
                break
        if not included:
            return False
    return True

def main(wf):

    isWifiDevice = re.match(".*:5555$", serial)
    isEmulator = re.match("emulator-55[5-8][02468]", serial)
    arg = wf.args[0].strip()
    log.debug(arg)
    addAll = False
    if arg == '':
        addAll = True

    itemCount = 0

    # SHOW INSTALLED PACKAGES
    title = "Show apps list"

    if addAll or (wordMatch(arg, title + " start launch uninstall force stop clear info") and not arg.startswith("ins")):
        func = ""
        if "start" in arg.lower() or "launch" in arg.lower():
            title = "Select app to launch"
            func = "start_app"
        elif "uninstall" in arg.lower():
            title = "Select app to uninstall"
            func = "uninstall_app"
        elif "force" in arg.lower() or "stop" in arg.lower():
            title = "Select app to force stop"
            func = "force_stop"
        elif "clear" in arg.lower():
            title = "Select app to clear data"
            func = "clear_app_data"
        elif "info" in arg.lower():
            title = "Select app to show app info"
            func = "app_info"
        it = wf.add_item(title=title,
                         uid="list_app",
                         arg="list_app",
                         valid=True)
        it.setvar("func", func)
        if func == "":
            m = it.add_modifier('cmd', "Select app to launch")
            m.setvar("func", "start_app")

            m = it.add_modifier('alt', "Select app to uninstall")
            m.setvar("func", "uninstall_app")

            m = it.add_modifier('ctrl', "Select app to force stop")
            m.setvar("func", "force_stop")

            m = it.add_modifier('fn', "Select app to clear data")
            m.setvar("func", "clear_app_data")

            m = it.add_modifier('shift', "Select app to show app info")
            m.setvar("func", "app_info")
        itemCount += 1

    # INSTALL APK
    title = "Install apk"

    if addAll or wordMatch(arg, title):
        wf.add_item(title=title,
                    uid="install_apk",
                    arg="install_apk",
                    subtitle="install apk or apks",
                    valid=True)
        itemCount += 1

    # SCREENSHOT
    title = "Take screenshot"

    if addAll or wordMatch(arg, title):
        it = wf.add_item(title=title,
                         uid="screenshot",
                         arg="screenshot:to_clipboard",
                         subtitle="take a screenshot and copy to clipboard, `cmd` to copy to desktop",
                         valid=True)
        it.add_modifier(
            'cmd', 'take a screenshot and copy to desktop', arg="screenshot:to_desktop")
        itemCount += 1

    # OPEN SETTINGS
    title = "Open settings"

    if addAll or wordMatch(arg, title):
        it = wf.add_item(title=title,
                         uid="open_settings",
                         arg="open_settings",
                         subtitle="'cmd' - Dev Tool, 'alt' - WiFi, 'ctrl' - App, 'fn' - Date, 'shift' - Accessibility",
                         valid=True)
        it.add_modifier('cmd', 'Open Developer Settings',
                        arg="open_settings:developer_options")
        it.add_modifier('alt', 'Open WiFi Settings', arg="open_settings:wifi")
        it.add_modifier('fn', 'Open Date Settings', arg="open_settings:date")
        it.add_modifier('shift', 'Open Accessibility Settings',
                        arg="open_settings:accessibility")
        it.add_modifier('ctrl', 'Open Application Settings',
                        arg="open_settings:application")
        itemCount += 1

    # TOGGLE DEBUG LAYOUT
    title = "Toggle debug layout"

    if addAll or wordMatch(arg, title):
        it = wf.add_item(title=title,
                         uid="debug_layout",
                         arg="debug_layout",
                         valid=True)
        itemCount += 1
        it.add_modifier('cmd', 'Toggle pointer location',
                        arg="pointer_location")
        it.add_modifier('alt', 'Toggle show taps', arg="show_taps")
        it.add_modifier('ctrl', 'Toggle GPU profile', arg="gpu_profile")
        it.add_modifier('fn', 'Toggle GPU overdraw', arg="gpu_overdraw")
        it.add_modifier('shift', 'Turn off everything', arg="debug_off")

    # DEMO MODE
    if api and int(api) >= 23:
        title = "Toggle demo mode"

        if addAll or wordMatch(arg, title):
            wf.add_item(title=title,
                        uid="demo_mode",
                        arg="demo_mode",
                        valid=True)
            itemCount += 1

    # REBOOT SYSTEM
    title = "Reboot system"

    if addAll or wordMatch(arg, title) or wordMatch(arg, "restart"):
        it = wf.add_item(title=title,
                         uid="adb_reboot",
                         arg="adb_reboot",
                         subtitle="'cmd' - Bootloader, 'alt' - Recovery, 'ctrl' - Sideload",
                         valid=True)
        itemCount += 1
        it.add_modifier('cmd', 'Bootloader', arg="adb_reboot:bootloader")
        it.add_modifier('alt', 'Recovery', arg="adb_reboot:recovery")
        it.add_modifier('ctrl', 'Sideload', arg="adb_reboot:sideload")

    # CONNECT OVER WIFI
    if not isWifiDevice and not isEmulator and ip:
        title = "Connect over Wi-Fi"

        if addAll or wordMatch(arg, title):
            wf.add_item(title=title,
                        subtitle=ip,
                        uid="debug_wifi",
                        arg="debug_wifi",
                        valid=True)
            itemCount += 1

    # KEY INPUT
    title = "Keyevent input"

    if addAll or wordMatch(arg, title):
        it = wf.add_item(title=title,
                         uid="keyevent_input",
                         arg="keyevent_input",
                         valid=True)
        it.add_modifier("cmd", "Back", arg='keyevent_input_KEYCODE_BACK')
        it.add_modifier("alt", "Home", arg='keyevent_input_KEYCODE_HOME')
        it.add_modifier("ctrl", "App Switch",
                        arg='keyevent_input_KEYCODE_APP_SWITCH')
        it.add_modifier("fn", "Escape", arg='keyevent_input_KEYCODE_ESCAPE')
        it.add_modifier("shift", "Power", arg='keyevent_input_KEYCODE_POWER')
        itemCount += 1

    # DUMP STACK
    title = "Dump task stacks"

    if addAll or wordMatch(arg, title):
        it = wf.add_item(title=title,
                         uid="dump_stack",
                         arg="dump_stack",
                         valid=True)
        itemCount += 1
        it.add_modifier('cmd', 'Dump the first application',
                        arg="dump_stack:first_app")
        it.add_modifier('alt', 'Dump the first stack',
                        arg="dump_stack:first_stack")

    # DEMO MODE
    if api and int(api) >= 23:
        title = "Dump notifications"

        if addAll or wordMatch(arg, title):
            it = wf.add_item(title=title,
                        uid="dump_notification",
                        arg="dump_notification",
                        valid=True)
            itemCount += 1
            it.add_modifier('alt', 'Exclude empty notification', arg="dump_notification:exclude_empty")

    # Screen Copy with scrcpy
    title = "Screen Copy with scrcpy"
    log.debug("Screen Copy with scrcpy")
    try:
        rc = subprocess.check_output(['/usr/local/bin/scrcpy', '-v'])
    except OSError:
        log.error("OS error 1")
        try:
            rc = subprocess.check_output(['/opt/homebrew/bin/scrcpy', '-v'])
        except OSError:
            log.error("OS error 2")
            rc = -1
    if rc != -1 and (addAll or wordMatch(arg, title)) and not isEmulator:
        it = wf.add_item(title=title,
                         uid="scr_cpy",
                         arg="scr_cpy",
                         valid=True)

        it.setvar("dimension", 0)
        mod = it.add_modifier(
            "cmd", subtitle="Run with maximum resolution restriction 1024")
        mod.setvar("dimension", 1024)
        mod = it.add_modifier(
            "alt", subtitle="Record screen and save to User folder")
        mod.setvar("record", 1)
        log.debug("Manufacturer {0}".format(
            os.getenv("ro.product.manufacturer")))
        if os.getenv("ro.product.manufacturer") == "Oculus" and os.getenv("name") == "Quest 2":
            mod = it.add_modifier("ctrl", subtitle="Quest 2 - Mirror one eye")
            mod.setvar("crop", "1832:1920:0:0")
        itemCount += 1
    log.debug(rc)

    # COMMAND HISTORY
    title = "Command history"
    lastFuncs = wf.cached_data('last_func:' + os.getenv("his_tag"), max_age=0)

    if (addAll or wordMatch(arg, title)) and lastFuncs and len(lastFuncs) > 0:
        it = wf.add_item(title=title,
                         subtitle="show command history",
                         arg="cmd_history",
                         valid=True)
        itemCount += 1
        it.add_modifier('cmd', 'clear command history',
                        arg="cmd_history:clear")

    idx = 1
    while idx > 0:
        config = os.getenv('self_script_device_%d' % idx)
        if config:
            title = config.split("|")[0]
            if addAll or wordMatch(arg, title) or wordMatch(arg, "Self script"):
                path = config.split("|")[1]
                it = wf.add_item(title=title,
                                 subtitle="with script: %s" % path,
                                 arg="self_script_device_%d" % idx,
                                 valid=True)
                it.setvar("self_script_device", config)
                mod = it.add_modifier("cmd", subtitle="apply cmd modifier")
                mod.setvar("mod", "cmd")
                mod = it.add_modifier("alt", subtitle="apply alt modifier")
                mod.setvar("mod", "alt")
                mod = it.add_modifier("ctrl", subtitle="apply ctrl modifier")
                mod.setvar("mod", "ctrl")
                mod = it.add_modifier("fn", subtitle="apply fn modifier")
                mod.setvar("mod", "fn")
                mod = it.add_modifier("shift", subtitle="apply shift modifier")
                mod.setvar("mod", "shift")
            idx = idx + 1
        else:
            idx = -1

    # CUSTOM ACTION
    if itemCount == 0:
        it = wf.add_item(title="Execute custom command for " + serial,
                         subtitle="adb " + arg,
                         arg="adb_cmd:in_terminal:" + arg,
                         valid=True)
        m = it.add_modifier('cmd', 'Run without opening terminal',
                            arg="adb_cmd:in_background:" + arg)

    wf.send_feedback()

if __name__ == '__main__':
    wf = Workflow()
    log = wf.logger
    sys.exit(wf.run(main))


# -*- coding: utf-8 -*-
"""
Created on Sun Aug  2 16:49:25 2020

@author: solis

module uses flopy extensively (https://modflowpy.github.io/flopydoc)

functionallity
Plot heads time series at the end of each stress period
    functions plot_heads_sg & plot_heads_ug
Gets heads time series at the end of each stress period
    functions get_heads_sg & get_heads_ug
"""
import numpy as np

def plot_heads_sg(org: str, rows_cols: tuple, dir_out: str, **kwargs):
    """
    It plots heads simulated in the last time step of each stress period.
        Heads are read from a bhd file in a structured grid (no DISV)
    args:
        org: bhd file
        rows_cols: list of pairs (row, column) to be plotted
        dir_out str: directory where png plots will be saved
    optional args
        layers tuple: list of layers to be plotted; is not present all layers
            will bel plotted
        ylabel str: label in y axis
        more optional args in get_heads_sg, get_requested_layers and xy_plot_1g
    """
    from os.path import join

    if 'ylabel' in kwargs:
        ylabel = kwargs['ylabel']
    else:
        ylabel = 'm asl'

    irows_cols = [(item[0] - 1, item[1] - 1) for item in rows_cols]
    hds, times = get_heads_sg(org, irows_cols, **kwargs)
    ilayers = get_requested_layers(hds.shape[1], **kwargs)

    for i, rc in enumerate(irows_cols):
        heads_ly = []
        leg_ly = []
        for j, layer in enumerate(ilayers):
            title = f'Head in row {rc[0]+1:n}, column {rc[1]+1:n}'
            dst = join(dir_out, f'hd_row_{rc[0]+1:n}_col_{rc[1]+1:n}.png')
            heads_ly.append(hds[i, j, :])
            leg_ly.append(f'ly {layer+1:n}')
        xy_plot_1g(title, times, heads_ly, leg_ly, ylabel, dst, **kwargs)

def plot_heads_ug(org: str, cells: tuple, dir_out: str, **kwargs):
    """
    It plots heads simulated in the last time step of each stress period.
        Heads are read from a bhd file in a unstructured grid (DISV)
    args:
        org: bhd file
        cells: list o cells to be plotted
        dir_out str: directory where png plots will be saved. If not present
            plot is shown in the screen
        xlabel str: label in x axis
    optional args
        layers tuple: list of layers to be plotted; is not present all layers
            will bel plotted
        ylabel str: label in y axis
        more optional args in get_heads_ug, get_requested_layers and xy_plot_1g
    """
    from os.path import join

    if 'ylabel' in kwargs:
        ylabel = kwargs['ylabel']
    else:
        ylabel = 'm asl'

    icells = [item-1 for item in cells]
    hds, times = get_heads_ug(org, icells, **kwargs)
    if 'layers' in kwargs:
        ilayers = [item-1 for item in kwargs['layers']]
    else:
        ilayers = [item for item in range(hds.shape[1])]
    for i, cell in enumerate(icells):
        heads_ly = []
        leg_ly = []
        for j, layer in enumerate(ilayers):
            title = f'Head in cell {cell+1:n}'
            dst = join(dir_out, f'hd_cell_{cell+1:n}.png')
            heads_ly.append(hds[i, j, :])
            leg_ly.append(f'ly {layer+1:n}')
        xy_plot_1g(title, times, heads_ly, leg_ly, ylabel, dst, **kwargs)

def get_disv_dimension(org):
    """
    it returns grid dimension as a tuple with 3 items by reading disv file:
        nlay: layers number
        ncpl: cell per layer number
        nvert: vertex number
    """
    import os.path

    ext = os.path.splitext(org)[1]
    if ext.lower() != '.disv':
        raise ValueError('org must have disv extension')

    i = 0
    kwords = ('NLAY', 'NCPL', 'NVERT')
    disvd = []
    fi = open(org)
    for line in fi:
        if kwords[i] in line:
            disvd.append(int(line.split()[1]))
            if len(disvd) == 3:
                return tuple(disvd)
            i += 1
    raise ValueError(f'missed key words in {org}')

def get_heads_sg(org: str, rows_cols: list, **kwargs):
    """
    it returns a np.array with the heads in the last time_step of each stress
        period for cells referendes as pairs (row, column). The heads are read
        from org, that is a file with .bhd extension.
    """
    import flopy.utils.binaryfile as bf

    fi = bf.HeadFile(org)
    _, ii = get_last_ts_sp(fi, **kwargs)
    times = fi.get_times()

    hds = fi.get_data(kstpkper=(0, 0))
    nlays = hds.shape[0]
    hd_ts = np.empty((len(rows_cols), nlays, len(ii)), np.float32)

    ilayers = get_requested_layers(nlays, **kwargs)

    # TODO:
    for i, rc in enumerate(rows_cols):
        for j, layer in enumerate(ilayers):
            hds = fi.get_ts((layer, rc[0], rc[1]))
            for k, i1 in enumerate(ii):
                hd_ts[i, j, k] = hds[i1][1]
    return hd_ts, times

def get_heads_ug(org: str, cells: list, **kwargs):
    """
    it returns a np.array with the heads in the last time_step of each stress
        period for cells in cells. The heads are read from org, that is a file
        with .bhd extension.
    """
    import flopy.utils.binaryfile as bf

    fi = bf.HeadFile(org)
    ts_sp, _ = get_last_ts_sp(fi, **kwargs)
    times = np.array(fi.get_times(), np.float32)

    for i, item in enumerate(ts_sp):
        hds = fi.get_data(kstpkper=item)
        if i == 0:
            nlays = hds.shape[0]
            hd_ts = np.empty((len(cells), nlays, len(ts_sp)), np.float32)

        for j, cell in enumerate(cells):
            for ilay in range(nlays):
                hd_ts[j, ilay, i] = hds[ilay, 0, cell]
    return hd_ts, times

def get_last_ts_sp(fi, **kwargs):
    """
    it returns 2 items:
        a list of pairs (time_step, stress_period) in witch time_step is the
            last time_step
        a list
    args
    fi: object heads binary file
    """
    import sqlite3

    create_db = \
    'create table t1 (time_step integer, stress_period integer, ' +\
    'primary key (time_step, stress_period) )'
    insert = \
    'insert into t1 (time_step, stress_period) values (?, ?)'
    select1 = 'select distinct stress_period from t1 order by stress_period'
    select2 = 'select max(time_step) from t1 where stress_period=?'

    con = sqlite3.connect(':memory:')
    cur = con.cursor()

    ts_sp = fi.get_kstpkper()  # time steps & stress periods

    if 'time_steps' not in kwargs or \
    'time_steps' in kwargs and not kwargs['time_steps']:
        cur.execute(create_db)

        for i, item in enumerate(ts_sp):
            cur.execute(insert, (int(item[0]), int(item[1])))
        cur.execute(select1)
        stress_periods = [ row[0] for row in cur.fetchall()]
        tsl_sp = []
        ii = []
        for i, stress_period in enumerate(stress_periods):
            cur.execute(select2, (stress_period,))
            row = cur.fetchone()
            tsl_sp.append((row[0], stress_period))
            ii.append(i)
        return tuple(tsl_sp), tuple(ii)
    else:
        return ts_sp, tuple([i for i in range(len(ts_sp))])

def get_requested_layers(nlays: int, **kwargs):
    """
    get the list of the index of layers to plott
   """
    if 'layers' in kwargs:
        return [item-1 for item in kwargs['layers']]
    else:
        return [item for item in range(nlays)]

def xy_plot_1g(title: str, x: list, ys: list, legs: list, ylabel: str,
               dst: str, **kwargs: dict):
    """
    plot a png file with head time series in a cell. As ys is a list
        of head time series in all or at least one layer, several series
        can be drawn in the same axis (the figure has only one axis in
        matplotlib terminlogy)
    args
        title: figure title
        x: list of stress periods
        ys: list of ts each element corresponds to one layer
        legs: list of legends, each one correspond to ys element
        ylabel: y axis label
        dst: output file name
    optional args
        xlabel: x axis label, default str 'stress periods'
        to_screen: figures are show in the screen, default bool False
    returns
        None
    """
    from datetime import date
    import warnings
    import matplotlib.pyplot as plt
    import matplotlib as mpl
    # parámetros específicos
    warnings.simplefilter(action='ignore', category=FutureWarning)
    mpl.rc('font', size=8)
    mpl.rc('axes', labelsize=8, titlesize= 10, grid=True)
    mpl.rc('axes.spines', right=False, top=False)
    mpl.rc('xtick', direction='out', top=False)
    mpl.rc('ytick', direction='out', right=False)
    mpl.rc('lines', linewidth=0.8, linestyle='-', marker='.', markersize=4)
    mpl.rc('legend', fontsize=8, framealpha=0.5, loc='best')

    fig, ax = plt.subplots()

    plt.suptitle(title)
    ax.set_ylabel(ylabel)
    if 'xlabel' in kwargs:
        ax.set_xlabel(kwargs['xlabel'])

    if isinstance(x[0], date):
        fig.autofmt_xdate()

    for y1, leg1 in zip(ys,legs):
        ax.plot(x, y1, label=leg1)
    plt.legend(framealpha=0.5, loc='best')

    if 'to_screen' in kwargs and  kwargs['to_screen']:
        plt.show()

    fig.savefig(dst)

    plt.close('all')
    plt.rcdefaults()

def plot_heads(org, lrc: list, ylabel: str, **kwargs):
    """
    Deprecated, use plot_head_sg instead

    it plots heads from bhd modflow 6 ourput files; heads can be plotted
        with stress periods or time steps in x axis
    now only structured grids can be used but unstructured grids will be
        addedd soon
    to read bhs fiel ir uses flopy

    https://modflowpy.github.io/flopydoc/binaryfile.html
    optional arguments
        date0 str: a str date representation as yyyy-mm-dd (it's not used yet)
        dir_out str: directory where png plots will be saved. If nor present
            plot is shown in the screen
        xlabel str: label in x axis -xlabel-
        time_step bool: if present and is True time steps are represented
    """
    import sqlite3
    import flopy.utils.binaryfile as bf
    create_db = \
    'create table t1 (fid integer, time_step integer, stress_period integer, ' +\
    'primary key (fid, time_step, stress_period) )'
    insert = \
    'insert into t1 (fid, time_step, stress_period) values (?, ?, ?)'
    select1 = 'select distinct stress_period from t1 order by stress_period'
    select2 = 'select max(fid) from t1 where stress_period=?'

    con = sqlite3.connect(':memory:')
    cur = con.cursor()

    fi = bf.HeadFile(org)
    if 'time_steps' not in kwargs or \
    'time_steps' in kwargs and not kwargs['time_steps']:
        cur.execute(create_db)
        # Get a list of unique stress periods and time steps in the file
        kstpkper = fi.get_kstpkper()
        for i, item in enumerate(kstpkper):
            cur.execute(insert, (int(i), int(item[0]), int(item[1])))
        cur.execute(select1)
        stress_periods = [ row[0] for row in cur.fetchall()]
        ii = []
        for stress_period in stress_periods:
            cur.execute(select2, (stress_period,))
            ii1 = cur.fetchone()
            ii.append(ii1[0])

    hds = fi.get_ts(lrc)
    fi.close()

    if 'time_steps' in kwargs and kwargs['time_steps']:
        x = hds[:, 0]
        suffix = '_ts'
    else:
        x = np.array([i for i in range(len(stress_periods))], np.float32)
        suffix = '_sp'
        y = np.empty((len(stress_periods)), np.float32)

    for i in range(len(lrc)):
        title = f'Head in layer {lrc[i][0]:n}, row {lrc[i][1]:n}, ' +\
        f'col {lrc[i][2]:n}'
        if 'dir_out' in kwargs:
            kwargs['name_file'] = \
            f'L{lrc[i][0]:n}_R{lrc[i][1]:n}_C{lrc[i][2]:n}' + suffix
        print(title)

        ycol = i + 1
        if 'time_steps' in kwargs and kwargs['time_steps']:
            y = hds[:, ycol]
        else:
            for j, ii1 in enumerate(ii):
                y[j] = hds[ii1, ycol]

        xy_plot_1g(title, x, y, ylabel, kwargs)
    con.close()