text:
# Title: Optimization result saving
# Description: This incldes hourly dispatch/import&export/hydro level/cost/detailed information related to the model variable.

from __future__ import print_function, division, absolute_import

import os
import pandas as pd
import logging
from pyomo.environ import *
from pyomo.core import *
from pyomo.core import Constraint
from pyomo.core.expr.numeric_expr import *

logger = logging.getLogger(__name__)

formatter = logging.Formatter(
    "%(asctime)s - %(levelname)s - %(name)s - %(message)s")

file_handler = logging.FileHandler('log/results_save.log')
file_handler.setLevel(logging.DEBUG)
file_handler.setFormatter(formatter)

stream_handler = logging.StreamHandler()
stream_handler.setFormatter(formatter)

logger.addHandler(file_handler)
logger.addHandler(stream_handler)

sce = []
annualcost = []
capex = []
inv_gen = []
inv_stor = []
inv_grid = []
opexfix = []
fix_gen = []
fix_stor = []
fix_grid = []
opexvar = []
var_gen = []
var_stor = []
var_startup = []
var_co2 = []
var_exchange = []
nuc = []
pv_large = []
pv_small = []
CCGT_new = []
OCGT_new = []
onshore = []
offshore = []
ires = []
hydro_ror = []
hydro_dam = []
phs_power = []
phs_energy = []
battery1_power = []
battery1_energy = []
battery4_power = []
battery4_energy = []
battery8_power = []
battery8_energy = []
ptg_power = []
importtranscapacity = []
exporttranscapacity = []
nucpro = []
pvlargepro = []
pvsmallpro = []
onshorepro = []
offshorepro = []
gasccgtpro = []
gasocgtpro = []
hydro_rorpro = []
hydro_dampro = []
phs_char = []
phs_disc = []
battery1_char = []
battery1_disc = []
battery4_char = []
battery4_disc = []
battery8_char = []
battery8_disc = []
ptg_char = []
ptg_disc = []
excess = []
exchange = []
demand = []
nucper = []
resper = []
fossilper = []
emission = []
totalemission = []
shadowprice = []

def detail_save_results_no_uc(instance, sce_name, outputpath):

    dirName = outputpath + sce_name

    cobject = getattr(instance, 'ElecBalanceConstraint')

    # Create target directory if it doesn't exist
    if not os.path.exists(dirName):
        os.mkdir(dirName)

    save_path = dirName + '/cost.csv'

    with open(save_path, 'w') as f:
        # f.write('Total system discounted cost:\n')

        f.write(
            '\nTotal regional discounted cost(bneuro) all studied period:\n')
        f.write('region,type,value\n')
        for r in instance.REGION_ID:
            exchangecost = 0
            for (y, ny) in instance.YEARPERIOD:
                for i in range(0, ny):
                    exchangecost = exchangecost + (1 + instance.SDR)**(
                        instance.REFYEAR - y - i) * sum(
                            ExchangeHourlyCostFunction(instance, r, y))

            f.write('%s,%s,%s\n' %
                    (r, 'total',
                     (instance.regionalcost[r].value + exchangecost) / 1e9))
            f.write('%s,%s,%s\n' %
                    (r, 'inv', instance.regionalcostinv[r].value / 1e9))
            f.write('%s,%s,%s\n' %
                    (r, 'fix', instance.regionalcostfix[r].value / 1e9))
            f.write('%s,%s,%s\n' %
                    (r, 'var', instance.regionalcostvar[r].value / 1e9))
            f.write('%s,%s,%s\n' %
                    (r, 'co2', instance.regionalcostco2[r].value / 1e9))
            f.write('%s,%s,%s\n' % (r, 'exchange', exchangecost / 1e9))

        f.write('\nRegional year non-discounted cost(bneuro):\n')
        f.write('region,year,type,value\n')

        for r in instance.REGION_ID:
            for y in instance.YEAR:
                f.write('%s,%s,%s,%s\n' %
                        (r, y, 'average_shadow_price(euro/MWh)',
                         sum(instance.dual[cobject[r, y, h]]
                             for h in instance.TIMESLICE) / 8760))
                f.write(
                    '%s,%s,%s,%s\n' %
                    (r, y, 'year',
                     (instance.regionalyearcost[r, y].value +
                      sum(ExchangeHourlyCostFunction(instance, r, y))) / 1e9))
                f.write('%s,%s,%s,%s\n' %
                        (r, y, 'investment (bneuro)',
                         instance.regionalyearcostinv[r, y].value / 1e9))
                f.write('%s,%s,%s,%s\n' %
                        (r, y, 'inv-gen (bneuro)',
                         instance.regionalyearcostinvgen[r, y].value / 1e9))
                f.write('%s,%s,%s,%s\n' %
                        (r, y, 'inv-stor (bneuro)',
                         instance.regionalyearcostinvstor[r, y].value / 1e9))
                f.write('%s,%s,%s,%s\n' %
                        (r, y, 'inv-grid (bneuro)',
                         instance.regionalyearcostinvgrid[r, y].value / 1e9))
                f.write('%s,%s,%s,%s\n' %
                        (r, y, 'opexfix (bneuro)',
                         instance.regionalyearcostfix[r, y].value / 1e9))
                f.write('%s,%s,%s,%s\n' %
                        (r, y, 'fix-gen (bneuro)',
                         instance.regionalyearcostfixgen[r, y].value / 1e9))
                f.write('%s,%s,%s,%s\n' %
                        (r, y, 'fix-stor (bneuro)',
                         instance.regionalyearcostfixstor[r, y].value / 1e9))
                f.write('%s,%s,%s,%s\n' %
                        (r, y, 'fix-grid (bneuro)',
                         instance.regionalyearcostfixgrid[r, y].value / 1e9))
                f.write('%s,%s,%s,%s\n' %
                        (r, y, 'opexvar (bneuro)',
                         instance.regionalyearcostvar[r, y].value / 1e9))
                f.write('%s,%s,%s,%s\n' %
                        (r, y, 'var-gen (bneuro)',
                         instance.regionalyearcostvargen[r, y].value / 1e9))
                f.write('%s,%s,%s,%s\n' %
                        (r, y, 'var-stor (bneuro)',
                         instance.regionalyearcostvarstor[r, y].value / 1e9))
                f.write('%s,%s,%s,%s\n' %
                        (r, y, 'var-loadshedding (bneuro)',
                         instance.regionalyearcostvarloadshedding[r, y].value /
                         1e9))
                f.write(
                    '%s,%s,%s,%s\n' %
                    (r, y, 'var-startup (bneuro)',
                     instance.regionalyearcostvarstartup[r, y].value / 1e9))
                f.write('%s,%s,%s,%s\n' %
                        (r, y, 'var-co2 (bneuro)',
                         instance.regionalyearcostco2[r, y].value / 1e9))
                f.write(
                    '%s,%s,%s,%s\n' %
                    (r, y, 'var-exchange (bneuro)',
                     sum(ExchangeHourlyCostFunction(instance, r, y)) / 1e9))

        f.write('\nRegional year cost per technology:\n')
        f.write('region,year,technology,type,value\n')
        for r, y, d in instance.REGION_YEAR_TECH_GENERATION:
            f.write('%s,%s,%s,%s,%s\n' %
                    (r, y, d, 'gen-inv (bneuro)',
                     instance.regionalyearcostinvgentech[r, y, d].value / 1e9))
            f.write('%s,%s,%s,%s,%s\n' %
                    (r, y, d, 'gen-fix (bneuro)',
                     instance.regionalyearcostfixgentech[r, y, d].value / 1e9))
            f.write('%s,%s,%s,%s,%s\n' %
                    (r, y, d, 'gen-var (bneuro)',
                     instance.regionalyearcostvargentech[r, y, d].value / 1e9))
        for r, y, tt in instance.REGION_YEAR_TECH_STORAGE:
            f.write(
                '%s,%s,%s,%s,%s\n' %
                (r, y, tt, 'stor-inv (bneuro)',
                 instance.regionalyearcostinvstortech[r, y, tt].value / 1e9))
            f.write(
                '%s,%s,%s,%s,%s\n' %
                (r, y, tt, 'stor-fix (bneuro)',
                 instance.regionalyearcostfixstortech[r, y, tt].value / 1e9))
            f.write(
                '%s,%s,%s,%s,%s\n' %
                (r, y, tt, 'stor-var (bneuro)',
                 instance.regionalyearcostvarstortech[r, y, tt].value / 1e9))
        for r in instance.REGION_ID:
            for y in instance.YEAR:
                for l in instance.LINE:
                    f.write(
                        '%s,%s,%s,%s,%s\n' %
                        (r, y, l, 'grid-inv (bneuro)',
                         instance.regionalyearcostinvgridline[r, y, l].value /
                         1e9))
                    f.write(
                        '%s,%s,%s,%s,%s\n' %
                        (r, y, l, 'grid-fix (bneuro)',
                         instance.regionalyearcostfixgridline[r, y, l].value /
                         1e9))

    save_path1 = dirName + "/detail.csv"
    with open(save_path1, 'w') as f:
        f.write('Production composition:\n')
        f.write('region,year,type,value\n')
        for r in instance.REGION_ID:
            for y in instance.YEAR:
                f.write('%s,%s,%s,%s\n' %
                        (r, y, 'RES%',
                         round(
                             1 - instance.annualelecnucgen[r, y].value /
                             instance.annualelecgen[r, y].value -
                             instance.annualelecfossilgen[r, y].value /
                             instance.annualelecgen[r, y].value, 3)))
                f.write('%s,%s,%s,%s\n' %
                        (r, y, 'NUC%',
                         round(
                             instance.annualelecnucgen[r, y].value /
                             instance.annualelecgen[r, y].value, 3)))
                f.write('%s,%s,%s,%s\n' %
                        (r, y, 'FOSSIL%',
                         round(
                             instance.annualelecfossilgen[r, y].value /
                             instance.annualelecgen[r, y].value, 3)))

        f.write('\nNew installed capacity (GW):\n')
        f.write('region,year,technology,value\n')
        for r, y, t in instance.REGION_YEAR_TECH_GENERATION:
            f.write('%s,%s,%s,%s\n' %
                    (r, y, t,
                     instance.newinstalledgencapacity[r, y, t].value / 1000))
        for r, y, tt in instance.REGION_YEAR_TECH_STORAGE:
            f.write('%s,%s,%s,%s\n' %
                    (r, y, tt + '_power',
                     instance.newinstalledstorpower[r, y, tt].value / 1000))
            f.write('%s,%s,%s,%s\n' %
                    (r, y, tt + '_energy',
                     instance.newinstalledstorenergy[r, y, tt].value / 1000))

        f.write('\nAccumulated new installed capacity (GW):\n')
        f.write('region,year,technology,value\n')
        for r, y, t in instance.REGION_YEAR_TECH_GENERATION:
            f.write(
                '%s,%s,%s,%s\n' %
                (r, y, t,
                 instance.accunewinstalledgencapacity[r, y, t].value / 1000))
        for r, y, tt in instance.REGION_YEAR_TECH_STORAGE:
            f.write(
                '%s,%s,%s,%s\n' %
                (r, y, tt + '_power',
                 instance.accunewinstalledstorpower[r, y, tt].value / 1000))
            f.write(
                '%s,%s,%s,%s\n' %
                (r, y, tt + '_energy',
                 instance.accunewinstalledstorenergy[r, y, tt].value / 1000))

        f.write('\nTotal installed capacity (GW):\n')
        f.write('region,year,technology,value\n')
        for r, y, t in instance.REGION_YEAR_TECH_GENERATION:
            f.write('%s,%s,%s,%s\n' %
                    (r, y, t,
                     instance.totalinstalledgencapacity[r, y, t].value / 1000))
        for r, y, tt in instance.REGION_YEAR_TECH_STORAGE:
            f.write('%s,%s,%s,%s\n' %
                    (r, y, tt + '_power',
                     instance.totalinstalledstorpower[r, y, tt].value / 1000))
            f.write('%s,%s,%s,%s\n' %
                    (r, y, tt + '_energy (GWh)',
                     instance.totalinstalledstorenergy[r, y, tt].value / 1000))

        f.write('\nNew installed transmission capacity (GW):\n')
        f.write('line,year,value\n')
        for l in instance.LINE:
            for y in instance.YEAR:
                f.write(
                    '%s,%s,%s\n' %
                    (l, y, instance.newinstalledimporttranscapacity[l, y].value
                     / 1000))
                f.write(
                    '%s,%s,%s\n' %
                    (l, y, instance.newinstalledexporttranscapacity[l, y].value
                     / 1000))

        f.write('\nAccumulated new installed transmission capacity (GW):\n')
        f.write('line,year,value\n')
        for l in instance.LINE:
            for y in instance.YEAR:
                f.write(
                    '%s,%s,%s\n' %
                    (l, y, instance.newinstalledimporttranscapacity[l, y].value
                     / 1000))
                f.write(
                    '%s,%s,%s\n' %
                    (l, y, instance.newinstalledexporttranscapacity[l, y].value
                     / 1000))

        f.write('\nTotal installed transmission capacity (GW):\n')
        f.write('line,year,value\n')
        for l in instance.LINE:
            for y in instance.YEAR:
                f.write(
                    '%s,%s,%s\n' %
                    (l, y,
                     instance.totalinstalledimporttranscapacity[l, y].value /
                     1000))
                f.write(
                    '%s,%s,%s\n' %
                    (l, y,
                     instance.totalinstalledexporttranscapacity[l, y].value /
                     1000))

        f.write('\nAnnual demand (TWh):\n')
        f.write('region,year,value\n')
        for r in instance.REGION_ID:
            for y in instance.YEAR:
                f.write('%s,%s,%s\n' %
                        (r, y, round(instance.annualdemand[r, y].value / 1e6)))

        f.write('\nAnnual production (TWh):\n')
        f.write('region,year,technology,value\n')
        for r, y, d in instance.REGION_YEAR_TECH_GENERATION:
            f.write(
                '%s,%s,%s,%s\n' %
                (r, y, d,
                 round(instance.annualelectechgen[r, y, d].value / 1e6, 2)))
        for r, y, tt in instance.REGION_YEAR_TECH_STORAGE:
            if tt in instance.TECH_DSM:
                f.write(
                    '%s,%s,%s,%s\n' %
                    (r, y, tt + '_delayed',
                     round(instance.annualelectechchar[r, y, tt].value / 1e6,
                           2)))
                f.write(
                    '%s,%s,%s,%s\n' %
                    (r, y, tt + '_served',
                     round(-instance.annualelectechdisc[r, y, tt].value / 1e6,
                           2)))
            else:
                f.write(
                    '%s,%s,%s,%s\n' %
                    (r, y, tt + '_char',
                     round(-instance.annualelectechchar[r, y, tt].value / 1e6,
                           2)))
                f.write(
                    '%s,%s,%s,%s\n' %
                    (r, y, tt + '_disc',
                     round(instance.annualelectechdisc[r, y, tt].value / 1e6,
                           2)))
        for r in instance.REGION_ID:
            for y in instance.YEAR:
                f.write(
                    '%s,%s,%s,%s\n' %
                    (r, y, 'exchange',
                     round(instance.annualelecexchange[r, y].value / 1e6, 2)))
                f.write(
                    '%s,%s,%s,%s\n' %
                    (r, y, 'excess',
                     round(-instance.annualelecexcess[r, y].value / 1e6, 2)))
                f.write(
                    '%s,%s,%s,%s\n' %
                    (r, y, 'load_shedding',
                     round(instance.annualelecloadshedding[r, y].value / 1e6,
                           2)))
                f.write('%s,%s,%s,%s\n' %
                        (r, y, 'naturalgas',
                         round(instance.annualelecnaturalgas[r, y].value / 1e6,
                               2)))
                f.write(
                    '%s,%s,%s,%s\n' %
                    (r, y, 'flex_biogas',
                     round(instance.annualelecbiogas[r, y].value / 1e6, 2)))
                f.write('%s,%s,%s,%s\n' %
                        (r, y, 'gtp',
                         round(instance.annualelecgtp[r, y].value / 1e6, 2)))

        f.write('\nAnnual emissions (MT CO2eq):\n')
        f.write('REGION_ID,YEAR, value\n')
        for r in instance.REGION_ID:
            for y in instance.YEAR:
                f.write('%s, %s, %s\n' %
                        (r, y,
                         round(instance.annualemission[r, y].value / 1e9, 2)))

        f.write('\nEmissions per kwh (gCO2eq/kWh):\n')
        f.write('REGION_ID,YEAR, value\n')
        for r in instance.REGION_ID:
            for y in instance.YEAR:
                f.write('%s, %s, %s\n' %
                        (r, y,
                         round(
                             instance.annualemission[r, y].value /
                             instance.annualdemand[r, y].value, 2)))

    for r in instance.REGION_ID:
        for y in instance.YEAR:
            save_path = dirName + '/' + str(r) + '_' + str(
                y) + '_' + 'production1_MW.csv'
            save_path1 = dirName + '/' + str(r) + '_' + str(
                y) + '_' + 'production_MW.csv'
            with open(save_path, 'w') as f:
                f.write('hour,technology,value\n')
                for h in instance.TIMESLICE:
                    for d in instance.TECH_GENERATION:
                        if [r, y, d] in instance.REGION_YEAR_TECH_GENERATION:
                            f.write('%s,%s,%s\n' %
                                    (h, d,
                                     round(instance.electechgen[r, y, d,
                                                                h].value)))
                    for d in instance.TECH_GAS:
                        if [r, y, d] in instance.REGION_YEAR_TECH_GAS:
                            f.write(
                                '%s,%s,%s\n' %
                                (h, d + '_naturalgas',
                                 round(instance.electechnaturalgas[r, y, d,
                                                                   h].value)))
                            f.write('%s,%s,%s\n' %
                                    (h, d + '_biogas',
                                     round(instance.electechbiogas[r, y, d,
                                                                   h].value *
                                           instance.GenFullLoadEffi[y, d])))
                            f.write(
                                '%s,%s,%s\n' %
                                (h, d + '_gtp',
                                 round(instance.electechgtp[r, y, d, h].value *
                                       instance.GenFullLoadEffi[y, d])))

                    for tt in instance.TECH_STORAGE:
                        if [r, y, tt] in instance.REGION_YEAR_TECH_STORAGE:
                            if tt in instance.TECH_DSM:
                                f.write(
                                    '%s,%s,%s\n' %
                                    (h, tt + '_delayed',
                                     round(instance.electechchar[r, y, tt,
                                                                 h].value)))
                                f.write(
                                    '%s,%s,%s\n' %
                                    (h, tt + '_served',
                                     round(-instance.electechdisc[r, y, tt,
                                                                  h].value)))
                                f.write(
                                    '%s,%s,%s\n' %
                                    (h, tt + '_delayed_stored',
                                     round(instance.electechstored[r, y, tt,
                                                                   h].value)))
                            else:
                                f.write(
                                    '%s,%s,%s\n' %
                                    (h, tt + '_char',
                                     round(-instance.electechchar[r, y, tt,
                                                                  h].value)))
                                f.write(
                                    '%s,%s,%s\n' %
                                    (h, tt + '_disc',
                                     round(instance.electechdisc[r, y, tt,
                                                                 h].value)))
                                f.write(
                                    '%s,%s,%s\n' %
                                    (h, tt + '_stored',
                                     round(instance.electechstored[r, y, tt,
                                                                   h].value)))

                    f.write('%s,%s,%s\n' %
                            (h, 'demand', round(instance.Demand[r, y, h])))
                    f.write(
                        '%s,%s,%s\n' %
                        (h, 'shadow_price', instance.dual[cobject[r, y, h]]))
                    f.write('%s,%s,%s\n' %
                            (h, 'exchange',
                             round(instance.elecexchange[r, y, h].value)))
                    f.write('%s,%s,%s\n' %
                            (h, 'excess',
                             round(-instance.elecexcess[r, y, h].value)))
                    f.write('%s,%s,%s\n' %
                            (h, 'load_shedding',
                             round(instance.elecloadshedding[r, y, h].value)))

                f.close()

                open(save_path1, 'w')
                pd.read_csv(save_path).pivot(index='hour',
                                             columns='technology',
                                             values='value').to_csv(save_path1)
                os.remove(save_path)

    for y in instance.YEAR:
        save_path = dirName + '/' + str(y) + '_' + 'transmission1_MW.csv'
        save_path1 = dirName + '/' + str(y) + '_' + 'transmission_MW.csv'
        with open(save_path, 'w') as f:
            f.write('hour,line,value\n')
            for h in instance.TIMESLICE:
                for l in instance.LINE:
                    f.write(
                        '%s,%s,%s\n' %
                        (h, l, round(instance.eleclexchange[l, y, h].value)))
            f.close()

            open(save_path1, 'w')
            pd.read_csv(save_path).pivot(index='hour',
                                         columns='line',
                                         values='value').to_csv(save_path1)
            os.remove(save_path)

    save_path = dirName + '/' + 'HydroDamWeekIniLevel.csv'
    with open(save_path, 'w') as f:
        f.write('region,year,week,tech,storage-level,weekprod(MWh)\n')
        for w in instance.WEEK:
            for r, y, t in instance.REGION_YEAR_TECH_HD:
                f.write(
                    '%s,%s,%s,%s,%s,%s\n' %
                    (r, y, w, t,
                     round(instance.elechdweekinilevel[r, y, t, w].value, 3),
                     round(instance.elechdweekprod[r, y, t, w].value)))

    return

def detail_save_results_uc(instance, sce_name, outputpath):

    dirName = outputpath + sce_name

    cobject = getattr(instance, 'ElecBalanceConstraint')

    # Create target directory if it doesn't exist
    if not os.path.exists(dirName):
        os.mkdir(dirName)

    save_path = dirName + '/cost.csv'

    with open(save_path, 'w') as f:
        # f.write('Total system discounted cost:\n')

        f.write(
            '\nTotal regional discounted cost(bneuro) all studied period:\n')
        f.write('region,type,value\n')
        for r in instance.REGION_ID:
            exchangecost = 0
            for (y, ny) in instance.YEARPERIOD:
                for i in range(0, ny - 1):
                    exchangecost = exchangecost + (1 + instance.SDR)**(
                        instance.REFYEAR - y - i) * sum(
                            ExchangeHourlyCostFunction(instance, r, y))

            f.write('%s,%s,%s\n' %
                    (r, 'total',
                     (instance.regionalcost[r].value + exchangecost) / 1e9))
            f.write('%s,%s,%s\n' %
                    (r, 'inv', instance.regionalcostinv[r].value / 1e9))
            f.write('%s,%s,%s\n' %
                    (r, 'fix', instance.regionalcostfix[r].value / 1e9))
            f.write('%s,%s,%s\n' %
                    (r, 'var', instance.regionalcostvar[r].value / 1e9))
            f.write('%s,%s,%s\n' %
                    (r, 'co2', instance.regionalcostco2[r].value / 1e9))
            f.write('%s,%s,%s\n' % (r, 'exchange', exchangecost / 1e9))

        f.write('\nRegional year non-discounted cost(bneuro):\n')
        f.write('region,year,type,value\n')

        for r in instance.REGION_ID:
            for y in instance.YEAR:
                f.write('%s,%s,%s,%s\n' %
                        (r, y, 'average_shadow_price(euro/MWh)',
                         sum(instance.dual[cobject[r, y, h]]
                             for h in instance.TIMESLICE) / 8760))
                f.write(
                    '%s,%s,%s,%s\n' %
                    (r, y, 'year',
                     (instance.regionalyearcost[r, y].value +
                      sum(ExchangeHourlyCostFunction(instance, r, y))) / 1e9))
                f.write('%s,%s,%s,%s\n' %
                        (r, y, 'investment (bneuro)',
                         instance.regionalyearcostinv[r, y].value / 1e9))
                f.write('%s,%s,%s,%s\n' %
                        (r, y, 'inv-gen (bneuro)',
                         instance.regionalyearcostinvgen[r, y].value / 1e9))
                f.write('%s,%s,%s,%s\n' %
                        (r, y, 'inv-stor (bneuro)',
                         instance.regionalyearcostinvstor[r, y].value / 1e9))
                f.write('%s,%s,%s,%s\n' %
                        (r, y, 'inv-grid (bneuro)',
                         instance.regionalyearcostinvgrid[r, y].value / 1e9))
                f.write('%s,%s,%s,%s\n' %
                        (r, y, 'opexfix (bneuro)',
                         instance.regionalyearcostfix[r, y].value / 1e9))
                f.write('%s,%s,%s,%s\n' %
                        (r, y, 'fix-gen (bneuro)',
                         instance.regionalyearcostfixgen[r, y].value / 1e9))
                f.write('%s,%s,%s,%s\n' %
                        (r, y, 'fix-stor (bneuro)',
                         instance.regionalyearcostfixstor[r, y].value / 1e9))
                f.write('%s,%s,%s,%s\n' %
                        (r, y, 'fix-grid (bneuro)',
                         instance.regionalyearcostfixgrid[r, y].value / 1e9))
                f.write('%s,%s,%s,%s\n' %
                        (r, y, 'opexvar (bneuro)',
                         instance.regionalyearcostvar[r, y].value / 1e9))
                f.write('%s,%s,%s,%s\n' %
                        (r, y, 'var-gen (bneuro)',
                         instance.regionalyearcostvargen[r, y].value / 1e9))
                f.write('%s,%s,%s,%s\n' %
                        (r, y, 'var-stor (bneuro)',
                         instance.regionalyearcostvarstor[r, y].value / 1e9))
                f.write('%s,%s,%s,%s\n' %
                        (r, y, 'var-loadshedding (bneuro)',
                         instance.regionalyearcostvarloadshedding[r, y].value /
                         1e9))
                f.write(
                    '%s,%s,%s,%s\n' %
                    (r, y, 'var-startup (bneuro)',
                     instance.regionalyearcostvarstartup[r, y].value / 1e9))
                f.write('%s,%s,%s,%s\n' %
                        (r, y, 'var-co2 (bneuro)',
                         instance.regionalyearcostco2[r, y].value / 1e9))
                f.write(
                    '%s,%s,%s,%s\n' %
                    (r, y, 'var-exchange (bneuro)',
                     sum(ExchangeHourlyCostFunction(instance, r, y)) / 1e9))

        f.write('\nRegional year cost per technology:\n')
        f.write('region,year,technology,type,value\n')
        for r, y, d in instance.REGION_YEAR_TECH_GENERATION:
            f.write('%s,%s,%s,%s,%s\n' %
                    (r, y, d, 'gen-inv (bneuro)',
                     instance.regionalyearcostinvgentech[r, y, d].value / 1e9))
            f.write('%s,%s,%s,%s,%s\n' %
                    (r, y, d, 'gen-fix (bneuro)',
                     instance.regionalyearcostfixgentech[r, y, d].value / 1e9))
            f.write('%s,%s,%s,%s,%s\n' %
                    (r, y, d, 'gen-var (bneuro)',
                     instance.regionalyearcostvargentech[r, y, d].value / 1e9))
        for r, y, tt in instance.REGION_YEAR_TECH_STORAGE:
            f.write(
                '%s,%s,%s,%s,%s\n' %
                (r, y, tt, 'stor-inv (bneuro)',
                 instance.regionalyearcostinvstortech[r, y, tt].value / 1e9))
            f.write(
                '%s,%s,%s,%s,%s\n' %
                (r, y, tt, 'stor-fix (bneuro)',
                 instance.regionalyearcostfixstortech[r, y, tt].value / 1e9))
            f.write(
                '%s,%s,%s,%s,%s\n' %
                (r, y, tt, 'stor-var (bneuro)',
                 instance.regionalyearcostvarstortech[r, y, tt].value / 1e9))
        for r in instance.REGION_ID:
            for y in instance.YEAR:
                for l in instance.LINE:
                    f.write(
                        '%s,%s,%s,%s,%s\n' %
                        (r, y, l, 'grid-inv (bneuro)',
                         instance.regionalyearcostinvgridline[r, y, l].value /
                         1e9))
                    f.write(
                        '%s,%s,%s,%s,%s\n' %
                        (r, y, l, 'grid-fix (bneuro)',
                         instance.regionalyearcostfixgridline[r, y, l].value /
                         1e9))

    save_path1 = dirName + "/detail.csv"
    with open(save_path1, 'w') as f:
        f.write('Production composition:\n')
        f.write('region,year,type,value\n')
        for r in instance.REGION_ID:
            for y in instance.YEAR:
                f.write('%s,%s,%s,%s\n' %
                        (r, y, 'RES%',
                         round(
                             1 - instance.annualelecnucgen[r, y].value /
                             instance.annualelecgen[r, y].value -
                             instance.annualelecfossilgen[r, y].value /
                             instance.annualelecgen[r, y].value, 3)))
                f.write('%s,%s,%s,%s\n' %
                        (r, y, 'NUC%',
                         round(
                             instance.annualelecnucgen[r, y].value /
                             instance.annualelecgen[r, y].value, 3)))
                f.write('%s,%s,%s,%s\n' %
                        (r, y, 'FOSSIL%',
                         round(
                             instance.annualelecfossilgen[r, y].value /
                             instance.annualelecgen[r, y].value, 3)))

        f.write('\nNew installed capacity (GW):\n')
        f.write('region,year,technology,value\n')
        for r, y, t in instance.REGION_YEAR_TECH_GENERATION:
            f.write('%s,%s,%s,%s\n' %
                    (r, y, t,
                     instance.newinstalledgencapacity[r, y, t].value / 1000))
        for r, y, tt in instance.REGION_YEAR_TECH_STORAGE:
            f.write('%s,%s,%s,%s\n' %
                    (r, y, tt + '_power',
                     instance.newinstalledstorpower[r, y, tt].value / 1000))
            f.write('%s,%s,%s,%s\n' %
                    (r, y, tt + '_energy',
                     instance.newinstalledstorenergy[r, y, tt].value / 1000))

        f.write('\nAccumulated new installed capacity (GW):\n')
        f.write('region,year,technology,value\n')
        for r, y, t in instance.REGION_YEAR_TECH_GENERATION:
            f.write(
                '%s,%s,%s,%s\n' %
                (r, y, t,
                 instance.accunewinstalledgencapacity[r, y, t].value / 1000))
        for r, y, tt in instance.REGION_YEAR_TECH_STORAGE:
            f.write(
                '%s,%s,%s,%s\n' %
                (r, y, tt + '_power',
                 instance.accunewinstalledstorpower[r, y, tt].value / 1000))
            f.write(
                '%s,%s,%s,%s\n' %
                (r, y, tt + '_energy',
                 instance.accunewinstalledstorenergy[r, y, tt].value / 1000))

        f.write('\nTotal installed capacity (GW):\n')
        f.write('region,year,technology,value\n')
        for r, y, t in instance.REGION_YEAR_TECH_GENERATION:
            f.write('%s,%s,%s,%s\n' %
                    (r, y, t,
                     instance.totalinstalledgencapacity[r, y, t].value / 1000))
        for r, y, tt in instance.REGION_YEAR_TECH_STORAGE:
            f.write('%s,%s,%s,%s\n' %
                    (r, y, tt + '_power',
                     instance.totalinstalledstorpower[r, y, tt].value / 1000))
            f.write('%s,%s,%s,%s\n' %
                    (r, y, tt + '_energy (GWh)',
                     instance.totalinstalledstorenergy[r, y, tt].value / 1000))

        f.write('\nNew installed transmission capacity (GW):\n')
        f.write('line,year,value\n')
        for l in instance.LINE:
            for y in instance.YEAR:
                f.write(
                    '%s,%s,%s\n' %
                    (l, y, instance.newinstalledimporttranscapacity[l, y].value
                     / 1000))
                f.write(
                    '%s,%s,%s\n' %
                    (l, y, instance.newinstalledexporttranscapacity[l, y].value
                     / 1000))

        f.write('\nAccumulated new installed transmission capacity (GW):\n')
        f.write('line,year,value\n')
        for l in instance.LINE:
            for y in instance.YEAR:
                f.write(
                    '%s,%s,%s\n' %
                    (l, y, instance.newinstalledimporttranscapacity[l, y].value
                     / 1000))
                f.write(
                    '%s,%s,%s\n' %
                    (l, y, instance.newinstalledexporttranscapacity[l, y].value
                     / 1000))

        f.write('\nTotal installed transmission capacity (GW):\n')
        f.write('line,year,value\n')
        for l in instance.LINE:
            for y in instance.YEAR:
                f.write(
                    '%s,%s,%s\n' %
                    (l, y,
                     instance.totalinstalledimporttranscapacity[l, y].value /
                     1000))
                f.write(
                    '%s,%s,%s\n' %
                    (l, y,
                     instance.totalinstalledexporttranscapacity[l, y].value /
                     1000))

        f.write('\nAnnual demand (TWh):\n')
        f.write('region,year,value\n')
        for r in instance.REGION_ID:
            for y in instance.YEAR:
                f.write('%s,%s,%s\n' %
                        (r, y, round(instance.annualdemand[r, y].value / 1e6)))

        f.write('\nAnnual production (TWh):\n')
        f.write('region,year,technology,value\n')
        for r, y, d in instance.REGION_YEAR_TECH_GENERATION:
            f.write(
                '%s,%s,%s,%s\n' %
                (r, y, d,
                 round(instance.annualelectechgen[r, y, d].value / 1e6, 2)))
        for r, y, tt in instance.REGION_YEAR_TECH_STORAGE:
            if tt in instance.TECH_DSM:
                f.write(
                    '%s,%s,%s,%s\n' %
                    (r, y, tt + '_delayed',
                     round(instance.annualelectechchar[r, y, tt].value / 1e6,
                           2)))
                f.write(
                    '%s,%s,%s,%s\n' %
                    (r, y, tt + '_served',
                     round(-instance.annualelectechdisc[r, y, tt].value / 1e6,
                           2)))
            else:
                f.write(
                    '%s,%s,%s,%s\n' %
                    (r, y, tt + '_char',
                     round(-instance.annualelectechchar[r, y, tt].value / 1e6,
                           2)))
                f.write(
                    '%s,%s,%s,%s\n' %
                    (r, y, tt + '_disc',
                     round(instance.annualelectechdisc[r, y, tt].value / 1e6,
                           2)))
        for r in instance.REGION_ID:
            for y in instance.YEAR:
                f.write(
                    '%s,%s,%s,%s\n' %
                    (r, y, 'exchange',
                     round(instance.annualelecexchange[r, y].value / 1e6, 2)))
                f.write(
                    '%s,%s,%s,%s\n' %
                    (r, y, 'excess',
                     round(-instance.annualelecexcess[r, y].value / 1e6, 2)))
                f.write(
                    '%s,%s,%s,%s\n' %
                    (r, y, 'load_shedding',
                     round(instance.annualelecloadshedding[r, y].value / 1e6,
                           2)))
                f.write('%s,%s,%s,%s\n' %
                        (r, y, 'naturalgas',
                         round(instance.annualelecnaturalgas[r, y].value / 1e6,
                               2)))
                f.write(
                    '%s,%s,%s,%s\n' %
                    (r, y, 'flex_biogas',
                     round(instance.annualelecbiogas[r, y].value / 1e6, 2)))
                f.write('%s,%s,%s,%s\n' %
                        (r, y, 'gtp',
                         round(instance.annualelecgtp[r, y].value / 1e6, 2)))

        f.write('\nAnnual emissions (MT CO2eq):\n')
        f.write('REGION_ID,YEAR, value\n')
        for r in instance.REGION_ID:
            for y in instance.YEAR:
                f.write('%s, %s, %s\n' %
                        (r, y,
                         round(instance.annualemission[r, y].value / 1e9, 2)))

        f.write('\nEmissions per kwh (gCO2eq/kWh):\n')
        f.write('REGION_ID,YEAR, value\n')
        for r in instance.REGION_ID:
            for y in instance.YEAR:
                f.write('%s, %s, %s\n' %
                        (r, y,
                         round(
                             instance.annualemission[r, y].value /
                             instance.annualdemand[r, y].value, 2)))

    for r in instance.REGION_ID:
        for y in instance.YEAR:
            save_path = dirName + '/' + str(r) + '_' + str(
                y) + '_' + 'production1_MW.csv'
            save_path1 = dirName + '/' + str(r) + '_' + str(
                y) + '_' + 'production_MW.csv'
            with open(save_path, 'w') as f:
                f.write('hour,technology,value\n')
                for h in instance.TIMESLICE:
                    for d in instance.TECH_GENERATION:
                        if [r, y, d] in instance.REGION_YEAR_TECH_GENERATION:
                            f.write('%s,%s,%s\n' %
                                    (h, d,
                                     round(instance.electechgen[r, y, d,
                                                                h].value)))

                    for d in instance.TECH_GAS:
                        if [r, y, d] in instance.REGION_YEAR_TECH_GAS:
                            f.write(
                                '%s,%s,%s\n' %
                                (h, d + '_naturalgas',
                                 round(instance.electechnaturalgas[r, y, d,
                                                                   h].value)))
                            f.write('%s,%s,%s\n' %
                                    (h, d + '_biogas',
                                     round(instance.electechbiogas[r, y, d,
                                                                   h].value *
                                           instance.GenFullLoadEffi[y, d])))
                            f.write(
                                '%s,%s,%s\n' %
                                (h, d + '_gtp',
                                 round(instance.electechgtp[r, y, d, h].value *
                                       instance.GenFullLoadEffi[y, d])))

                    for d in instance.TECH_UC:
                        if [r, y, d] in instance.REGION_YEAR_TECH_UC_Gen:
                            f.write('%s,%s,%s\n' %
                                    (h, d + '_committed',
                                     round(instance.eleccommitgen[r, y, d,
                                                                  h].value)))

                            f.write('%s,%s,%s\n' %
                                    (h, d + '_startup',
                                     round(instance.elecstartupgen[r, y, d,
                                                                   h].value)))

                            f.write('%s,%s,%s\n' %
                                    (h, d + '_shutdown',
                                     round(instance.elecshutdowngen[r, y, d,
                                                                    h].value)))

                            f.write(
                                '%s,%s,%s\n' %
                                (h, d + '_aFCRUp',
                                 round(instance.aFCRupGen[r, y, d, h].value)))

                            f.write('%s,%s,%s\n' %
                                    (h, d + '_aFCRDown',
                                     round(instance.aFCRdownGen[r, y, d,
                                                                h].value)))

                            f.write(
                                '%s,%s,%s\n' %
                                (h, d + '_aFRRup',
                                 round(instance.aFRRupGen[r, y, d, h].value)))

                            f.write('%s,%s,%s\n' %
                                    (h, d + '_aFRRdown',
                                     round(instance.aFRRdownGen[r, y, d,
                                                                h].value)))

                            f.write(
                                '%s,%s,%s\n' %
                                (h, d + '_mFRRup',
                                 round(instance.mFRRupGen[r, y, d, h].value)))

                            f.write('%s,%s,%s\n' %
                                    (h, d + '_mFRRdown',
                                     round(instance.mFRRdownGen[r, y, d,
                                                                h].value)))

                        if [r, y, d] in instance.REGION_YEAR_TECH_UC_Stor:
                            f.write(
                                '%s,%s,%s\n' %
                                (h, d + '_char_committed',
                                 round(instance.eleccommitstorchar[r, y, d,
                                                                   h].value)))

                            f.write(
                                '%s,%s,%s\n' %
                                (h, d + '_char_startup',
                                 round(instance.elecstartupstorchar[r, y, d,
                                                                    h].value)))

                            f.write(
                                '%s,%s,%s\n' %
                                (h, d + '_char_shutdown',
                                 round(
                                     instance.elecshutdownstorchar[r, y, d,
                                                                   h].value)))

                            f.write('%s,%s,%s\n' %
                                    (h, d + '_aFCRUpChar',
                                     round(instance.aFCRupStorChar[r, y, d,
                                                                   h].value)))

                            f.write(
                                '%s,%s,%s\n' %
                                (h, d + '_aFCRDownChar',
                                 round(instance.aFCRdownStorChar[r, y, d,
                                                                 h].value)))

                            f.write('%s,%s,%s\n' %
                                    (h, d + '_aFRRupChar',
                                     round(instance.aFRRupStorChar[r, y, d,
                                                                   h].value)))

                            f.write(
                                '%s,%s,%s\n' %
                                (h, d + '_aFRRdownChar',
                                 round(instance.aFRRdownStorChar[r, y, d,
                                                                 h].value)))

                            f.write('%s,%s,%s\n' %
                                    (h, d + '_mFRRupChar',
                                     round(instance.mFRRupStorChar[r, y, d,
                                                                   h].value)))

                            f.write(
                                '%s,%s,%s\n' %
                                (h, d + '_mFRRdownChar',
                                 round(instance.mFRRdownStorChar[r, y, d,
                                                                 h].value)))

                            f.write(
                                '%s,%s,%s\n' %
                                (h, d + '_disc_committed',
                                 round(instance.eleccommitstordisc[r, y, d,
                                                                   h].value)))

                            f.write(
                                '%s,%s,%s\n' %
                                (h, d + '_discstartup',
                                 round(instance.elecstartupstordisc[r, y, d,
                                                                    h].value)))

                            f.write(
                                '%s,%s,%s\n' %
                                (h, d + '_disc_shutdown',
                                 round(
                                     instance.elecshutdownstordisc[r, y, d,
                                                                   h].value)))

                            f.write('%s,%s,%s\n' %
                                    (h, d + '_aFCRUpDisc',
                                     round(instance.aFCRupStorDisc[r, y, d,
                                                                   h].value)))

                            f.write(
                                '%s,%s,%s\n' %
                                (h, d + '_aFCRDownDisc',
                                 round(instance.aFCRdownStorDisc[r, y, d,
                                                                 h].value)))

                            f.write('%s,%s,%s\n' %
                                    (h, d + '_aFRRupDisc',
                                     round(instance.aFRRupStorDisc[r, y, d,
                                                                   h].value)))

                            f.write(
                                '%s,%s,%s\n' %
                                (h, d + '_aFRRdownDisc',
                                 round(instance.aFRRdownStorDisc[r, y, d,
                                                                 h].value)))

                            f.write('%s,%s,%s\n' %
                                    (h, d + '_mFRRupDisc',
                                     round(instance.mFRRupStorDisc[r, y, d,
                                                                   h].value)))

                            f.write(
                                '%s,%s,%s\n' %
                                (h, d + '_mFRRdownDisc',
                                 round(instance.mFRRdownStorDisc[r, y, d,
                                                                 h].value)))

                    for tt in instance.TECH_STORAGE:
                        if [r, y, tt] in instance.REGION_YEAR_TECH_STORAGE:
                            if tt in instance.TECH_DSM:
                                f.write(
                                    '%s,%s,%s\n' %
                                    (h, tt + '_delayed',
                                     round(instance.electechchar[r, y, tt,
                                                                 h].value)))
                                f.write(
                                    '%s,%s,%s\n' %
                                    (h, tt + '_served',
                                     round(-instance.electechdisc[r, y, tt,
                                                                  h].value)))
                                f.write(
                                    '%s,%s,%s\n' %
                                    (h, tt + '_delayed_stored',
                                     round(instance.electechstored[r, y, tt,
                                                                   h].value)))
                            else:
                                f.write(
                                    '%s,%s,%s\n' %
                                    (h, tt + '_char',
                                     round(-instance.electechchar[r, y, tt,
                                                                  h].value)))
                                f.write(
                                    '%s,%s,%s\n' %
                                    (h, tt + '_disc',
                                     round(instance.electechdisc[r, y, tt,
                                                                 h].value)))
                                f.write(
                                    '%s,%s,%s\n' %
                                    (h, tt + '_stored',
                                     round(instance.electechstored[r, y, tt,
                                                                   h].value)))

                    f.write('%s,%s,%s\n' %
                            (h, 'demand', round(instance.Demand[r, y, h])))
                    f.write(
                        '%s,%s,%s\n' %
                        (h, 'shadow_price', instance.dual[cobject[r, y, h]]))

                    f.write('%s,%s,%s\n' %
                            (h, 'exchange',
                             round(instance.elecexchange[r, y, h].value)))
                    f.write('%s,%s,%s\n' %
                            (h, 'excess',
                             round(-instance.elecexcess[r, y, h].value)))
                    f.write('%s,%s,%s\n' %
                            (h, 'load_shedding',
                             round(instance.elecloadshedding[r, y, h].value)))
                    f.write(
                        '%s,%s,%s\n' %
                        (h, 'FCR_required', round(instance.FCRRequired[r, y])))
                    f.write('%s,%s,%s\n' %
                            (h, 'aFRRup_required',
                             round(instance.aFRRuprequired[r, y, h].value)))
                    f.write('%s,%s,%s\n' %
                            (h, 'aFRRdown_required',
                             round(instance.aFRRdownrequired[r, y, h].value)))
                    f.write('%s,%s,%s\n' %
                            (h, 'mFRRup_required',
                             round(instance.mFRRuprequired[r, y, h].value)))
                    f.write('%s,%s,%s\n' %
                            (h, 'mFRRdown_required',
                             round(instance.mFRRdownrequired[r, y, h].value)))

                f.close()

                open(save_path1, 'w')
                pd.read_csv(save_path).pivot(index='hour',
                                             columns='technology',
                                             values='value').to_csv(save_path1)
                os.remove(save_path)

    for y in instance.YEAR:
        save_path = dirName + '/' + str(y) + '_' + 'transmission1_MW.csv'
        save_path1 = dirName + '/' + str(y) + '_' + 'transmission_MW.csv'
        with open(save_path, 'w') as f:
            f.write('hour,line,value\n')
            for h in instance.TIMESLICE:
                for l in instance.LINE:
                    f.write(
                        '%s,%s,%s\n' %
                        (h, l, round(instance.eleclexchange[l, y, h].value)))
            f.close()

            open(save_path1, 'w')
            pd.read_csv(save_path).pivot(index='hour',
                                         columns='line',
                                         values='value').to_csv(save_path1)
            os.remove(save_path)

    save_path = dirName + '/' + 'HydroDamWeekIniLevel.csv'
    with open(save_path, 'w') as f:
        f.write('region,year,week,tech,storage-level,weekprod(MWh)\n')
        for w in instance.WEEK:
            for r, y, t in instance.REGION_YEAR_TECH_HD:
                f.write(
                    '%s,%s,%s,%s,%s,%s\n' %
                    (r, y, w, t,
                     round(instance.elechdweekinilevel[r, y, t, w].value, 3),
                     round(instance.elechdweekprod[r, y, t, w].value)))

    return

def whole_result_append(instance, r, y, scenario):
    cobject = getattr(instance, 'ElecBalanceConstraint')
    sce.append(scenario)
    annualcost.append(instance.regionalyearcost[r, y].value +
                      sum(ExchangeHourlyCostFunction(instance, r, y)))
    capex.append(instance.regionalyearcostinv[r, y].value)
    inv_gen.append(instance.regionalyearcostinvgen[r, y].value)
    inv_stor.append(instance.regionalyearcostinvstor[r, y].value)
    inv_grid.append(instance.regionalyearcostinvgrid[r, y].value)
    opexfix.append(instance.regionalyearcostfix[r, y].value)
    fix_gen.append(instance.regionalyearcostfixgen[r, y].value)
    fix_stor.append(instance.regionalyearcostfixstor[r, y].value)
    fix_grid.append(instance.regionalyearcostfixgrid[r, y].value)
    opexvar.append(instance.regionalyearcostvar[r, y].value)
    var_gen.append(instance.regionalyearcostvargen[r, y].value)
    var_stor.append(instance.regionalyearcostvarstor[r, y].value)
    var_startup.append(instance.regionalyearcostvarstartup[r, y].value)
    var_co2.append(instance.regionalyearcostco2[r, y].value)
    var_exchange.append(sum(ExchangeHourlyCostFunction(instance, r, y)))
    nuc.append(instance.totalinstalledgencapacity[r, y, 'Nuclear_new'].value)
    CCGT_new.append(instance.totalinstalledgencapacity[r, y, 'CCGT_new'].value)
    OCGT_new.append(instance.totalinstalledgencapacity[r, y, 'OCGT_new'].value)
    pv_large.append(instance.totalinstalledgencapacity[r, y, 'PV_large'].value)
    pv_small.append(instance.totalinstalledgencapacity[r, y, 'PV_small'].value)
    onshore.append(instance.totalinstalledgencapacity[r, y,
                                                      'Wind_onshore'].value)
    offshore.append(instance.totalinstalledgencapacity[r, y,
                                                       'Wind_offshore'].value)
    ires.append(instance.irescapacity[r, y].value)
    hydro_ror.append(instance.totalinstalledgencapacity[r, y,
                                                        'Hydro_ROR'].value)
    hydro_dam.append(instance.totalinstalledgencapacity[r, y,
                                                        'Hydro_Dam'].value)
    phs_power.append(instance.totalinstalledstorpower[r, y, 'PHS20'].value)
    phs_energy.append(instance.totalinstalledstorenergy[r, y, 'PHS20'].value)
    nucpro.append(instance.annualelectechgen[r, y, 'Nuclear_new'].value)
    pvlargepro.append(instance.annualelectechgen[r, y, 'PV_large'].value)
    pvsmallpro.append(instance.annualelectechgen[r, y, 'PV_small'].value)
    onshorepro.append(instance.annualelectechgen[r, y, 'Wind_onshore'].value)
    offshorepro.append(instance.annualelectechgen[r, y, 'Wind_offshore'].value)
    gasccgtpro.append(instance.annualelectechgen[r, y, 'CCGT_new'].value)
    gasocgtpro.append(instance.annualelectechgen[r, y, 'OCGT_new'].value)
    hydro_rorpro.append(instance.annualelectechgen[r, y, 'Hydro_ROR'].value)
    hydro_dampro.append(instance.annualelectechgen[r, y, 'Hydro_Dam'].value)
    phs_char.append(-instance.annualelectechchar[r, y, 'PHS20'].value)
    phs_disc.append(instance.annualelectechdisc[r, y, 'PHS20'].value)
    exchange.append(instance.annualelecexchange[r, y].value)
    excess.append(-instance.annualelecexcess[r, y].value)
    demand.append(instance.annualdemand[r, y].value)
    nucper.append(
        round(
            instance.annualelecnucgen[r, y].value /
            instance.annualelecgen[r, y].value, 3))
    resper.append(
        round(
            1 - instance.annualelecnucgen[r, y].value /
            instance.annualelecgen[r, y].value -
            instance.annualelecfossilgen[r, y].value /
            instance.annualelecgen[r, y].value, 3))
    fossilper.append(
        round(
            instance.annualelecfossilgen[r, y].value /
            instance.annualelecgen[r, y].value, 3))
    emission.append(instance.annualemission[r, y].value)
    totalemission.append(
        sum(instance.annualemission[rr, y].value for rr in instance.REGION_ID))
    shadowprice.append(
        sum(instance.dual[cobject[r, y, h]]
            for h in instance.TIMESLICE) / 8760)

    importtranscapacity.append(0)
    exporttranscapacity.append(0)

    battery1_power.append(0)
    battery1_energy.append(0)
    battery4_power.append(0)
    battery4_energy.append(0)
    battery8_power.append(0)
    battery8_energy.append(0)
    battery1_char.append(0)
    battery1_disc.append(0)
    battery4_char.append(0)
    battery4_disc.append(0)
    battery8_char.append(0)
    battery8_disc.append(0)

    ptg_power.append(0)
    ptg_char.append(0)
    ptg_disc.append(0)

    return

def whole_result_save(sce_name):
    df = pd.DataFrame(
        data={
            "sce": sce,
            "annualcost(bn)": [round(x / 1e9, 2) for x in annualcost],
            "capex": [round(x / 1e9, 2) for x in capex],
            "inv_gen": [round(x / 1e9, 2) for x in inv_gen],
            "inv_stor": [round(x / 1e9, 2) for x in inv_stor],
            "inv_grid": [round(x / 1e9, 2) for x in inv_grid],
            "opexfix": [round(x / 1e9, 2) for x in opexfix],
            "fix_gen": [round(x / 1e9, 2) for x in fix_gen],
            "fix_stor": [round(x / 1e9, 2) for x in fix_stor],
            "fix_grid": [round(x / 1e9, 2) for x in fix_grid],
            "opexvar": [round(x / 1e9, 2) for x in opexvar],
            "var_gen": [round(x / 1e9, 2) for x in var_gen],
            "var_stor": [round(x / 1e9, 2) for x in var_stor],
            "var_startup": [round(x / 1e9, 2) for x in var_startup],
            "var_co2": [round(x / 1e9, 2) for x in var_co2],
            "var_exchange": [round(x / 1e9, 2) for x in var_exchange],
            "nuc(GW)": [round(x / 1000, 1) for x in nuc],
            "CCGT_new(GW)": [round(x / 1000, 1) for x in CCGT_new],
            "OCGT_new(GW)": [round(x / 1000, 1) for x in OCGT_new],
            "pv_large(GW)": [round(x / 1000, 1) for x in pv_large],
            "pv_small(GW)": [round(x / 1000, 1) for x in pv_small],
            "onshore(GW)": [round(x / 1000, 1) for x in onshore],
            "offshore(GW)": [round(x / 1000, 1) for x in offshore],
            "ires(GW)": [round(x / 1000, 1) for x in ires],
            "hydro_ror(GW)": [round(x / 1000, 1) for x in hydro_ror],
            "hydro_dam(GW)": [round(x / 1000, 1) for x in hydro_dam],
            "phs_power(GW)": [round(x / 1000, 1) for x in phs_power],
            "phs_energy(GWh)": [round(x / 1000, 1) for x in phs_energy],
            "ptg_power(GW)": [round(x / 1000, 1) for x in ptg_power],
            "battery1_power(GW)": [round(x / 1000, 1) for x in battery1_power],
            "battery1_energy(GWh)":
            [round(x / 1000, 1) for x in battery1_energy],
            "battery4_power(GW)": [round(x / 1000, 1) for x in battery4_power],
            "battery4_energy(GWh)":
            [round(x / 1000, 1) for x in battery4_energy],
            "battery8_power(GW)": [round(x / 1000, 1) for x in battery8_power],
            "battery8_energy(GWh)":
            [round(x / 1000, 1) for x in battery8_energy],
            "import(GW)": [round(x / 1000, 1) for x in importtranscapacity],
            "export(GW)": [round(x / 1000, 1) for x in exporttranscapacity],
            "nucpro(TWh)": [round(x / 1e6, 1) for x in nucpro],
            "pvlargepro(TWh)": [round(x / 1e6, 1) for x in pvlargepro],
            "pvsmallpro(TWh)": [round(x / 1e6, 1) for x in pvsmallpro],
            "onshorepro(TWh)": [round(x / 1e6, 1) for x in onshorepro],
            "offshorepro(TWh)": [round(x / 1e6, 1) for x in offshorepro],
            "gasccgtpro(TWh)": [round(x / 1e6, 1) for x in gasccgtpro],
            "gasocgtpro(TWh)": [round(x / 1e6, 1) for x in gasocgtpro],
            "hydro_rorpro(TWh)": [round(x / 1e6, 1) for x in hydro_rorpro],
            "hydro_dampro(TWh)": [round(x / 1e6, 1) for x in hydro_dampro],
            "phs_char(TWh)": [round(x / 1e6, 1) for x in phs_char],
            "phs_disc(TWh)": [round(x / 1e6, 1) for x in phs_disc],
            "ptg_char(TWh)": [round(x / 1e6, 1) for x in ptg_char],
            "ptg_disc(TWh)": [round(x / 1e6, 1) for x in ptg_disc],
            "battery1_char(TWh)": [round(x / 1e6, 1) for x in battery1_char],
            "battery1_disc(TWh)": [round(x / 1e6, 1) for x in battery1_disc],
            "battery4_char(TWh)": [round(x / 1e6, 1) for x in battery4_char],
            "battery4_disc(TWh)": [round(x / 1e6, 1) for x in battery4_disc],
            "battery8_char(TWh)": [round(x / 1e6, 1) for x in battery8_char],
            "battery8_disc(TWh)": [round(x / 1e6, 1) for x in battery8_disc],
            "excess&loss(TWh)": [round(x / 1e6, 1) for x in excess],
            "exchange(TWh)": [round(x / 1e6, 1) for x in exchange],
            "demand(TWh)": [round(x / 1e6) for x in demand],
            "nucper": [round(x, 3) for x in nucper],
            "resper": [round(x, 3) for x in resper],
            "fossilper": [round(x, 3) for x in fossilper],
            "emission(MT)": [round(x / 1e9, 2) for x in emission],
            "totalemission(MT)": [round(x / 1e9, 2) for x in totalemission],
            "averageshadowprice(euro/MWh)": shadowprice
        })

    dirName = str(os.getcwd()).replace('\\', '/') + "/output_data/" + sce_name

    # Create target directory if it doesn't exist
    if not os.path.exists(dirName):
        os.mkdir(dirName)

    save_path = dirName + '/result_' + sce_name + '.csv'
    df.to_csv(save_path, sep=',', index=False)

    return

def ExchangeHourlyCostFunction(instance, r, y):
    cobject = getattr(instance, 'ElecBalanceConstraint')

    exchangehourlycostlist = []

    for h in instance.TIMESLICE:
        exchangehourlycost = 0
        for l in instance.LINE:
            if (instance.eleclexchange[l, y, h].value *
                    instance.LineRegion[l, r]) > 0:  # import
                exchangehourlycost = exchangehourlycost + instance.eleclexchange[
                    l, y, h].value * instance.LineRegion[l, r] * instance.dual[
                        cobject[instance.ContraryRegion[l, r], y, h]] / (
                            1 - instance.TransLosses[y, instance.TransTech[l]]
                            * instance.TransDis[l] / 100)

            else:  #export
                exchangehourlycost = exchangehourlycost + instance.eleclexchange[
                    l, y, h].value * instance.LineRegion[l, r] * instance.dual[
                        cobject[r, y, h]] / (
                            1 - instance.TransLosses[y, instance.TransTech[l]]
                            * instance.TransDis[l] / 100)
        exchangehourlycostlist.append(exchangehourlycost)

    return exchangehourlycostlist

if __name__ == "__main__":
    pass

max_stars_repo_name: YilingCAI/ESMO

max_issues_repo_name: YilingCAI/ESMO

max_forks_repo_name: YilingCAI/ESMO

reason: from pyomo


text:
# coding: utf-8

# In[1]:

#Download the libraries
import sys
import nltk
import re
import pandas as pd
import numpy as np
import csv
import os
import random
import numpy as np
from nltk.sentiment.vader import SentimentIntensityAnalyzer
from collections import Counter
from textblob import TextBlob
from textblob import Blobber
from textblob.sentiments import NaiveBayesAnalyzer

# Setting working directory

# In[2]:

#print os.getcwd();

# Read the file

# In[3]:

#os.chdir("/Users/imacair/Desktop/Products3/")

# In[4]:

#Converts text into ASCII

# In[5]:

data_t = sys.argv[1]
data_t = data_t.decode('ascii', 'replace')

# In[6]:

#lowercase
data_t = data_t.lower()

# Abbriviation translation

# In[7]:

with open('./ML/Python/abbrev.csv', mode='r') as infile:
reader = csv.reader(infile)
replacement = {rows[0].lower(): rows[1].lower() for rows in reader}

# In[19]:

result = pd.DataFrame()
result = data_t

# In[20]:

for i in range(len(result)):
data_t = ' '.join([replacement.get(w, w) for w in data_t.split()])

# In[10]:

from nltk.corpus import stopwords
s = set(stopwords.words('english'))
for i in range(len(result)):
data_t = ' '.join(filter(lambda w: not w in s, data_t.split()))

# In[11]:

#lowercase
data_t = data_t.lower()
#Remove urls
data_t = data_t.replace(
r'(http.*) |(http.*)$|\n',
"", )
#Remove twitter handles
data_t = data_t.replace(r"@\\w+", "")
#remove htmls
data_t = data_t.replace(r'<.*?>', "")
#Remove citations
data_t = data_t.replace(r'@[a-zA-Z0-9]*', "")
#remove _
#data_t = data_t.str.replace(r'\_+',"")

# Spliting the data
#

# In[12]:

#Use vader package to get the sentiment
analyzer = SentimentIntensityAnalyzer()
res = pd.DataFrame(index=range(0, 1), columns={'SentimentVader'})

#Convert sentiment to neu, neg, pos
for i in range(1):
vs = analyzer.polarity_scores(data_t)
if ((vs['pos'] > 0)):
res.values[i] = 'pos'
elif ((vs['neg'] < 0)):
res.values[i] = 'neg'
else:
res.values[i] = 'neu'
vader = res.SentimentVader

# In[13]:

#Use textblob to get polarity of text
res6 = data_t
testimonial = TextBlob(data_t)
res6 = testimonial.sentiment.polarity
#Convert polarity to normal pos, neg, neu
textblob1 = res6
if ((res6 > 0)):
textblob1 = 'pos'
elif ((res6 < 0)):
textblob1 = 'neg'
else:
textblob1 = 'neu'

# In[14]:

#Use textblob to get polarity of text with Naive Bayes analyzer
tb = Blobber(analyzer=NaiveBayesAnalyzer())
textblob2 = pd.DataFrame(index=range(0, 1), columns={'sentimentNB'})
textblob2['sentimentNB'] = tb(data_t).sentiment.classification

# In[15]:

vader = np.asarray(vader)

#Create a Data Frame
df = [vader]
df = pd.DataFrame(df)
df = df.transpose()
df.columns = ['vader']
df["SentimentPat"] = textblob1
df["sentimentNB"] = textblob2
#df

# In[16]:

#Find the maximum in a row (Majority voting)
df2 = pd.DataFrame(index=range(0, 1), columns={''})
for i in range(1):
d = Counter(df.ix[i, :])
dfin = d.most_common(1)[0][0]
df2.values[i] = dfin
#df["final"] = df2
data = df2.values[0]
data = ''.join(data)

# In[17]:

print(data)

# In[ ]:

# In[ ]:

ext: py

reason: import numpy


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Oct 18 00:38:48 2019

@author: matias

Comparo los resultados de la integracion en Python con los de Octave
"""

import numpy as np
from matplotlib import pyplot as plt
import pandas as pd
from scipy.interpolate import interp1d

import sys
import os
from os.path import join as osjoin

import time

from scipy.integrate import solve_ivp
from scipy.integrate import simps as simps
from scipy.integrate import cumtrapz as cumtrapz

from pc_path import definir_path
path_git, path_datos_global = definir_path()

os.chdir(path_git)
sys.path.append('./Software/Funcionales')
from funciones_int import plot_sol
from funciones_cambio_parametros import params_fisicos_to_modelo

# Importo los datos de Octave H(z) y d_L
df = pd.read_csv(
        '/home/matias/Documents/Tesis/tesis_licenciatura/Software/Integracion numérica/Python/analisis_int_solve_ivp/datos_octave.txt'
                 , header = None, delim_whitespace=True)

def dX_dz(z, variables,*params_modelo):
    '''Defino el sistema de ecuaciones a resolver. El argumento params_modelo
    es una lista donde los primeros n-1 elementos son los parametros del sistema,
    mientras que el útimo argumento especifica el modelo en cuestión,
    matematicamente dado por la función gamma.'''

    x = variables[0]
    y = variables[1]
    v = variables[2]
    w = variables[3]
    r = variables[4]

    [B,D,N] = params_modelo
    gamma = lambda r,b,d: ((1+d*r) * ((1+d*r)**2 - b)) / (2*b*d*r)
    G = gamma(r,B,D)

    s0 = (-w + x**2 + (1+v)*x - 2*v + 4*y) / (z+1)
    s1 = - (v*x*G - x*y + 4*y - 2*y*v) / (z+1)
    s2 = -v * (x*G + 4 - 2*v) / (z+1)
    s3 = w * (-1 + x + 2*v) / (z+1)
    s4 = -(x*r*G)/(1+z)
    return [s0,s1,s2,s3,s4]

sistema_ec=dX_dz
z_inicial=0
z_final=3
cantidad_zs = 20000
max_step=0.01
max_step=np.inf
verbose=True

x_0 = -0.339
y_0 = 1.246
v_0 = 1.64
w_0 = 1 + x_0 + y_0 - v_0
r_0 = 41
ci = [x_0, y_0, v_0, w_0, r_0] #Condiciones iniciales
cond_iniciales=ci

#c1_true = 1
#c2_true = 1/19
#n=1
params_modelo=[1,1/19,1]
H0=73.48

#%% Forma nueva de integrar
zs = np.linspace(z_inicial,z_final,cantidad_zs)

sol_1 = solve_ivp(sistema_ec, [z_inicial,z_final],
      cond_iniciales,t_eval=zs,args=params_modelo, max_step=max_step)
int_v_1 =  cumtrapz(sol_1.y[2]/(1+sol_1.t),sol_1.t,initial=0)
E1 = (1+sol_1.t)**2 * np.exp(-int_v_1)
H_1 = H0 * E1

if (len(zs)!=len(sol_1.t)):
    print('Está integrando mal!')

#%% Comparo los errores porcentuales para Python y Octave
f = interp1d(np.array(df[0]),np.array(df[1]))
restas = (1-(np.array(f(zs))/E1)) *100
plt.close()
plt.figure()
plt.plot(zs, restas,label=r'$\Gamma=\Gamma(C_{1},C_{2})$')
plt.xlabel('z (redshift)')
plt.ylabel(r'$\frac{H_{python}-H_{octave}}{H_{python}}\times $%$100$', size =20)
plt.legend(loc='best')
plt.grid(True)
#%% Comparo los E(z)
plt.figure()
plt.plot(zs,E1,label='Python')
plt.plot(zs,f(zs),'-.',label='Octave')
plt.xlabel('z (redshift)')
plt.ylabel('E(z)')
plt.legend(loc='best')
plt.grid(True)

#plt.plot(sol_1.t,sol_1.y[2])

#%% 17/07: Lo que sigue es viejo. Comparo los d_L(z)
g = interp1d(np.array(df[0]), np.array(df[2]))
restas_dl = (d_L-np.array(g(zs_dl)))/d_L
plt.close()
plt.figure()

plt.plot(zs_dl, restas_dl,label=r'$\Gamma=\Gamma(y,v)$')
plt.xlabel('z(redshift)')
plt.ylabel(r'$\frac{d_{python}-d_{octave}}{d_{python}}$, size =20)
plt.legend(loc='best')
plt.grid(True)

#%% No da tan bien!
plt.figure()
plt.plot(zs_dl,d_L)
plt.plot(zs_dl,g(zs_dl))

reason: import numpy,from scipy


text:
import os
import cv2
import torch
import random
import numpy as np
import pandas as pd
from PIL import Image
from skimage import transform, io
from torchvision import datasets, transforms
from torch.utils.data import DataLoader
from torch.utils.data.sampler import WeightedRandomSampler

class Dataset():
	def __init__(self, train_dir, val_dir, basic_types = None):
		self.train_dir = train_dir
		self.val_dir = val_dir
		self.basic_types = basic_types

	def get_loader(self, sz, bs, get_size = False, get_class_names = False, get_each_class_size = False, data_transforms = None):
		if(self.basic_types == None):
			if(data_transforms is None):
				data_transforms = {
					'train' : transforms.Compose([
						transforms.RandomResizedCrop(sz),
						transforms.RandomHorizontalFlip(),
						transforms.ToTensor(),
						transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
					]),
					'val' : transforms.Compose([
						transforms.Resize(int(sz*1.2)),
						transforms.CenterCrop(sz),
						transforms.ToTensor(),
						transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
					])
				}

			train_dataset = datasets.ImageFolder(self.train_dir, data_transforms['train'])
			val_dataset = datasets.ImageFolder(self.val_dir, data_transforms['val'])
			class_names = train_dataset.classes

			train_classes_count = []
			for cur_dir in class_names:
				count = len([file for file in os.listdir(os.path.join(self.train_dir, cur_dir)) if file[0] != '.'])
				train_classes_count.append(count)

			val_classes_count = []
			for cur_dir in class_names:
				count = len([file for file in os.listdir(os.path.join(self.val_dir, cur_dir)) if file[0] != '.'])
				val_classes_count.append(count)

			train_loader = DataLoader(train_dataset, batch_size = bs, shuffle = True)
			val_loader = DataLoader(val_dataset, batch_size = bs, shuffle = False)

			train_dataset_size = len(train_dataset)
			val_dataset_size = len(val_dataset)
			sizes = {
				'train_dset_size' : train_dataset_size,
				'val_dset_size' : val_dataset_size
			}
			each_class_size = {
				'train_classes_count' : train_classes_count,
				'val_classes_count' : val_classes_count
			}

			returns = (train_loader, val_loader)
			if(get_size):
				returns = returns + (sizes,)
			if(get_class_names):
				returns = returns + (class_names,)
			if(get_each_class_size):
				returns = returns + (each_class_size,)

		if(self.basic_types == 'MNIST'):
			if(data_transforms is None):
				data_transforms = {
					'train' : transforms.Compose([
						transforms.Resize(sz),
						transforms.ToTensor(),
						transforms.Normalize([0.5], [0.5])
					]),
					'val' : transforms.Compose([
						transforms.Resize(sz),
						transforms.ToTensor(),
						transforms.Normalize([0.5], [0.5])
					])
				}

			train_dataset = datasets.MNIST(self.train_dir, train = True, download = True, transform = data_transforms['train'])
			val_dataset = datasets.MNIST(self.val_dir, train = False, download = True, transform = data_transforms['val'])

			train_loader = DataLoader(train_dataset, batch_size = bs, shuffle = True)
			val_loader = DataLoader(val_dataset, batch_size = bs, shuffle = False)

			train_dataset_size = len(train_dataset)
			val_dataset_size = len(val_dataset)
			sizes = {
				'train_dset_size' : train_dataset_size,
				'val_dset_size' : val_dataset_size
			}

			returns = (train_loader, val_loader)
			if(get_size):
				returns = returns + (sizes,)

		elif(self.basic_types == 'CIFAR'):
			if(data_transforms is None):
				data_transforms = {
					'train' : transforms.Compose([
						transforms.Resize(sz),
						transforms.ToTensor(),
						transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])
					]),
					'val' : transforms.Compose([
						transforms.Resize(sz),
						transforms.ToTensor(),
						transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])
					])
				}

			train_dataset = datasets.CIFAR10(self.train_dir, train = True, download = True, transform = data_transforms['train'])
			val_dataset = datasets.CIFAR10(self.val_dir, train = False, download = True, transform = data_transforms['val'])

			train_loader = DataLoader(train_dataset, batch_size = bs, shuffle = True)
			val_loader = DataLoader(val_dataset, batch_size = bs, shuffle = False)

			train_dataset_size = len(train_dataset)
			val_dataset_size = len(val_dataset)
			sizes = {
				'train_dset_size' : train_dataset_size,
				'val_dset_size' : val_dataset_size
			}

			returns = (train_loader, val_loader)
			if(get_size):
				returns = returns + (sizes,)

		#elif(self.basic_types == 'Segmentation'):

		return returns

class Segmentation_Dataset():
	def __init__(self, input_dir, target_dir, input_transform, target_transform, RGB_list):
		self.input_dir = input_dir
		self.target_dir = target_dir
		self.input_transform = input_transform
		self.target_transform = target_transform
		self.RGB_list = RGB_list

		self.image_name_list = []
		for file in os.listdir(input_dir):
			if(file.endswith('.png') or file.endswith('.jpeg') or file.endswith('.jpg') or file.endswith('.bmp')):
				self.image_name_list.append(file)

		def __len__(self):
			return len(self.image_name_list)

		def __getitem__(self, idx):
			input_img = Image.open(os.path.join(self.input_dir, self.image_name_list[idx]))
			target_img = Image.open(os.path.join(self.target_dir, self.image_name_list[idx]))
			target_img = np.array(target_img)
			for i, (name, color) in enumerate(self.RGB_list):
				pos = np.where((target_img[:, :, 0] == color[2]) & (target_img[:, :, 1] == color[1]) & (target_img[:, :, 2] == color[0]))
				target_img[pos] = i
			target_img = target_img[:, :, 0]

			input_img = self.input_transform(input_img)
			target_img = self.target_transform(target_img)

			sample = (input_img, target_img)
			return sample

'''
RGB_list = [
	{'Animal', (64, 128, 64)},
	{'Archway', (192, 0, 128)},
	{'Bicyclist', (0, 128, 192)},
	{'Bridge', (0, 128, 64)},
	{'Building', (128, 0, 0)},
	{'Car', (64, 0, 128)},
	{'CartLuggagePram', (64, 0, 192)},
	{'Child', (192, 128, 64)},
	{'Column_Pole', (192, 192, 128)},
	{'Fence', (64, 64, 128)},
	{'LaneMkgsDriv', (128, 0, 192)},
	{'LaneMkgsNonDriv', (192, 0, 64)},
	{'Misc_Text', (128, 128, 64)},
	{'MotorcycleScooter', (192, 0, 192)},
	{'OtherMoving', (128, 64, 64)},
	{'ParkingBlock', (64, 192, 128)},
	{'Pedestrian', (64, 64, 0)},
	{'Road', (128, 64, 128)},
	{'RoadShoulder', (128, 128, 192)},
	{'Sidewalk', (0, 0, 192)},
	{'SignSymbol', (192, 128, 128)},
	{'Sky', (128, 128, 128)},
	{'SUVPickupTruck', (64, 128, 192)},
	{'TrafficCone', (0, 0, 64)},
	{'TrafficLight', (0, 64, 64)},
	{'Train', (192, 64, 128)},
	{'Tree', (128, 128, 0)},
	{'Truck_Bus', (192, 128, 192)},
	{'Tunnel', (64, 0, 64)},
	{'VegetationMisc', (192, 192, 0)},
	{'Void', (0, 0, 0)},
	{'Wall', (64, 192, 0)}
]
'''

max_stars_repo_name: ericlearning/supervised-learning

max_issues_repo_name: ericlearning/supervised-learning

max_forks_repo_name: ericlearning/supervised-learning


text:
import os
import numpy as np
from scipy import io as sio
import SimpleITK as sitk

def loadimg(file, target_resolution, ):
    if file.endswith('.mat'):
        filecont = sio.loadmat(file)
        img = filecont['img']
        for z in range(img.shape[-1]):  # Flip the image upside down
            img[:, :, z] = np.flipud(img[:, :, z])
        img = np.swapaxes(img, 0, 1)
    elif file.endswith('.tif'):
        img = loadtiff3d(file)
        img = np.swapaxes(img, 0, 2)
        img = np.flip(img, axis=1)
    elif file.endswith('.mhd'):
        from scipy.ndimage.interpolation import zoom
        mhd = sitk.ReadImage(file)
        img = sitk.GetArrayFromImage(mhd)
        # Resample the image to isotropic resolution
        print('Resample Image to isotropic resolution 1mmx1mmx1mm')
        sx, sy, sz = mhd.GetSpacing()
        img = zoom(img, (sz / target_resolution,
                         sy / target_resolution,
                         sx / target_resolution), order=0)
        img = np.transpose(img, (2, 1, 0))
    elif file.endswith('.nii') or file.endswith('.nii.gz'):
        import nibabel as nib
        img = nib.load(file)
        img = img.get_data()
    else:
        raise IOError("The extension of " + file +
                      'is not supported. File extension supported are: *.tif, *.mat, *.nii')

    # img = reduce_to_8_bit(img, strict=True, report=True)

    return img

def loadtiff3d(filepath, out='Numpy'):
    """Load a tiff file into 3D numpy array"""
    # im = sitk.ReadImage(filepath, sitk.sitkUInt16, imageIO='TIFFImageIO')
    im = sitk.ReadImage(filepath, imageIO='TIFFImageIO')
    if out == 'Numpy':
        im = sitk.GetArrayFromImage(im)
    elif out == 'SITK':
        pass
    else:
        raise ValueError(f'Invalid output format {out}, please use either "Numpy" or "SITK"')
    return im

def writetiff3d(filepath, block):
    from libtiff import TIFF
    try:
        os.remove(filepath)
    except OSError:
        pass

    tiff = TIFF.open(filepath, mode='w')
    block = np.swapaxes(block, 0, 1)

    for z in range(block.shape[2]):
        tiff.write_image(np.flipud(block[:, :, z]), compression=None)
    tiff.close()

def loadswc(filepath):
    '''
    Load swc file as a N X 7 numpy array
    '''
    swc = []
    with open(filepath) as f:
        lines = f.read().split("\n")
        for l in lines:
            if not l.startswith('#'):
                cells = l.split(' ')
                if len(cells) == 7:
                    cells = [float(c) for c in cells]
                    # cells[2:5] = [c-1 for c in cells[2:5]]
                    swc.append(cells)
    return np.array(swc)

def saveswc(filepath, swc):
    if swc.shape[1] > 7:
        swc = swc[:, :7]

    with open(filepath, 'w') as f:
        for i in range(swc.shape[0]):
            print('%d %d %.3f %.3f %.3f %.3f %d' %
                  tuple(swc[i, :].tolist()), file=f)

def crop(img, thr):
    """Crop a 3D block with value > thr"""
    ind = np.argwhere(img > thr)
    x = ind[:, 0]
    y = ind[:, 1]
    z = ind[:, 2]
    xmin = max(x.min() - 10, 0)
    xmax = min(x.max() + 10, img.shape[0])
    ymin = max(y.min() - 10, 1)
    ymax = min(y.max() + 10, img.shape[1])
    zmin = max(z.min() - 10, 2)
    zmax = min(z.max() + 10, img.shape[2])

    return img[xmin:xmax, ymin:ymax, zmin:zmax], np.array(
        [[xmin, xmax], [ymin, ymax], [zmin, zmax]])

def world2ras(voxpos):
    '''Get the vox2ras-tkr transform. Inspired
    by get_vox2ras_tkr in
    https://discourse.slicer.org/t/building-the-ijk-to-ras-transform-from-a-nrrd-file/1513
    '''
    x, y, z = voxpos
    lps_to_ras = np.diag([-1, -1, 1, 1])
    p = lps_to_ras.dot(np.asarray([x, y, z, 1]).T)
    return np.squeeze(p)[:3]

def swc2world(swc, mhd, spacing, slicer=False):
    # First transfer the image coordinates to the original image size
    sp = mhd.GetSpacing()
    swc[:, 2] *= spacing[0] / sp[0]
    swc[:, 3] *= spacing[1] / sp[1]
    swc[:, 4] *= spacing[2] / sp[2]

    # USe SimpleITK to transform back to physical coordinates
    for i in range(swc.shape[0]):
        swc[i, 2:5] = mhd.TransformContinuousIndexToPhysicalPoint(swc[i, 2:5])
        swc[i, 5] *= spacing[0]

    # Transform to RAS spacing coordinates that can be rendered in 3D Slicer if requested
    if slicer:
        print('Converting the vtk coordinates to RAS space')
        for i in range(swc.shape[0]):
            swc[i, 2:5] = world2ras(swc[i, 2:5])

    return swc

def swc2vtk(swc, outvtkpath):
    swc_arr = swc.get_array()
    nnode = swc_arr.shape[0]

    vtkstr = '# vtk DataFile Version 2.0\n'
    vtkstr += 'Generated with Rivuletpy\n'
    vtkstr += 'ASCII\n'
    vtkstr += 'DATASET POLYDATA\n'
    vtkstr += 'POINTS {} float\n'.format(nnode)

    id2vtkidx = {}
    for i in range(nnode):
        vtkstr += '{} {} {}\n'.format(swc_arr[i, 2],
                                      swc_arr[i, 3],
                                      swc_arr[i, 4])
        id2vtkidx[int(swc_arr[i, 0])] = i

    linectr = 0
    vtklinestr = ''
    for i in range(nnode):
        id, pid = swc_arr[i, 0], swc_arr[i, -1]
        if pid >= 0 and int(pid) in id2vtkidx:
            linectr += 1
            vtklinestr += '{} {} {}\n'.format(2,
                                              id2vtkidx[int(id)],
                                              id2vtkidx[int(pid)])

    vtkstr += 'LINES {} {}\n'.format(linectr, linectr * 3)
    vtkstr += vtklinestr

    vtkstr += "POINT_DATA {}\n".format(nnode)
    vtkstr += "SCALARS contourArray double\n"
    vtkstr += "LOOKUP_TABLE default\n"

    for i in range(nnode):
        vtkstr += '{}\n'.format(swc_arr[i, -2])

    vtkstr += "SCALARS indicatorArray char\n"
    vtkstr += "LOOKUP_TABLE default\n"

    for i in range(nnode):
        vtkstr += '0\n'

    with open(outvtkpath, 'w') as f:
        f.write(vtkstr)

max_stars_repo_name: twhoekstra/rivuletpy

max_issues_repo_name: twhoekstra/rivuletpy

max_forks_repo_name: twhoekstra/rivuletpy


text:
import numpy as np
from sympy.ntheory import factorint

def fft_nice(n):
    '''Given integer ``n``, check if it is at the best handling
    size fot FFTW according to
    http://www.fftw.org/fftw2_doc/fftw_3.html
    '''
    factors = factorint(n)
    max_prime = max(factors)
    if max_prime <= 7:
        return True
    elif max_prime <= 13:
        if factors.get(11, 0) + factors.get(13, 0) <= 1:
            return True
    return False

def next_fft_size(n):
    '''choose the next good FFTW size >= n
    '''
    while not fft_nice(n):
        n += 1
    return n

max_stars_repo_name: ickc/dautil-py

max_issues_repo_name: ickc/dautil-py

max_forks_repo_name: ickc/dautil-py

reason: import numpy,from sympy


text:
"""
Q Learning is based on the Bellman Update Function:
    U(S) = R(S) + gamma * max of E[U(S')]

Since we do not know the actual value of U(S'), we approximate it by max(A) of Q(S, A)
and iteratively update Q(S,A) so it converges to U(S')

Q(S,A) = (1-alpha) * Q(S,A) + alpha * (reward + gamma * max Q(S', A'))

Also, to prevent the agent from stuck in a local maxima, we use the hyper-parameter
epsilon to encourage the agent to explore unseen area.
"""

from time import sleep
import numpy as np
import gym
import random
import math

def print_frames(frames):
    for i, frame in enumerate(frames):
        print(frame['frame'])
        print(f'Timestamp: {i + 1}')
        print(f"State: {frame['state']}")
        print(f"Action: {frame['action']}")
        print(f"Reward: {frame['reward']}")
        sleep(0.1)

class Q_Learn:
    def __init__(self, env=gym.make('Taxi-v3'), alpha=0.1, gamma=0.6, epsilon_high=0.1, epsilon_low=0.0005, decay=200):
        self.alpha = alpha
        self.gamma = gamma
        self.epsilon_high = epsilon_high
        self.epsilon_low = epsilon_low
        self.decay = decay
        self.env = env
        self.q_table = np.zeros((env.observation_space.n, env.action_space.n))
        self.steps = 0

        # for approximate Q Learning
        self.features = {}
        self.weights = {}

    def reset_steps(self):
        self.steps = 0

    def epsilon_decay(self):
        return self.epsilon_low + (self.epsilon_high - self.epsilon_low) * math.exp(-self.steps / self.decay)

    def get_features(self, state, action):
        # just an example
        self.features['north'] = 1.0
        self.features['south'] = 1.0
        self.features['west'] = 1.0
        self.features['east'] = 1.0

    def get_q_value(self):
        res = 0
        for f in self.features:
            if f not in self.weights:
                self.weights[f] = 1.0
            res += self.features[f] * self.weights[f]
        return res

    def get_q_values(self, state):
        actions = self.env.action_space.n

        res = np.zeros(actions)
        for a in range(actions):
            self.get_features(state, a)
            res[a] = self.get_q_value()
        return res

    def approx_train(self, episodes=10000):
        """
        Training an approximate Q_learning algorithm.
        This approach approximates the Q value by a set of features
        multiplied by their weights. In another word Q = sum of (wi * fi)
        This way we do not really need to update the exact Q values.
        """
        env = self.env
        self.reset_steps()

        for i in range(episodes):
            state = env.reset()

            epochs, penalties, reward = 0, 0, 0
            done = False

            while not done:
                qs = self.get_q_values(state)
                actions = env.action_space.n
                diff = np.zeros(actions)
                epsilon = self.epsilon_decay()

                if random.uniform(0, 1) < epsilon:
                    action = env.action_space.sample()
                else:
                    for a in range(actions):
                        self.get_features(state, a)
                        old_value = self.get_q_value()
                        next_state, reward, done, info = env.step(a)
                        next_max = np.max(self.get_q_values(next_state))
                        diff[a] = next_max - old_value
                    action = np.argmax(diff)

                self.steps += 1
                next_state, reward, done, info = env.step(action)

                for f in self.features:
                    self.weights[f] = self.weights[f] + self.alpha * (reward + self.gamma * diff[action])

                if reward == -10:
                    penalties += 1

                state = next_state
                epochs += 1

            if i % (episodes // 10) == 0:
                print(f'episode: {i}, epochs elapsed: {epochs}, penalties incurred: {penalties}')
                print(f'weights: {self.weights}')

    def train(self, episodes=10000):
        env = self.env
        self.reset_steps()

        for i in range(episodes):
            state = env.reset()

            epochs, penalties, reward = 0, 0, 0
            done = False

            while not done:
                epsilon = self.epsilon_decay()
                if random.uniform(0, 1) < epsilon:
                    action = env.action_space.sample()
                else:
                    action = np.argmax(self.q_table[state])

                self.steps += 1
                next_state, reward, done, info = env.step(action)

                old_value = self.q_table[state, action]
                next_max = np.max(self.q_table[next_state])

                new_value = (1 - self.alpha) * old_value + self.alpha * (reward + self.gamma * next_max)
                self.q_table[state, action] = new_value

                if reward == -10:
                    penalties += 1

                state = next_state
                epochs += 1

            if i % (episodes // 10) == 0:
                print(f'episode: {i}, epochs elapsed: {epochs}, penalties incurred: {penalties}')

    def act(self, approx=False):
        env = self.env
        state = env.reset()
        epochs, penalties, reward = 0, 0, 0
        frames = []

        done = False

        while not done:
            if not approx:
                action = np.argmax(self.q_table[state])
            else:
                action = np.argmax(self.get_q_values(state))
            state, reward, done, info = env.step(action)

            if reward == -10:
                penalties += 1

            # for animation
            frames.append({
                'frame': env.render(mode='ansi'),
                'state': state,
                'action': action,
                'reward': reward
            })

            epochs += 1

        print(f'epochs elapsed: {epochs}, penalties incurred: {penalties}')
        print_frames(frames)

if __name__ == '__main__':
    q_learn = Q_Learn()
    print('start training...')
    q_learn.train()
    print(f"Animating performance...")
    q_learn.act(approx=False)

max_stars_repo_name: BobbyZhouZijian/AI-Algo-Implmentations

max_issues_repo_name: BobbyZhouZijian/AI-Algo-Implmentations

max_forks_repo_name: BobbyZhouZijian/AI-Algo-Implmentations


text:
import matplotlib.pyplot as plt
import numpy as np

fig, ax = plt.subplots()

x = np.loadtxt('data')
x = np.array((x))
y = range(0, x.size)
x_ = np.loadtxt('data_no')
y_ = range(0, x_.size)
plt.xlabel('epochs')
plt.ylabel('validation error')
plt.plot(y,x, label='No Augmentation')
plt.plot(y_,x_, label='Augmentation with random rotation' )
plt.gca().legend(('Training with noisy data', 'No Augmentation'))
plt.show()


text:
import sys
import numpy
import csv

def describe(a):
    mean = numpy.mean(a)
    median = numpy.median(a)
    std = numpy.std(a) / numpy.sqrt(len(a))
    return (mean, median, std, len(a))
    print "Mean: %f; Median: %f; STDErr: %f; N: %d" % (mean, median, std, len(a))

def load_node_file(fname, header_row = True):
    data_d = {}
    with open(fname) as fd:
        f = csv.reader(fd)
        for ix, row_data in enumerate(f):
            if header_row and ix == 0:
                assert row_data[0] == 'variant'
                continue
            # heading = (variant, size)
            heading = (row_data[0].strip(), row_data[1].strip())
            if heading not in data_d:
                data_d[heading] = ([] , [])
            tputs, missrates = data_d[heading]
            tputs.append(float(row_data[2]))
            missrates.append(float(row_data[3]))
    return data_d

if __name__ == "__main__":
    fname = sys.argv[1]
    data_d = load_node_file(fname)
    print "workload: %s (note throughputs are kreqs/s)" % fname

    rows = {}

    for (_, size) in data_d.keys():
        if size not in rows:
            rows[size] = []

    for size, row in rows.items():
        row.append( "%dk" % (int(size) / 10**3) )

        tput_def = None
        tput_hyp = None

        for variant in ["default", "hyper"]:
            tputs, missrates = data_d[(variant, size)]

            tput_mean, _, tput_std, _ = describe([i / 10**3 for i in tputs])
            missrate_mean = describe(missrates)[0]

            if variant == "default":
                tput_def = tput_mean
            if variant == "hyper":
                tput_hyp = tput_mean

            row.append( "$%.1f \\pm %.2f$" % (tput_mean, tput_std) )
            row.append( "%.2f" % (missrate_mean) )

        tput_delta = (tput_hyp - tput_def) / tput_hyp
        row.append("%.1f\\%%" % (100*tput_delta))

    for size, row in rows.items():
        print " & ".join(row),
        print " \\\\"

max_stars_repo_name: kantai/hyperbolic-caching

max_issues_repo_name: kantai/hyperbolic-caching

max_forks_repo_name: kantai/hyperbolic-caching


text:
import torch
import numpy as np

class MSE_with_regulariztion(torch.nn.Module):
    """
    loss function for hand made
    In pytorch, hand made loss function requires two method, __init__ and forward
    """

    def __init__(self, vid, l_lambda=0.01):
        """

        """
        super(MSE_with_regulariztion, self).__init__()
        self.lam = l_lambda
        self.loss_fun = torch.nn.MSELoss(reduction='sum')
        self.vid = vid

    def forward(self, act, feat):
        vid = self.vid

        loss = self.loss_fun(act, feat)

        img_mask = np.zeros_like(vid.detach().numpy())
        img_mask[-1] = 1

        diff = vid - torch.roll(vid, 1, 0)
        diff_masked = torch.masked_select(diff, torch.FloatTensor(img_mask).bool()).requires_grad_()

        loss += self.lam * torch.sum(diff_masked **2)

        return loss  # / (max_val * res_field_size **2

class CorrLoss(torch.nn.Module):
    """
    Loss function correlation coefficent replaced with MSE
    """

    def __init__(self):
        super(CorrLoss, self).__init__()

    def forward(self, act, feat):

        feat_shape = feat.shape
        act_flat = act.view(feat_shape[0], -1)
        feat_flat = feat.view(feat_shape[0], -1)

        x = act_flat
        y = feat_flat

        vx = x - torch.mean(x)
        vy = y - torch.mean(y)

        corr = torch.sum(vx * vy) / (torch.sqrt(torch.sum(vx**2))* torch.sqrt(torch.sum(vy**2)))
        return  - corr

def convert_for_corrmat(feat):
    feat_shape = feat.shape
    feat_flatten = feat.view(feat_shape[0], feat_shape[1], -1)
    return feat_flatten

def cov_torch(m, y=None):
    if y is not None:
        m = torch.cat((m, y), dim=0)
    m_exp = torch.mean(m,dim=2)
    x = m.permute(0,2,1) - m_exp[:, None]
    x = x.permute(0,2,1)
    #return x
    cov = 1 / (x.size(2) -1) * torch.einsum('bnm,bmk->bnk', x, x.permute(0,2,1))
    denom = 1. /torch.einsum('bn,bm->bnm',x.std(2),x.std(2))
    return cov * denom

class FeatCorrLoss(torch.nn.Module):
    """
    Loss function correlation coefficent replaced with MSE
    """

    def __init__(self):
        super(FeatCorrLoss, self).__init__()
        self.loss_fun = torch.nn.MSELoss()

    def forward(self, act, feat):

        feat_shape = feat.shape
        #act_flat = act.view(feat_shape[0], -1)
        #feat_flat = feat.view(feat_shape[0], -1)

        x = convert_for_corrmat(act)
        y = convert_for_corrmat(feat)

        x_mat = cov_torch(x)
        y_mat = cov_torch(y)

        return self.loss_fun(x_mat, y_mat)

class MergeLoss(torch.nn.Module):
    """
    Loss function with multiple loss
    """

    def __init__(self, loss_list, weight_list = None):
        super(MergeLoss, self).__init__()
        self.loss_list = loss_list
        if weight_list is None:
            self.weight_list = torch.ones(len(loss_list))
            self.weight_list = self.weight_list / torch.tensor(len(self.weight_list))
        else:
            self.weight_list = weight_list

    def forward(self, act, feat):
        loss = 0
        for i in range(len(self.weight_list)):
            loss_fun = self.loss_list[i]
            weight = self.weight_list[i]
            loss += loss_fun(act,feat) * weight

        return loss