text:
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Selection Node

This is a SQL Query Execution Plan Node.

This Node eliminates elminiates records which do not match a predicate using a
DNF (Disjunctive Normal Form) interpretter.

Predicates in the same list are joined with an AND/Intersection (all must be True)
and predicates in adjacent lists are joined with an OR/Union (any can be True).
This allows for non-trivial filters to be written with just tuples and lists.

The predicates are in _tuples_ in the form (`key`, `op`, `value`) where the `key`
is the value looked up from the record, the `op` is the operator and the `value`
is a literal.
"""
from typing import Iterable, Union

import numpy
from pyarrow import Table

from opteryx.engine.attribute_types import TOKEN_TYPES
from opteryx.engine.planner.operations.base_plan_node import BasePlanNode
from opteryx.engine.query_statistics import QueryStatistics
from opteryx.exceptions import SqlError
from opteryx.utils.columns import Columns

class InvalidSyntaxError(Exception):
    """
    Unable to interpret the Condition
    """

    pass

def _evaluate(predicate: Union[tuple, list], table: Table) -> bool:
    """
    Evaluate a table against a DNF selection.

    This is done by creating a mask for the values to return - we evaluate the page
    against a predicate (including resolving child predicates) and then AND or OR the
    masks together to return the rows that match the predicate.
    """

    from opteryx.third_party.pyarrow_ops import ifilters

    columns = Columns(table)

    # If we have a tuple extract out the key, operator and value and do the evaluation
    if isinstance(predicate, tuple):

        # if we're pointlessly nested, break out
        if len(predicate) == 1:
            return _evaluate(predicate=predicate[0], table=table)

        # handle NOT statements
        if len(predicate) == 2 and predicate[0] == "NOT":
            # calculate the answer of the non-negated condition (positive)
            positive_result = _evaluate(predicate=predicate[1], table=table)
            # negate it by removing the values in the positive results from
            # all of the possible values (mask)
            mask = numpy.arange(table.num_rows, dtype=numpy.int32)
            return numpy.setdiff1d(mask, positive_result, assume_unique=True)

        # this is a function in the selection
        if len(predicate) == 3 and isinstance(predicate[2], dict):
            # The function has already been evaluated, so we can use the existing
            # results
            if predicate[0] in table.column_names:
                predicate = (
                    (predicate[0], TOKEN_TYPES.IDENTIFIER),
                    "=",
                    (True, TOKEN_TYPES.BOOLEAN),
                )
            # The function has not already been evaluated, so we need to do this.
            # The evaluation SHOULD be done as part of the evaluation node, but
            # presently it only evaluates in the SELECT clause.
            else:
                ## TODO: push this to the evaluation node
                import pyarrow

                from opteryx.engine.functions import FUNCTIONS

                function = predicate[2]

                arg_list = []
                # go through the arguments and build arrays of the values
                for arg in function["args"]:
                    if arg[1] == TOKEN_TYPES.IDENTIFIER:
                        # get the column from the dataset
                        mapped_column = columns.get_column_from_alias(
                            arg[0], only_one=True
                        )
                        arg_list.append(table[mapped_column].to_numpy())
                    else:
                        # it's a literal, just add it
                        arg_list.append(arg[0])

                if len(arg_list) == 0:
                    arg_list = [
                        table.num_rows,
                    ]

                calculated_values = FUNCTIONS[function["function"]](*arg_list)
                if isinstance(calculated_values, (pyarrow.lib.StringScalar)):
                    calculated_values = [[calculated_values.as_py()]]
                table = pyarrow.Table.append_column(
                    table, predicate[0], calculated_values
                )

                predicate = (
                    (predicate[0], TOKEN_TYPES.IDENTIFIER),
                    "=",
                    (True, TOKEN_TYPES.BOOLEAN),
                )

        if (
            len(predicate) == 3
            and len(predicate[0]) == 2
            and predicate[0][1] == TOKEN_TYPES.IDENTIFIER
            and (predicate[0][0] not in table.column_names)
        ):
            raise SqlError(f"Field `{predicate[0][0]}` does not exist.")

        if len(predicate[0]) == 3 and isinstance(predicate[0][2], dict):
            raise SqlError(
                "WHERE clauses cannot contain function calls as part of an expression."
            )

        if not isinstance(predicate[0], tuple):
            return _evaluate(predicate[0], table)

        # filters from pyarrow_ops only filters on a single predicate
        return ifilters(table, predicate)

    # If we have a list, we're going to recurse and call ourselves with the items in
    # the list
    if isinstance(predicate, list):
        # Are all of the entries tuples?
        # We AND them together
        mask = None
        if all(isinstance(p, tuple) for p in predicate):
            # default to all selected
            mask = numpy.arange(table.num_rows, dtype=numpy.int32)
            for part in predicate:
                mask = numpy.intersect1d(mask, _evaluate(part, table))
            return mask  # type:ignore

        # Are all of the entries lists?
        # We OR them together
        if all(isinstance(p, list) for p in predicate):
            # default to none selected
            mask = numpy.zeros(0, dtype=numpy.int32)
            for part in predicate:
                mask = numpy.union1d(mask, _evaluate(part, table))  # type:ignore
            return mask  # type:ignore

        # if we're here the structure of the filter is wrong
        raise InvalidSyntaxError("Unable to evaluate Filter")  # pragma: no cover

    raise InvalidSyntaxError("Unable to evaluate Filter")  # pragma: no cover

def _evaluate_subqueries(predicate):
    """
    Traverse the filters looking for where we have query execution plans, these
    are subqueries as part of IN and NOT IN conditions which we need to resolve
    in order to evaluate the predicate.
    """
    if (
        isinstance(predicate, tuple)
        and len(predicate) == 2
        and predicate[1] == TOKEN_TYPES.QUERY_PLAN
    ):
        import pyarrow

        table_result = pyarrow.concat_tables(predicate[0].execute())
        if len(table_result) == 0:
            SqlError("Subquery in WHERE clause - column not found")
        if len(table_result.columns) != 1:
            raise SqlError("Subquery in WHERE clause - returned more than one column")
        value_list = table_result.column(0).to_numpy()
        # performing IN with a set is much faster than numpy arrays
        value_list = set(value_list)
        return (value_list, TOKEN_TYPES.LIST)
    if isinstance(predicate, tuple):
        return tuple(_evaluate_subqueries(p) for p in predicate)
    if isinstance(predicate, list):
        return list(_evaluate_subqueries(p) for p in predicate)
    return predicate

def _map_columns(predicate, columns):
    """
    This rewrites the filters to refer to the internal column names.
    """
    if isinstance(predicate, tuple):
        if len(predicate) > 1 and predicate[1] == TOKEN_TYPES.IDENTIFIER:
            identifier = columns.get_column_from_alias(predicate[0])
            if len(identifier) == 1:
                return (
                    identifier[0],
                    TOKEN_TYPES.IDENTIFIER,
                )
            return predicate
        return tuple(_map_columns(p, columns) for p in predicate)
    if isinstance(predicate, list):
        return list(_map_columns(p, columns) for p in predicate)
    return predicate

class SelectionNode(BasePlanNode):
    def __init__(self, statistics: QueryStatistics, **config):
        self._filter = config.get("filter")
        self._unfurled_filter = None
        self._mapped_filter = None

    @property
    def config(self):  # pragma: no cover
        return str(self._filter)

    @property
    def name(self):  # pragma: no cover
        return "Selection"

    def execute(self, data_pages: Iterable) -> Iterable:

        if isinstance(data_pages, Table):
            data_pages = (data_pages,)

        # we should always have a filter - but harm checking
        if self._filter is None:
            yield from data_pages

        else:
            # if any values in the filters are subqueries, we have to execute them
            # before we can continue.
            self._unfurled_filter = _evaluate_subqueries(self._filter)

            for page in data_pages:

                # what we want to do is rewrite the filters to refer to the column names
                # NOT rewrite the column names to match the filters
                if self._mapped_filter is None:
                    columns = Columns(page)
                    self._mapped_filter = _map_columns(self._unfurled_filter, columns)

                mask = _evaluate(self._mapped_filter, page)
                yield page.take(mask)

max_stars_repo_name: mabel-dev/waddles

max_issues_repo_name: mabel-dev/waddles

max_forks_repo_name: mabel-dev/waddles


text:
from math import *
import numpy as np

def seven_segment(pattern):

    def to_bool(a):
        if a==1:
            return True
        return False

    def hor(d):
        if d:
            print(" _ ")
        else:
            print("   ")

    def vert(d1,d2,d3):
        word=""

        if d1:
            word="|"
        else:
            word=" "

        if d3:
            word+="_"
        else:
            word+=" "

        if d2:
            word+="|"
        else:
            word+=" "

        print(word)

    pattern_b=list(map(to_bool,pattern))

    hor(pattern_b[0])
    vert(pattern_b[1],pattern_b[2],pattern_b[3])
    vert(pattern_b[4],pattern_b[5],pattern_b[6])

    number=0
    for i in range(0,4):
        if pattern_b[7+i]:
            number+=pow(2,i)
    print(int(number))

six=[1,1,-1,1,1,1,1,-1,1,1,-1]
three=[1,-1,1,1,-1,1,1,1,1,-1,-1]
one=[-1,-1,1,-1,-1,1,-1,1,-1,-1,-1]

print("reference patterns:")
seven_segment(three)
seven_segment(six)
seven_segment(one)
print("---")

# create the matrix of connection weights W
weight_matrix = np.zeros((11, 11))

# put the patterns in a list so we can iterate over them
patterns = [six, three, one]

# we want to iterate over every unique (i,j) pair
for i in range(0, 11):
    for j in range(i+1, 11):
        # compute the weight value at this point
        v = 0
        for pattern in patterns:
            v += pattern[i] * pattern[j]
        v /= len(patterns)

        # set the appropriate values in the matrix
        weight_matrix[i, j] = v
        weight_matrix[j, i] = v

# define a function to iterate a pattern according
# to the McCulloch-Pitts relation
def evolve(pattern):
    # create an array in which to store the new pattern
    p = []

    # compute each value based on the original pattern
    for i in range(len(pattern)):
        v = 0
        for j in range(len(pattern)):
            v += weight_matrix[i][j] * pattern[j]
        if v > 0:
            v = 1
        else:
            v = -1
        p.append(v)

    # check if we have converged
    for i in range(len(p)):
        if p[i] != pattern[i]:
            return p # HAVE NOT CONVERGED

    # otherwise return none if we have converged
    return None

# create a function to keep evolving a pattern until it converges
def converge(pattern):
    p = pattern
    print("original: ", p)
    while True:
        candidate = evolve(p)
        if candidate == None:
            print("converged:", p)
            return p
        p = candidate
        print("step:     ", p)

print("\ntest1")

test=[1,-1,1,1,-1,1,1,-1,-1,-1,-1]
seven_segment(test)
test=converge(test)
seven_segment(test)

print("\ntest2")

test=[1,1,1,1,1,1,1,-1,-1,-1,-1]
seven_segment(test)
test=converge(test)
seven_segment(test)

ext: py

max_stars_repo_name: awnumar/computational-neuroscience

max_issues_repo_name: awnumar/computational-neuroscience

max_forks_repo_name: awnumar/computational-neuroscience

reason: import numpy


text:
import torch
import torch.nn as nn
import torch.nn.functional as F
from mmcv.cnn import xavier_init
from mmcv import Config
import numpy as np
from .resnet_tpn import *

class Identity(nn.Module):

  def __init__(self):
    super(Identity, self).__init__()

  def forward(self, x):
    return x

class CONV(nn.Module):
   def __init__(
       self,
       inplanes,
       planes,
       kernel_size,
       stride,
       padding,
       bias=False,
       groups=1,
       ):
       super(CONV, self).__init__()
       self.conv = nn.Conv3d(inplanes, planes, kernel_size, stride, padding, bias=bias, groups=groups)
       self.bn = nn.BatchNorm3d(planes)
       self.relu = nn.ReLU(inplace=True)

   def forward(self, x):
       out = self.relu(self.bn(self.conv(x)))
       return out

class HEAD(nn.Module):
   def __init__(
       self,
       inplanes,
       planes,
       kernel_size,
       stride,
       padding,
       bias,
       conv_num=1,
       loss_weight=0.5,
       multilabel=False
       ):
       super(HEAD, self).__init__()

       self.convs = nn.ModuleList()
       for i in range(conv_num):
          dim_in = 2 ** (i)
          dim_out = 2 ** (i+1)
          self.convs.append(CONV(inplanes*dim_in, inplanes*dim_out, kernel_size, stride, padding, bias))

       self.loss_weight = loss_weight
       self.dropout = nn.Dropout(p=0.5)
       dim = 2 ** (conv_num)
       self.fc = nn.Linear(inplanes*dim, planes)
       self.new_cls = None
       self.indim = inplanes*dim
       self.outdim = planes
       self.multilabel = multilabel
   def init_weights(self):
       for m in self.modules():
            if isinstance(m, nn.Linear):
               nn.init.normal_(m.weight, 0, 0.01)
               nn.init.constant_(m.bias, 0)
            if isinstance(m, nn.Conv3d):
                xavier_init(m, distribution='uniform')
            if isinstance(m, nn.BatchNorm3d):
                m.weight.data.fill_(1)
                m.bias.data.fill_(0)

   def forward(self, x, target=None, fcn_testing=False):
       loss = dict()
       if len(self.convs) > 0:
           for i, conv in enumerate(self.convs):
               x = conv(x)

       if fcn_testing:
           # fcn testing
           if self.new_cls is None:
               # create a conv head
               self.new_cls = nn.Conv3d(self.indim,self.outdim,1,1,0).cuda()
               self.new_cls.weight.copy_(self.fc.weight.unsqueeze(-1).unsqueeze(-1).unsqueeze(-1))
               self.new_cls.bias.copy_(self.fc.bias)
               self.fc = None
           # predict the calss map
           x = self.new_cls(x)
           return x

       x = F.adaptive_avg_pool3d(x,1).squeeze(-1).squeeze(-1).squeeze(-1)
       x = self.dropout(x)
       x = self.fc(x)

       if target is None:
           return x
       if self.multilabel:
           loss['loss_aux'] = self.loss_weight * binary_weighted_multilabel_binary_cross_entropy(x, target, target.new_ones(target.shape))
       else:
           loss['loss_aux'] = self.loss_weight * F.cross_entropy(x, target)
       return loss

class LCONV(nn.Module):
    def __init__(self,
                 inplanes,
                 planes,
                 kernel_size,
                 stride,
                 padding,
                 bias=False,
                 groups=1,
                 norm=False,
                 activation=False,
                 downsample_type='avg',
                 downsample_position='before',
                 downsample_scale=8,
                 outconv=False,
                 ):
        super(LCONV, self).__init__()

        self.conv = nn.Conv3d(inplanes, planes, kernel_size, stride, padding, bias=bias, groups=groups)
        self.norm = nn.BatchNorm3d(planes) if norm else None
        self.relu = nn.ReLU(inplace=True) if activation else None

        assert(downsample_type in ['avg','max'])
        assert(downsample_position in ['before','after'])

        self.downsample_position = downsample_position

        self.pool = nn.AvgPool3d((downsample_scale, 1, 1), (downsample_scale, 1, 1), (0, 0, 0), ceil_mode=True, count_include_pad=False) if downsample_type == 'avg' else nn.MaxPool3d((downsample_scale, 1, 1), (downsample_scale, 1, 1), (0, 0, 0), ceil_mode=True)

        self.outconv = nn.Conv3d(planes, planes, (1,1,1), (1,1,1), (0,0,0), bias=False, groups=groups) if outconv else None

    def forward(self, x):
        if self.downsample_position == 'before':
           x = self.pool(x)

           x = self.conv(x)
           if self.norm is not None:
               x = self.norm(x)
           if self.relu is not None:
               x = self.relu(x)
           # 1x1 conv
           if self.outconv is not None:
               x = self.outconv(x)

        elif self.downsample_position == 'after':
           x = self.conv(x)
           if self.norm is not None:
               x = self.norm(x)
           if self.relu is not None:
               x = self.relu(x)
           x = self.pool(x)
           # 1x1 conv
           if self.outconv is not None:
               x = self.outconv(x)

        return x

class UPOP(nn.Module):
    def __init__(self,
                 inplanes,
                 planes,
                 groups=1,
                 norm=False,
                 activation=False,
                 upscale=(2,1,1),
                 upconv=False,
                 pre_act=False,
                ):
        super(UPOP, self).__init__()
        self.conv = nn.Conv3d(inplanes, planes, 1, 1, 0, bias=False, groups=groups) if upconv else None
        if not upconv:
           norm = False
           activation = False
        self.pre_act = pre_act
        if self.pre_act:
            self.norm = nn.BatchNorm3d(inplanes) if norm else None
        else:
            self.norm = nn.BatchNorm3d(planes) if norm else None
        self.relu = nn.ReLU(inplace=True) if activation else None

        #self.upop = nn.Upsample(scale_factor=upscale, mode='nearest')
        self.scale = upscale
    def forward(self, x):
        x = F.interpolate(x, scale_factor=self.scale, mode='nearest')
        if self.pre_act:
            if self.norm is not None:
                x = self.norm(x)
            if self.relu is not None:
                x = self.relu(x)
            if self.conv is not None:
                x = self.conv(x)
        else:
            if self.conv is not None:
                x = self.conv(x)
            if self.norm is not None:
                x = self.norm(x)
            if self.relu is not None:
                x = self.relu(x)
        return x

class FCONV(nn.Module):
    def __init__(self,
                inplanes,
                planes,
                kernel_size,
                stride,
                padding,
                bias=False,
                groups=1,
                norm=False,
                activation=False,
                pre_act=False,
                ):
        super(FCONV, self).__init__()

        self.conv = nn.Conv3d(inplanes, planes, kernel_size, stride, padding, bias=bias, groups=groups)
        self.pre_act = pre_act
        if self.pre_act:
            self.norm = nn.BatchNorm3d(inplanes) if norm else None
        else:
            self.norm = nn.BatchNorm3d(planes) if norm else None
        self.relu = nn.ReLU(inplace=True) if activation else None

    def forward(self, x):
        if self.pre_act:
            if self.norm is not None:
               x = self.norm(x)
            if self.relu is not None:
               x = self.relu(x)
            x = self.conv(x)

        else:
            x = self.conv(x)
            if self.norm is not None:
               x = self.norm(x)
            if self.relu is not None:
               x = self.relu(x)
        return x

class DOWNOP(nn.Module):
    def __init__(self,
                 inplanes,
                 planes,
                 kernel_size,
                 stride,
                 padding,
                 bias=False,
                 groups=1,
                 norm=False,
                 activation=False,
                 downsample_type='avg',
                 downsample_position='before',
                 downsample_scale=(1,2,2),
                 outconv=False,
                 pre_act=False,
                 ):
        super(DOWNOP, self).__init__()

        self.conv = nn.Conv3d(inplanes, planes, kernel_size, stride, padding, bias=bias, groups=groups)
        if pre_act:
            self.norm = nn.BatchNorm3d(inplanes) if norm else None
        else:
            self.norm = nn.BatchNorm3d(planes) if norm else None

        self.relu = nn.ReLU(inplace=True) if activation else None
        self.pre_act = pre_act
        assert(downsample_type in ['avg','max','none','sampler'])
        assert(downsample_position in ['before','after'])

        self.downsample_position = downsample_position

        self.pool = nn.AvgPool3d(downsample_scale, downsample_scale, (0, 0, 0), ceil_mode=True, count_include_pad=False) if downsample_type == 'avg' else nn.MaxPool3d(downsample_scale, downsample_scale, (0, 0, 0), ceil_mode=True)

        self.outconv = nn.Conv3d(planes, planes, (1,1,1), (1,1,1), (0,0,0), bias=False, groups=groups) if outconv else None

        if downsample_type == 'sampler':
           self.pool = None
           self.outconv = None
           self.conv = None
           self.norm = None
           self.relu = None
           self.tscale = int(downsample_scale[0])
        else:
           self.tscale = None

    def forward(self, x):
        if self.tscale is not None:
           return x[:,:,0::self.tscale,:,:]
        if self.downsample_position == 'before':
           x = self.pool(x)

        if self.pre_act:
           if self.norm is not None:
               x = self.norm(x)
           if self.relu is not None:
               x = self.relu(x)
           x = self.conv(x)

        else:
           x = self.conv(x)
           if self.norm is not None:
               x = self.norm(x)
           if self.relu is not None:
               x = self.relu(x)

        if self.downsample_position == 'after':
           x = self.pool(x)

        if self.outconv is not None:
           x = self.outconv(x)

        return x

class ADAPOOL(nn.Module):
    def __init__(self,
                 in_channels=[256,512,1024,2048],
                 mid_channels=[512,512,512,512],
                 out_channels=2048,
                 sampler='ds',
                 ds_scales=[(8,8,8),(4,4,4),(2,2,2),(1,1,1)],
                 kernel_size=(1,3,3),
                 stride=(1,1,1),
                 padding=(0,1,1),
                 bias=False,
                 groups=1,
                 norm=True,
                 activation=True,
                 ds_type='avg',
                 ds_position='before',
                 fusion_type='sum',
                 pre_act=False,
                 fz_groups=1,
                 no_fz_conv=False,
                 ):
        super(ADAPOOL, self).__init__()
        self.fusion_type = fusion_type
        assert(fusion_type in ['sum','concat','max'])
        assert(sampler in ['ds', 'us'])
        self.fusion_type = fusion_type

        self.ops = nn.ModuleList()
        num_ins = len(in_channels)
        for i in range(num_ins):
            if sampler == 'ds':
                op = DOWNOP(in_channels[i], mid_channels[i], kernel_size, stride, padding, bias, groups, norm, activation, ds_type, ds_position, ds_scales[i], False, pre_act)
            else:
                op = UPOP(in_channels[i], mid_channels[i], groups, norm, activation, ds_scales[i], True, pre_act)
            self.ops.append(op)

        if fusion_type == 'sum' or fusion_type == 'max':
            in_dims = mid_channels[0]
        else:
            in_dims = np.sum(mid_channels)

        if pre_act:
            self.fusion_conv = nn.Sequential(
                nn.BatchNorm3d(in_dims),
                nn.ReLU(inplace=True),
                nn.Conv3d(in_dims, out_channels, 1, 1, 0, bias=False, groups=fz_groups)
                )
        else:
            self.fusion_conv = nn.Sequential(
                nn.Conv3d(in_dims, out_channels, 1, 1, 0, bias=False, groups=fz_groups),
                nn.BatchNorm3d(out_channels),
                nn.ReLU(inplace=True)
                )
        if no_fz_conv:
            self.fusion_conv = None

    def forward(self, inputs):
        out = [self.ops[i](feature) for i, feature in enumerate(inputs)]
        if self.fusion_type == 'sum':
           out = torch.sum(torch.stack(out),0)
        elif self.fusion_type == 'max':
           out = torch.max(torch.stack(out),0)[0]
        elif self.fusion_type == 'concat':
           out = torch.cat(out, 1)
        if self.fusion_conv is not None:
           out = self.fusion_conv(out)
        return out

class SDS(nn.Module):
   def __init__(
       self,
       inplanes=[256,512,1024,2048],
       planes=2048,
       kernel_size=(1,3,3),
       stride=(1,2,2),
       padding=(0,1,1),
       bias=False,
       groups=1,
       slim=False,
       save_mode=False,
       ):
       super(SDS, self).__init__()

       self.sds = nn.ModuleList()
       self.save_mode = save_mode

       slim=slim
       if False:
           for i, dim in enumerate(inplanes):
               ds_factor = planes // dim
               if dim == planes:
                   current_op = Identity()
               else:
                   current_op = nn.Sequential(
                       nn.MaxPool3d((1, ds_factor, ds_factor), (1, ds_factor, ds_factor), (0, 0, 0), ceil_mode=True),
                       nn.Conv3d(dim, planes, kernel_size=kernel_size, stride=stride, padding=padding, bias=bias, groups=groups)
                   )
               self.sds.append(current_op)

       elif slim:
           if save_mode:
               self.sds = nn.ModuleList()
               op=nn.ModuleList()
               curr_op = nn.Sequential(
                   nn.MaxPool3d((1, 2, 2), (1, 2, 2), (0, 0, 0), ceil_mode=True),
                   nn.Conv3d(1024, planes, 1, 1, 0, bias=bias, groups=groups)
               )
               op.append(curr_op)
               self.sds.append(op)

               op=nn.ModuleList()
               op.append(nn.Conv3d(2048, planes, 1, 1, 0, bias=bias, groups=groups))
               self.sds.append(op)
           else:
               # slim version
               self.sds = nn.ModuleList()
               op=nn.ModuleList()
               op.append(CONV(inplanes[0], planes, kernel_size=kernel_size, stride=stride, padding=padding, bias=bias, groups=groups))
               self.sds.append(op)

               op=nn.ModuleList()
               op.append(CONV(inplanes[1], planes, kernel_size=1, stride=1, padding=0, bias=bias, groups=32))
               self.sds.append(op)

       else:
           for i, dim in enumerate(inplanes):
               op = nn.ModuleList()
               ds_factor = planes // dim
               ds_num  = int(np.log2(ds_factor))
               if ds_num < 1:
                  op = Identity()
               else:
                  for dsi in range(ds_num):
                      in_factor = 2 ** dsi
                      out_factor = 2 ** (dsi+1)
                      op.append(CONV(dim*in_factor, dim*out_factor, kernel_size=kernel_size, stride=stride, padding=padding, bias=bias, groups=groups))
               self.sds.append(op)

   def forward(self, inputs):
       out = []
       for i, feature in enumerate(inputs):
            if isinstance(self.sds[i], nn.ModuleList):
                out_ = inputs[i]
                for III, op in enumerate(self.sds[i]):
                    out_ = op(out_)
                out.append(out_)
            else:
                out.append(self.sds[i](inputs[i]))
       return out

class FZOP(nn.Module):
   def __init__(
       self,
       inplanes=2048,
       planes=1024,
       kernel_size=1,
       stride=1,
       padding=0,
       bias=False,
       groups=1,
       fz_type='add',
       ):
       super(FZOP, self).__init__()
       assert(fz_type in ['add','concat'])
       self.fz_type = fz_type
       if fz_type == 'concat':
            self.conv = nn.Conv3d(inplanes, planes, kernel_size, stride, padding, bias=bias, groups=groups)

   def forward(self, x, y):
       if self.fz_type == 'add':
            return x+y
       if self.fz_type == 'concat':
            return self.conv(torch.cat([x,y],1))

class TPN(nn.Module):

    def __init__(self,
                 num_seg=num_seg,
                 in_channels=[256,512,1024,2048],
                 out_channels=256,
                 return_original=True,
                 fz_conv=True,
                 swap=False,
                 TEST_MODE=False,
                 concat_fusion=False,
                 two_pyramid=False,
                 two_pyramid_fusion=None,
                 fz_pyramid_groups=1,
                 dynamic_sampler_rate=None,
                 fz_config=None,
                 res_config=None,
                 sds_config=None,
                 lconv_config=None,
                 upop_config=None,
                 fconv_config=None,
                 downop_config=None,
                 adapool_config=None,
                 aux_head_config=None,
                 aux_head_return=True,
                 ):
        super(TPN, self).__init__()
        self.num_seg = num_seg
        assert isinstance(in_channels, list)
        assert isinstance(out_channels, int)
        self.in_channels = in_channels
        self.out_channels = out_channels
        self.num_ins = len(in_channels)
        self.return_original = return_original
        self.aux_head_return = aux_head_return

        self.res_config = Config(res_config) if isinstance(res_config, dict) else res_config
        sds_config = Config(sds_config) if isinstance(sds_config, dict) else sds_config
        lconv_config = Config(lconv_config) if isinstance(lconv_config, dict) else lconv_config
        fconv_config = Config(fconv_config) if isinstance(fconv_config, dict) else fconv_config
        upop_config = Config(upop_config)  if isinstance(upop_config, dict) else upop_config
        downop_config = Config(downop_config) if isinstance(downop_config, dict) else downop_config
        aux_head_config = Config(aux_head_config) if isinstance(aux_head_config ,dict) else aux_head_config
        adapool_config = Config(adapool_config) if isinstance(adapool_config, dict) else adapool_config
        fz_config = Config(fz_config) if isinstance(fz_config, dict) else fz_config

        self.TEST_MODE = TEST_MODE

        self.concat_fusion = concat_fusion
        self.lconvs = nn.ModuleList()
        self.fconvs = nn.ModuleList()
        self.upops= nn.ModuleList()
        self.upfzops = nn.ModuleList()
        self.downops = nn.ModuleList()
        self.downfzops = nn.ModuleList()
        self.adapool_ops = ADAPOOL(**adapool_config) if adapool_config is not None else None
        self.sds = SDS(**sds_config) if sds_config is not None else None
        self.swap = swap

        self.tdfzops= nn.ModuleList()
        self.bufzops= nn.ModuleList()
        self.dynamic_sampler_rate = dynamic_sampler_rate
        if dynamic_sampler_rate is not None:
            assert(self.num_ins == len(dynamic_sampler_rate))

        for i in range(0, self.num_ins, 1):
            if dynamic_sampler_rate is None:
                inplanes = in_channels[i] if sds_config is None else sds_config.planes
            else:
                inplanes = in_channels[-1]

            planes = out_channels

            if lconv_config is not None:
                # overwrite the lconv_config
                lconv_config.param.downsample_scale = lconv_config.scales[i]
                lconv_config.param.inplanes = inplanes
                lconv_config.param.planes = planes
                lconv = LCONV(**lconv_config.param)
                self.lconvs.append(lconv)

            if fconv_config is not None:
                # overwrite the fconv_config
                fconv_config.param.inplanes = planes
                fconv_config.param.planes = planes
                fconv = FCONV(**fconv_config.param)
                self.fconvs.append(fconv)

            if i < self.num_ins - 1:
                if upop_config is not None:
                    # overwrite the upop_config
                    upop_config.param.inplanes = planes
                    upop_config.param.planes = planes
                    upop_config.param.upscale = upop_config.scales
                    upop = UPOP(**upop_config.param)
                    self.upops.append(upop)

                if downop_config is not None:
                    # overwrite the downop_config
                    downop_config.param.inplanes = planes
                    downop_config.param.planes = planes
                    downop_config.param.downsample_scale = downop_config.scales
                    downop = DOWNOP(**downop_config.param)
                    self.downops.append(downop)

                if fz_config is None:
                    tdfz_op = FZOP(-1,-1,-1,-1,-1,False,1,'add')
                    bufz_op = FZOP(-1,-1,-1,-1,-1,False,1,'add')
                else:
                    tdfz_op = FZOP(**fz_config)
                    bufz_op = FZOP(**fz_config)

                self.tdfzops.append(tdfz_op)
                self.bufzops.append(bufz_op)

        if self.res_config is not None and self.res_config.activation:
            assert(self.res_config.position in ['before_fconv', 'after_fconv', 'before_ada'])
            self.relu = nn.ReLU(inplace=True)

        if len(self.lconvs) == 0:
            self.lconvs = None
        if len(self.fconvs) == 0:
            self.fconvs = None
        if len(self.upops) == 0:
            self.upops = None
        if len(self.downops) == 0:
            self.downops = None
        if len(self.tdfzops) == 0:
            self.tdfzops = None
        if len(self.bufzops) == 0:
            self.bufzops = None
        if not concat_fusion:
            self.upfzops = None
            self.downfzops = None

        if self.lconvs is not None:
            if adapool_config is not None:
                out_dims = adapool_config.out_channels
            else:
                out_dims = out_channels
            if fz_conv:
                self.fusion_conv = nn.Sequential(
                    nn.Conv3d(out_dims, in_channels[-1], 1, 1, 0, bias=False),
                    nn.BatchNorm3d(in_channels[-1]),
                    nn.ReLU(inplace=True)
                    )
            else:
                self.fusion_conv = None

        # Two pyramid
        self.two_pyramid_fusion = two_pyramid_fusion
        if self.adapool_ops is not None and two_pyramid:
            self.adapool_ops2 = ADAPOOL(**adapool_config) if adapool_config is not None else None

            out_dims = adapool_config.out_channels
            if two_pyramid_fusion is None:
                indim = out_dims*2
            elif two_pyramid_fusion == 'concat':
                indim = out_dims*2
            elif two_pyramid_fusion == 'sum':
                indim = out_dims
            elif two_pyramid_fusion == 'max':
                indim = out_dims

            self.fz_pyramid = nn.Sequential(
                    nn.Conv3d(indim, 2048, 1, 1, 0, bias=False, groups=fz_pyramid_groups),
                    nn.BatchNorm3d(2048),
                    nn.ReLU(inplace=True)
                    )
        else:
            self.adapool_ops2 = None
            self.fz_pyramid = None

        if aux_head_config is not None:
           # overwrite aux_head_config
           self.aux_position = aux_head_config.position
           #assert(self.aux_position in ['ori','lconv','bottomup','fconv','topdown'])
           assert(self.aux_position in ['ori','sds'])

           aux_head_config.param.inplanes = self.in_channels[-2] if aux_head_config.position == 'ori' else sds_config.planes
           aux_head_config.param.planes = 400 if aux_head_config.param.planes < 0 else aux_head_config.param.planes

           self.aux_head = HEAD(**aux_head_config.param)
        else:
           self.aux_head = None
           self.aux_position = 'no_aux_head'

    # default init_weights for conv(msra) and norm in ConvModule
    def init_weights(self):
        for m in self.modules():
            if isinstance(m, nn.Conv3d):
                xavier_init(m, distribution='uniform')
            if isinstance(m, nn.BatchNorm3d):
                m.weight.data.fill_(1)
                m.bias.data.fill_(0)

        if self.fusion_conv is not None and self.return_original:
            self.fusion_conv[1].weight.data.fill_(0)
            self.fusion_conv[1].bias.data.fill_(0)
        if self.aux_head is not None:
            self.aux_head.init_weights()

    def forward(self, inputs, target=None):

        def trans(tensor):
            bs, c, h, w = tensor.shape
            tensor = tensor.reshape(int(bs//self.num_seg), self.num_seg, c, h, w)
            tensor = tensor.permute(0,2,1,3,4)
            return tensor

        inputs = [trans(x) for x in inputs]

        if self.dynamic_sampler_rate is None:
             assert len(inputs) == len(self.in_channels)

        loss = None

        # aux loss
        if self.aux_position == 'ori':
            loss = self.aux_head(inputs[-2], target) if self.aux_head is not None else None
            if self.aux_head_return:
                return inputs[-1], loss

        # sds
        if self.sds is not None:
            outs = self.sds(inputs)
        else:
            outs = inputs

        # aux loss
        if self.aux_position == 'sds':
            loss = self.aux_head(outs[-2], target) if self.aux_head is not None else None

        # dynamic sampling
        if self.dynamic_sampler_rate is not None:
            #outs = [inputs[-1][:,:,0::int(rate),:,:] for i, rate in enumerate(self.dynamic_sampler_rate)] # dynamic sampling
            outs = [F.max_pool3d(inputs[-1], (rate,1,1)) for i, rate in enumerate(self.dynamic_sampler_rate)] # maxpool to downsample

        # lconv
        if self.lconvs is not None:
            outs = [lconv(outs[i]) for i, lconv in enumerate(self.lconvs)]

        # get the lconv outs for pyramid 2
        if self.adapool_ops2 is not None:
            lconv_outs = outs

        # build top-down path
        if self.swap:
            # down operation i.e. top-down path
            if self.downops is not None:
                for i in range(self.num_ins-1, 0, -1):
                     if self.tdfzops is None:
                         outs[i - 1] = outs[i - 1] + self.downops[i-1](outs[i])
                     else:
                         outs[i - 1] = self.tdfzops[i-1](outs[i - 1], self.downops[i-1](outs[i]))

        else:
            # up operation i.e. top-down path
            if self.upops is not None:
                for i in range(self.num_ins-1, 0, -1):
                     if self.tdfzops is None:
                         outs[i - 1] = outs[i - 1] + self.upops[i-1](outs[i])
                     else:
                         outs[i - 1] = self.tdfzops[i-1](outs[i - 1], self.upops[i-1](outs[i]))

        # fconv
        if self.fconvs is not None:
            outs = [self.fconvs[i](outs[i]) for i in range(self.num_ins)]

        # get pyramid 1 outs
        if self.adapool_ops2 is not None:
            assert(self.fconvs is None)
            topdownouts = self.adapool_ops2(outs)
            outs = lconv_outs

        # build bottom-up path
        if self.swap:
            # up operation i.e. top-down path
            if self.upops is not None:
                for i in range(0, self.num_ins-1, 1):
                     if self.bufzops is None:
                         outs[i + 1] = outs[i + 1] + self.upops[i](outs[i])
                     else:
                         outs[i + 1] = self.bufzops[i](outs[i + 1], self.upops[i](outs[i]))

        else:
            # down operation i.e. bottom-up path
            if self.downops is not None:
                for i in range(0, self.num_ins-1, 1):
                     if self.bufzops is None:
                         outs[i+1] = outs[i+1] + self.downops[i](outs[i])
                     else:
                         outs[i+1] = self.bufzops[i](outs[i+1], self.downops[i](outs[i]))

        # adapool operation
        if self.adapool_ops is not None:
            outs = self.adapool_ops(outs)
            if self.fusion_conv is not None:
                 outs = self.fusion_conv(outs)
        else:
            if self.fusion_conv is not None:
                 outs = self.fusion_conv(outs[-1])
            else:
                 outs = outs[-1]

        # fuse two pyramid outs
        if self.fz_pyramid is not None:
            if self.two_pyramid_fusion is None:
                outs = self.fz_pyramid(torch.cat([topdownouts, outs],1))
            elif self.two_pyramid_fusion == 'max':
                outs = self.fz_pyramid(torch.max(torch.stack([topdownouts, outs]) ,0)[0])
            elif self.two_pyramid_fusion == 'sum':
                outs = self.fz_pyramid(torch.sum(torch.stack([topdownouts, outs]) ,0))

        return outs, loss

def main():
   res2 = torch.FloatTensor(8,256,8,56,56).cuda()
   res3 = torch.FloatTensor(8,512,8,28,28).cuda()
   res4 = torch.FloatTensor(8,1024,8,14,14).cuda()
   res5 = torch.FloatTensor(8,2048,8,7,7).cuda()
   #feature = tuple([res2, res3, res4, res5])
   feature = tuple([res4, res5])
   model = TPN(
        in_channels=[1024,2048],
        out_channels=1024,
        return_original=False,
        fz_conv=False,
        dynamic_sampler_rate=None,
        swap=False,
        two_pyramid=True,
        sds_config=dict(inplanes=[1024,2048],
           planes=2048,
           kernel_size=(1,1,1),
           stride=(1,1,1),
           padding=(0,0,0),
           bias=False,
           save_mode=True,
           groups=32),
        lconv_config=dict(scales=(8,8),
           param=dict(inplanes=-1,
           planes=-1,
           kernel_size=(3,1,1),
           stride=(1,1,1),
           padding=(1,0,0),
           bias=False,
           groups=32,
           norm=False,
           activation=False,
           downsample_type='max',
           downsample_position='after',
           downsample_scale=-1,
           outconv=False)),
        upop_config=dict(scales=(1,1,1),
           param=dict(inplanes=-1,
           planes=-1,
           groups=1,
           norm=False,
           activation=False,
           upscale=-1,
           upconv=False)),
        downop_config=dict(scales=(1,1,1),
           param=dict(inplanes=-1,
           planes=-1,
           kernel_size=(3,1,1),
           stride=(1,1,1),
           padding=(1,0,0),
           bias=False,
           groups=1,
           norm=False,
           activation=False,
           downsample_type='max',
           downsample_position='after',
           downsample_scale=-1,
           outconv=False)),
        adapool_config=dict(
           in_channels=[1024,1024],
           mid_channels=[1024,1024],
           out_channels=2048,
           sampler='ds',
           #ds_scales=[(8,8,8),(4,4,4),(2,2,2),(1,1,1)],
           ds_scales=[(1,1,1),(1,1,1)],
           ds_type='max',
           ds_position='before',
           kernel_size=(1,1,1),
           stride=(1,1,1),
           padding=(0,0,0),
           bias=False,
           groups=32,
           norm=False,
           activation=False,
           fusion_type='concat'),
        aux_head_config=dict(
           position='ori',
           param=dict(
           inplanes=-1,
           planes=-1,
           kernel_size=(1,3,3),
           stride=(1,2,2),
           padding=(0,1,1),
           bias=False,
           conv_num=1,
           loss_weight=0.5
           )),
       aux_head_return=False,
                 ).cuda()
   print(model)
   out, prob = model(feature)
   print(out.shape)
   #print(prob.shape)

if __name__=='__main__':
   main()

max_stars_repo_name: justimyhxu/temporal-shift-module

max_issues_repo_name: justimyhxu/temporal-shift-module

max_forks_repo_name: justimyhxu/temporal-shift-module


text:
"""
==========
Quickstart
==========
"""

###############################################################################
# If you have not installed *SMAC* yet take a look at the
# :ref:`installation instructions <installation>` and make sure that all the requirements are fulfilled.
# Examples to illustrate the usage of *SMAC* - either by reading in a scenario file,
# or by directly using *SMAC* in Python - are provided in the examples-folder.
#
# To get started, we will walk you through a few examples.
#
# * First, we explain the basic usage of *SMAC* by optimizing the
# :ref:`Branin <branin-example>`-function as a toy example.
# * Second, we explain the usage of *SMAC* within Python by optimizing a :ref:`Support Vector Machine <svm-example>`.
# * Third, we show a real-world example, using an algorithm-wrapper to optimize the
# :ref:`SPEAR SAT-solver <spear-example>`.
#
# .. _TAE: ../../tae.html

###############################################################################
# .. _branin-example:
#
# Branin
# ======
# First of, we'll demonstrate the usage of *SMAC* on the minimization of
# `the standard 2-dimensional continuous test function Branin <https://www.sfu.ca/~ssurjano/branin.html>`_.
#

import numpy as np

def branin(x):
    x1 = x[0]
    x2 = x[1]
    a = 1.
    b = 5.1 / (4. * np.pi ** 2)
    c = 5. / np.pi
    r = 6.
    s = 10.
    t = 1. / (8. * np.pi)
    ret = a * (x2 - b * x1 ** 2 + c * x1 - r) ** 2 + s * (1 - t) * np.cos(x1) + s
    return ret

###############################################################################
# This example aims to explain the basic usage of *SMAC*. There are different ways to use *SMAC*:
#
# f_min-wrapper
# ~~~~~~~~~~~~~
# The easiest way to use *SMAC* is to use the :func:`f_min SMAC wrapper
# <smac.facade.func_facade.fmin_smac>`.
# We import the fmin-function and wrap it around are simple branin function.
#

from smac.facade.func_facade import fmin_smac

x, cost, _ = fmin_smac(func=branin,  # function
                       x0=[0, 0],  # default configuration
                       bounds=[(-5, 10), (0, 15)],  # limits
                       maxfun=10,  # maximum number of evaluations
                       rng=3)  # random seed
print("Optimum at {} with cost of {}".format(x, cost))

###############################################################################
# This way, you can optimize a blackbox-function with minimal effort. However, to
# use all of *SMAC's* capability, a :ref:`scenario` object should be used.
#
# Command line
# ~~~~~~~~~~~~
# A more evolved example can be found in ``examples/quickstart/branin/``.
# In this directory you can find:
#
# * ``branin.py``
# * ``cmdline_wrapper.py``
# * ``scenario.txt``
# * ``param_config_space.pcs``
#
# To run the example scenario, change into the root directory of *SMAC* and type the following commands:
#
#
# .. code-block:: bash
#
#     cd examples/quickstart/branin
#     python ../../scripts/smac --scenario scenario.txt
#
# The Python command runs *SMAC* with the specified scenario. The scenario file consists of the following lines:
#
# .. literalinclude:: ../../../examples/quickstart/branin/scenario.txt
#
# The **algo** parameter specifies how *SMAC* calls the target algorithm to be optimized.
# This is further explained in the chapter about the Target Algorithm Evaluator (:ref:`TAE <tae>`).
# An algorithm call by *SMAC* is of the following format:
#
#     .. code-block:: bash
#
#         <algo> <instance> <instance specific> <cutoff time> <runlength> <seed> <algorithm parameters>
#         python branin.py 0 0 999999999.0 0 1148756733 -x1 -1.1338595629 -x2 13.8770222718
#
# The **paramfile** parameter tells *SMAC* which Parameter Configuration Space
# :ref:`PCS <paramcs>`-file to use. This file contains a list of the algorithm's parameters,
# their domains and default values:
#
#     .. literalinclude:: ../../../examples/quickstart/branin/param_config_space.pcs
#
# ``x1`` and ``x2`` are both continuous parameters. ``x1`` can take any real value
# in the range ``[-5, 10]``, ``x2`` in the range ``[0, 15]`` and both have
# the default value ``0``.
#
# The **run_obj** parameter specifies what *SMAC* is supposed to **optimize**. Here we optimize solution quality.
# The **runcount_limit** specifies the maximum number of algorithm calls.
#
# *SMAC* reads the results from the command line output. The wrapper returns the
# results of the algorithm in a specific format:
#
#     .. code-block:: bash
#
#         Result for SMAC: <STATUS>, <running time>, <runlength>, <quality>, <seed>, <instance-specifics>
#         Result for SMAC: SUCCESS, 0, 0, 48.948190, 1148756733
#
#     | The second line is the result of the above listed algorithm call.
#     | *STATUS:* can be either *SUCCESS*, *CRASHED*, *SAT*, *UNSAT*, *TIMEOUT*
#     | *running time:* is the measured running time for an algorithm call
#     | *runlength:* is the number of steps needed to find a solution
#     | *quality:* the solution quality
#     | *seed:* the seed that was used with the algorithm call
#     | *instance-specifics:* additional information
#
# *SMAC* will terminate with the following output:
#
#     .. code-block:: bash
#
#         INFO:intensifier:Updated estimated error of incumbent on 122 runs: 0.5063
#         DEBUG:root:Remaining budget: inf (wallclock), inf (ta costs), -6.000000 (target runs)
#         INFO:Stats:##########################################################
#         INFO:Stats:Statistics:
#         INFO:Stats:#Target algorithm runs: 506
#         INFO:Stats:Used wallclock time: 44.00 sec
#         INFO:Stats:Used target algorithm runtime: 0.00 sec
#         INFO:Stats:##########################################################
#         INFO:SMAC:Final Incumbent: Configuration:
#           x1, Value: 9.556406137303922
#           x2, Value: 2.429138598022513
#
# Furthermore, *SMACs* trajectory and runhistory will be stored in ``branin/``.
#

###############################################################################
# .. _svm-example:
#
# Using *SMAC* in Python: SVM
# ---------------------------
# To explain the use of *SMAC* within Python, let's look at a real-world example,
# optimizing the hyperparameters of a Support Vector Machine (SVM) trained on the widely known `IRIS-dataset
# <https://en.wikipedia.org/wiki/Iris_flower_data_set>`_.
# This example is located in :code:`examples/general/svm.py`.
#
# To use *SMAC* directly with Python, we first import the necessary modules
#

from ConfigSpace.conditions import InCondition
from ConfigSpace.hyperparameters import CategoricalHyperparameter, \
    UniformFloatHyperparameter, UniformIntegerHyperparameter
from sklearn import svm, datasets
from sklearn.model_selection import cross_val_score

# Import ConfigSpace and different types of parameters
from smac.configspace import ConfigurationSpace
from smac.facade.smac_hpo_facade import SMAC4HPO
# Import SMAC-utilities
from smac.scenario.scenario import Scenario

# We load the iris-dataset (a widely used benchmark)
iris = datasets.load_iris()

###############################################################################
# Next we need a configuration space from which to sample from

# Build Configuration Space which defines all parameters and their ranges
cs = ConfigurationSpace()

# We define a few possible types of SVM-kernels and add them as "kernel" to our cs
kernel = CategoricalHyperparameter("kernel", ["linear", "rbf", "poly", "sigmoid"], default_value="poly")
cs.add_hyperparameter(kernel)

# There are some hyperparameters shared by all kernels
C = UniformFloatHyperparameter("C", 0.001, 1000.0, default_value=1.0)
shrinking = CategoricalHyperparameter("shrinking", ["true", "false"], default_value="true")
cs.add_hyperparameters([C, shrinking])

# Others are kernel-specific, so we can add conditions to limit the searchspace
degree = UniformIntegerHyperparameter("degree", 1, 5, default_value=3)  # Only used by kernel poly
coef0 = UniformFloatHyperparameter("coef0", 0.0, 10.0, default_value=0.0)  # poly, sigmoid
cs.add_hyperparameters([degree, coef0])
use_degree = InCondition(child=degree, parent=kernel, values=["poly"])
use_coef0 = InCondition(child=coef0, parent=kernel, values=["poly", "sigmoid"])
cs.add_conditions([use_degree, use_coef0])

# This also works for parameters that are a mix of categorical and values from a range of numbers
# For example, gamma can be either "auto" or a fixed float
gamma = CategoricalHyperparameter("gamma", ["auto", "value"], default_value="auto")  # only rbf, poly, sigmoid
gamma_value = UniformFloatHyperparameter("gamma_value", 0.0001, 8, default_value=1)
cs.add_hyperparameters([gamma, gamma_value])
# We only activate gamma_value if gamma is set to "value"
cs.add_condition(InCondition(child=gamma_value, parent=gamma, values=["value"]))
# And again we can restrict the use of gamma in general to the choice of the kernel
cs.add_condition(InCondition(child=gamma, parent=kernel, values=["rbf", "poly", "sigmoid"]))

print(cs)

###############################################################################
# Of course we also define a function to evaluate the configured SVM on the IRIS-dataset.
# Some options, such as the *kernel* or *C*, can be passed directly.
# Others, such as *gamma*, need to be translated before the call to the SVM.

def svm_from_cfg(cfg):
    """ Creates a SVM based on a configuration and evaluates it on the
    iris-dataset using cross-validation.

    Parameters:
    -----------
    cfg: Configuration (ConfigSpace.ConfigurationSpace.Configuration)
        Configuration containing the parameters.
        Configurations are indexable!

    Returns:
    --------
    A crossvalidated mean score for the svm on the loaded data-set.
    """
    # For deactivated parameters, the configuration stores None-values.
    # This is not accepted by the SVM, so we remove them.
    cfg = {k: cfg[k] for k in cfg if cfg[k]}
    # We translate boolean values:
    cfg["shrinking"] = True if cfg["shrinking"] == "true" else False
    # And for gamma, we set it to a fixed value or to "auto" (if used)
    if "gamma" in cfg:
        cfg["gamma"] = cfg["gamma_value"] if cfg["gamma"] == "value" else "auto"
        cfg.pop("gamma_value", None)  # Remove "gamma_value"

    clf = svm.SVC(**cfg, random_state=42)
    scores = cross_val_score(clf, iris.data, iris.target, cv=5)
    return 1 - np.mean(scores)  # Minimize!

def_value = svm_from_cfg(cs.get_default_configuration())
print("Default Value: %.2f" % (def_value))

###############################################################################
# We need a Scenario-object to configure the optimization process.
# We provide a :ref:`list of possible options <scenario>` in the scenario.
#
# The initialization of a scenario in the code uses the same keywords as a
# scenario-file, which we used in the Branin example.
#

scenario = Scenario({"run_obj": "quality",  # we optimize quality (alternatively runtime)
                     "runcount-limit": 5,  # max. number of function evaluations; for this example set to a low number
                     "cs": cs,  # configuration space
                     "deterministic": "true"
                     })

###############################################################################
# Now we're ready to create a *SMAC*-instance, which handles the Bayesian
# Optimization-loop and calculates the incumbent.
# To automatically handle the exploration of the search space
# and evaluation of the function, SMAC needs as inputs the scenario object
# as well as the function.
#
smac = SMAC4HPO(scenario=scenario, rng=np.random.RandomState(42),
                tae_runner=svm_from_cfg)

incumbent = smac.optimize()

inc_value = svm_from_cfg(incumbent)
print("Optimized Value: %.2f" % (inc_value))

###############################################################################
# Internally SMAC keeps track of the number of algorithm calls and the remaining time budget via a Stats object.
#
# After successful execution of the optimization loop the Stats object outputs the result of the loop.
#
# .. code-block:: bash
#
#     INFO:smac.stats.stats.Stats:##########################################################
#     INFO:smac.stats.stats.Stats:Statistics:
#     INFO:smac.stats.stats.Stats:#Incumbent changed: 88
#     INFO:smac.stats.stats.Stats:#Target algorithm runs: 200 / 200.0
#     INFO:smac.stats.stats.Stats:Used wallclock time: 41.73 / inf sec
#     INFO:smac.stats.stats.Stats:Used target algorithm runtime: 17.44 / inf sec
#     INFO:smac.stats.stats.Stats:##########################################################
#     INFO:smac.facade.smac_facade.SMAC:Final Incumbent: Configuration:
#       C, Value: 357.4171743725004
#       coef0, Value: 9.593372746957046
#       degree, Value: 1
#       gamma, Value: 'value'
#       gamma_value, Value: 0.0029046235175726105
#       kernel, Value: 'poly'
#       shrinking, Value: 'false'
#
# We further query the target function at the incumbent, using the function evaluator
# so that as final output we can see the error value of the incumbent.
#
# .. code-block:: bash
#
#    Optimized Value: 0.02
#
# As a bonus, we can validate our results. This is more useful when optimizing on
# instances, but we include the code so it is easily applicable for any usecase.
#
# We can also validate our results (though this makes a lot more sense with instances)

smac.validate(config_mode='inc',  # We can choose which configurations to evaluate
              # instance_mode='train+test',  # Defines what instances to validate
              repetitions=100,  # Ignored, unless you set "deterministic" to "false" in line 95
              n_jobs=1)  # How many cores to use in parallel for optimization

###############################################################################
# .. _spear-example:
#
# Spear-QCP
# ---------
# For this example we use *SMAC* to optimize the runtime required by the SAT solver
# `Spear <http://www.domagoj-babic.com/index.php/ResearchProjects/Spear>`_
# to solve a small subset of the QCP-dataset.
#
# In *SMACs* root-directory type:
#
# .. code-block:: bash
#
#     cd examples/quickstart/spear_qcp && ls -l
#
# In this folder you see the following files and directories:
#
# * **features.txt**:
#
#     The :ref:`feature file <feature>` contains the features for each instance in a csv-format.
#
#     +--------------------+--------------------+--------------------+-----+
#     |      instance      | name of feature 1  | name of feature 2  | ... |
#     +====================+====================+====================+=====+
#     | name of instance 1 | value of feature 1 | value of feature 2 | ... |
#     +--------------------+--------------------+--------------------+-----+
#     |         ...        |          ...       |          ...       | ... |
#     +--------------------+--------------------+--------------------+-----+
#
# * **instances.txt**
#   The ref:`instance` contains the names of all
#   instances one might want to consider during the optimization process.
#
# * **scenario.txt**
#   The :ref:`scenario <scenario>` file contains all the necessary information about the configuration scenario at hand.
#
#   .. literalinclude:: ../../../examples/quickstart/spear_qcp/scenario.txt
#
#   For this example the following options are used:
#
#   * *algo:*
#
#     .. code-block:: bash
#
#         python -u ./target_algorithm/scripts/SATCSSCWrapper.py \
#             --mem-limit 1024 \
#             --script ./target_algorithm/spear-python/spearCSSCWrapper.py
#
#     This specifies the wrapper that *SMAC* executes with a pre-specified
#     syntax in order to evaluate the algorithm to be optimized.
#     This wrapper script takes an instantiation of the parameters as input,
#     runs the algorithm with these parameters, and returns
#     the cost of running the algorithm; since every algorithm has a different
#     input and output format, this wrapper acts as an interface between the
#     algorithm and *SMAC*, which executes the wrapper through a command line call.
#
#     An example call would look something like this:
#
#     .. code-block:: bash
#
#         <algo> <instance> <instance specifics> <cutoff time> <runlength> <seed> <algorithm parameters>
#
#     For *SMAC* to be able to interpret the results of the algorithm run,
#     the wrapper returns the results of the algorithm run as follows:
#
#     .. code-block:: bash
#
#         STATUS, running time, runlength, quality, seed, instance-specifics
#
#   * *paramfile:*
#
#     This parameter specifies which pcs-file to use and where it is located.
#
#     The PCS-file specifies the Parameter Configuration Space file, which
#     lists the algorithm's parameters, their domains, and default values (one per line)
#
#     In this example we are dealing with 26 parameters of which 12 are categorical
#     and 14 are continuous. Out of these 26
#     parameters, 9 parameters are conditionals (they are only active if
#     their parent parameter takes on a certain value).
#
#   * *execdir:* Specifies the directory in which the target algorithm will be run.
#
#   * *deterministic:* Specifies if the target algorithm is deterministic.
#
#   * *run_obj:* This parameter tells *SMAC* what is to be optimized, i.e. running time or (solution) quality.
#
#   * *overall_obj:* Specifies how to evaluate the error values, e.g as mean or PARX.
#
#   * *cutoff_time:* The target algorithms cutoff time.
#
#   * *wallclock-limit:* This parameter is used to give the time budget for the configuration task in seconds.
#
#   * *instance_file:* See instances.txt above.
#
#   * *feature_file:* See features.txt above.
#
# * **run.sh**
#   A shell script calling *SMAC* with the following command:
#
#   .. code-block:: bash
#
#       python ../../scripts/smac --scenario scenario.txt --verbose DEBUG``
#
#   This runs *SMAC* with the scenario options specified in the scenario.txt file.
#
# * **target_algorithms** contains the wrapper and the executable for Spear.
#
# * **instances** folder contains the instances on which *SMAC* will configure Spear.
#
# To run the example type one of the two commands below into a terminal:
#
# .. code-block:: bash
#
#     bash run.sh
#     python ../../scripts/smac --scenario scenario.txt
#
# *SMAC* will run for a few seconds and generate a lot of logging output.
# After *SMAC* finished the configuration process you'll get some final statistics about the configuration process:
#
# .. code-block:: bash
#
#    INFO:    ##########################################################
#    INFO:    Statistics:
#    INFO:    #Incumbent changed: 2
#    INFO:    #Target algorithm runs: 17 / inf
#    INFO:    Used wallclock time: 35.38 / 30.00 sec
#    INFO:    Used target algorithm runtime: 20.10 / inf sec
#    INFO:    ##########################################################
#    INFO:    Final Incumbent: Configuration:
#      sp-clause-activity-inc, Value: 0.9846527087294622
#      sp-clause-decay, Value: 1.1630090545101102
#      sp-clause-del-heur, Value: '0'
#      sp-first-restart, Value: 30
#      sp-learned-clause-sort-heur, Value: '10'
#      sp-learned-clauses-inc, Value: 1.2739749314202675
#      sp-learned-size-factor, Value: 0.8355014264152971
#      sp-orig-clause-sort-heur, Value: '1'
#      sp-phase-dec-heur, Value: '0'
#      sp-rand-phase-dec-freq, Value: '0.01'
#      sp-rand-phase-scaling, Value: 0.3488055907688382
#      sp-rand-var-dec-freq, Value: '0.05'
#      sp-rand-var-dec-scaling, Value: 0.46427056372562864
#      sp-resolution, Value: '0'
#      sp-restart-inc, Value: 1.7510945705535836
#      sp-update-dec-queue, Value: '1'
#      sp-use-pure-literal-rule, Value: '0'
#      sp-var-activity-inc, Value: 0.9000377944957962
#      sp-var-dec-heur, Value: '14'
#      sp-variable-decay, Value: 1.8292433459523076
#
#
#
# The statistics further show the used wallclock time, target algorithm running
# time and the number of executed target algorithm runs,
# and the corresponding budgets---here we exhausted the wallclock time budget.
#
# The directory in which you invoked *SMAC* now contains a new folder called **SMAC3-output_YYYY-MM-DD_HH:MM:SS**.
# The .json file contains the information about the target algorithms *SMAC* just
# executed. In this file you can see the *status* of the algorithm run, *misc*,
# the *instance* on which the algorithm was evaluated, which *seed* was used,
# how much *time* the algorithm needed and with which *configuration* the algorithm
# was run.
# In the folder *SMAC* generates a file for the runhistory, and two files for the trajectory.
#
#
# .. _hydra-example:
#
# Hydra on Spear-QCP
# ------------------
#
# For this example we use *Hydra* to build a portfolio on the same example data
# presented in :ref:`Spear-QCP <spear-example>`
# Hydra is a portfolio builder that aims to build a portfolio by iteratively adding complementary configurations to the
# already existing portfolio. To select these complementary configurations *Hydra* compares new configurations to the
# portfolio and only considers configurations that improve the portfolio performance.
# In the first iteration Hydra runs standard *SMAC* to determine a well performing configuration
# across all instances as a starting point for the portfolio. In following iterations
# *Hydra* adds one configuration that
# improves the portfolio performance.
#
# To run Hydra for three iterations you can run the following code in the spear-qcp example folder.
#
# .. code-block:: bash
#
#       python ../../scripts/smac --scenario scenario.txt --verbose DEBUG --mode Hydra --hydra_iterations 3
#
# As the individual SMAC scenario takes 30 seconds to run Hydra will run for ~90 seconds on this example.
# You will see the same output to the terminal as with standard SMAC. In the folder
# where you executed the above command,
# you will find a *hydra-output-yyy-mm-dd_hh:mm:ss_xyz* folder. This folder contains the results of all three performed
# SMAC runs, as well as the resulting portfolio (as pkl file).
#
# The resulting portfolio can be used with any algorithm selector such as
# `AutoFolio <https://github.com/mlindauer/AutoFolio>`_


text:
#This file will generate functions in polynomials
import numpy as np
import random
import matplotlib.pyplot as plt

class generateFunctions():
	#the initial function taking 4 inputs
	def __init__(self, x_vector, high_degree_vector, rangeLow, rangeHigh):
		#the input processing
		self.x_vector = x_vector
		self.high_degree_vector = high_degree_vector
		self.rangeLow = rangeLow
		self.rangeHigh = rangeHigh

		self.functionString = ""

	def generate(self):
		#allowed values for the highest degree and others can be zeros
		allowed_values = list(range(self.rangeLow,self.rangeHigh))
		allowed_values.remove(0)
		for i in range(len(self.x_vector)):
			highestVar = self.high_degree_vector[i]
			ppar = np.random.randint(low=self.rangeLow,high=self.rangeHigh,size=(highestVar+1))
			#make sure the highest is not zero coefficient
			if ppar[0] == 0:
				ppar[0] = random.choice(allowed_values)
			for j in range(len(ppar)):
				add = ""
				if ppar[j] != 0:
					add = str(ppar[j])
					if (highestVar-j) != 0:
						add = add +"*"+self.x_vector[i]
						if(highestVar-j)!=1:
							add = add +"^"+str(highestVar-j)
					if ppar[j] > 0:
						add = "+" + add
				self.functionString = self.functionString + add
		return self.functionString
#p = generateFunctions()
#function = p.generate()
#print(function)


text:
import tensorflow as tf
import numpy as np
import GGIPNN as GGIPNN
import GGIPNN_util as NN_util
import random
import os
import time
import datetime
from sklearn import metrics

# Parameters
# ==================================================

# Data loading params
tf.flags.DEFINE_string("embedding_file", "../pre_trained_emb/gene2vec_dim_200_iter_9.txt", "embedding file address, matrix txt file")
# Model Hyperparameters
tf.flags.DEFINE_float("l2_reg_lambda", 0, "L2 regularization lambda (default: 0.0)")
tf.flags.DEFINE_integer("embedding_dimension", 200, "embedding dimension")
tf.flags.DEFINE_float("dropout_keep_prob", 0.5, "Dropout keep probability ")
# Training parameters
tf.flags.DEFINE_integer("batch_size", 128, "Batch Size (default: 64)")
tf.flags.DEFINE_integer("num_epochs", 1, "Number of training epochs ")
tf.flags.DEFINE_integer("evaluate_every", 200, "Evaluate model on dev set after this many steps")
tf.flags.DEFINE_integer("checkpoint_every", 1000, "Save model after this many steps (default: 100)")
tf.flags.DEFINE_integer("num_checkpoints", 5, "Number of checkpoints to store (default: 5)")
# Misc Parameters
tf.flags.DEFINE_boolean("allow_soft_placement", True, "Allow device soft device placement")
tf.flags.DEFINE_boolean("log_device_placement", False, "Log placement of ops on devices")
tf.flags.DEFINE_boolean("use_pre_trained_gene2vec", True, "use_pre_trained_gene2vec") # if False, the embedding layer will be initialized randomly
tf.flags.DEFINE_boolean("train_embedding", False, "train_embedding") # if True, the embedding layer will be trained during the training

FLAGS = tf.app.flags.FLAGS
# FLAGS._parse_flags()
print("\nParameters:")
for attr, value in sorted(FLAGS.__flags.items()):
    print("{}={}".format(attr.upper(), value.value))
print("")

# Data loading data
x_train_raw_f = open("../predictionData/train_text.txt", 'r')
x_train_raw = x_train_raw_f.read().splitlines()
x_train_raw_f.close()
y_train_raw_f = open("../predictionData/train_label.txt", 'r')
y_train_raw = y_train_raw_f.read().splitlines()
y_train_raw_f.close()

x_test_raw_f = open("../predictionData/test_text.txt", 'r')
x_test_raw = x_test_raw_f.read().splitlines()
x_test_raw_f.close()
y_test_raw_f = open("../predictionData/test_label.txt", 'r')
y_test_raw = y_test_raw_f.read().splitlines()
y_test_raw_f.close()

x_valid_raw_f = open("../predictionData/valid_text.txt", 'r')
x_valid_raw = x_valid_raw_f.read().splitlines()
x_valid_raw_f.close()
y_valid_raw_f = open("../predictionData/valid_label.txt", 'r')
y_valid_raw = y_valid_raw_f.read().splitlines()
y_valid_raw_f.close()

all_text_voca = NN_util.myFitDict(x_train_raw+x_valid_raw+x_test_raw,2)
all_text_unshuffled = NN_util.myFit(x_train_raw+x_valid_raw+x_test_raw,2, all_text_voca)

x_train_len = len(x_train_raw)
x_valid_len = len(x_valid_raw)
x_test_len = len(x_test_raw)

x_train_unshuffled= all_text_unshuffled[:x_train_len]
x_valid_unshuffled= all_text_unshuffled[x_train_len:x_train_len+x_valid_len]
x_test_unshuffled= all_text_unshuffled[x_train_len+x_valid_len:]

total = x_train_raw+x_valid_raw+x_test_raw

# Randomly shuffle training data
random_indices = list(range(len(x_train_raw)))
random.shuffle(random_indices)
x_train = x_train_unshuffled[random_indices]
y_onehot = NN_util.oneHot(y_train_raw+y_valid_raw+y_test_raw)

y_train_onehot = y_onehot[:x_train_len]
y_train = y_train_onehot[random_indices]

x_dev = x_valid_unshuffled
y_dev = y_onehot[x_train_len:x_train_len+x_valid_len]

x_test = x_test_unshuffled
y_test = y_onehot[x_train_len+x_valid_len:]

all_predictions = []
all_score = []
all_y2 = []

predicationGS = y_test

print("total training size: " +str(len(y_train)))

print("total test size: " +str(len(y_test)))

print( "training start!")
print("Vocabulary Size: {:d}".format(len(all_text_voca)))
print("Train/Test split: {:d}/{:d}".format(len(y_train), len(y_test)))
print("GS size " + str(len(predicationGS)))

# Training
# ==================================================
graph = tf.Graph()
with graph.as_default():
    session_conf = tf.ConfigProto(
      allow_soft_placement=FLAGS.allow_soft_placement,
      log_device_placement=FLAGS.log_device_placement)
    sess = tf.Session(config=session_conf)
    with sess.as_default():
        cnn = GGIPNN.GGIPNN(
            sequence_length=x_train.shape[1],
            num_classes=y_train.shape[1],
            vocab_size=len(all_text_voca),
            embedTrain=FLAGS.train_embedding,
            embedding_size = FLAGS.embedding_dimension,
            l2_lambda=FLAGS.l2_reg_lambda
        )

        # Define Training procedure
        global_step = tf.Variable(0, name="global_step", trainable=False)
        optimizer = tf.train.AdamOptimizer(1e-3)
        grads_and_vars = optimizer.compute_gradients(cnn.loss)
        train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step)

        # Keep track of gradient values and sparsity (optional)
        grad_summaries = []
        for g, v in grads_and_vars:
            if g is not None:
                grad_hist_summary = tf.summary.histogram("{}/grad/hist".format(v.name), g)
                sparsity_summary = tf.summary.scalar("{}/grad/sparsity".format(v.name), tf.nn.zero_fraction(g))
                grad_summaries.append(grad_hist_summary)
                grad_summaries.append(sparsity_summary)
        grad_summaries_merged = tf.summary.merge(grad_summaries)

        # Output directory for models and summaries
        timestamp = str(int(time.time()))
        out_dir = os.path.abspath(os.path.join(os.path.curdir, "../runs", timestamp))
        print("Writing to {}\n".format(out_dir))

        # Summaries for loss and accuracy
        loss_summary = tf.summary.scalar("loss", cnn.loss)
        acc_summary = tf.summary.scalar("accuracy", cnn.accuracy)

        # Train Summaries
        train_summary_op = tf.summary.merge([loss_summary, acc_summary, grad_summaries_merged])
        train_summary_dir = os.path.join(out_dir, "summaries", "train")
        train_summary_writer = tf.summary.FileWriter(train_summary_dir, sess.graph)

        # Dev summaries
        dev_summary_op = tf.summary.merge([loss_summary, acc_summary])
        dev_summary_dir = os.path.join(out_dir, "summaries", "dev")
        dev_summary_writer = tf.summary.FileWriter(dev_summary_dir, sess.graph)

        # Checkpoint directory. Tensorflow assumes this directory already exists so we need to create it
        checkpoint_dir = os.path.abspath(os.path.join(out_dir, "checkpoints"))
        checkpoint_prefix = os.path.join(checkpoint_dir, "model")
        if not os.path.exists(checkpoint_dir):
            os.makedirs(checkpoint_dir)
        saver = tf.train.Saver(tf.global_variables(), max_to_keep=FLAGS.num_checkpoints)

        # Initialize all variables
        sess.run(tf.global_variables_initializer())
        if FLAGS.use_pre_trained_gene2vec:
            vocabulary = all_text_voca
            initW = None
            initW = NN_util.load_embedding_vectors(vocabulary,FLAGS.embedding_file,FLAGS.embedding_dimension)
            print("gene embedding file has been loaded\n")
            sess.run(cnn.W.assign(initW))
        else:
            print("embedding loading errors")
        def train_step(x_batch, y_batch):
            """
            A single training step
            """
            feed_dict = {
                cnn.input_x: x_batch,
                cnn.input_y: y_batch,
                cnn.dropout_keep_prob: FLAGS.dropout_keep_prob
            }
            _, step, summaries, loss, accuracy = sess.run(
                [train_op, global_step, train_summary_op, cnn.loss, cnn.accuracy],
                feed_dict)
            # time_str = datetime.datetime.now().isoformat()
            # print("{}: step {}, loss {:g}, acc {:g}".format(time_str, step, loss, accuracy))
            train_summary_writer.add_summary(summaries, step)

        def dev_step(x_batch, y_batch, writer=None):
            """
            Evaluates model on a dev set
            """
            feed_dict = {
                cnn.input_x: x_batch,
                cnn.input_y: y_batch,
                cnn.dropout_keep_prob: 1.0
            }
            step, summaries, loss, accuracy = sess.run(
                [global_step, dev_summary_op, cnn.loss, cnn.accuracy],
                feed_dict)
            time_str = datetime.datetime.now().isoformat()
            print("{}: step {}, loss {:g}, acc {:g}".format(time_str, step, loss, accuracy))
            if writer:
                writer.add_summary(summaries, step)

        # Generate batches
        batches = NN_util.batch_iter(
            list(zip(x_train, y_train)), FLAGS.batch_size, FLAGS.num_epochs)
        # Training loop. For each batch...
        for batch in batches:
            x_batch, y_batch = zip(*batch)
            train_step(x_batch, y_batch)
            current_step = tf.train.global_step(sess, global_step)
            if current_step % FLAGS.evaluate_every == 0:
                # print("\nEvaluation:")
                dev_step(x_dev, y_dev, writer=dev_summary_writer)
                # print("")
            if current_step % FLAGS.checkpoint_every == 0:
                path = saver.save(sess, checkpoint_prefix, global_step=current_step)
                # print("Saved model checkpoint to {}\n".format(path))

        input_x = graph.get_operation_by_name("input_x").outputs[0]
        dropout_keep_prob = graph.get_operation_by_name("dropout_keep_prob").outputs[0]
        # input_y = graph.get_operation_by_name("input_y").outputs[0]

        # Tensors we want to evaluate
        predictions_score = graph.get_operation_by_name("output/softmax_scores").outputs[0]
        predictions = graph.get_operation_by_name("output/predictions").outputs[0]
        y2 = graph.get_operation_by_name("embedding/embedchar").outputs[0]

        # Generate batches for one epoch
        batches = NN_util.batch_iter(list(x_test), FLAGS.batch_size, 1, shuffle=False)

        # Collect the predictions here

        for x_test_batch in batches:
            batch_score = sess.run(predictions_score, {input_x: x_test_batch, dropout_keep_prob: 1.0})
            all_score.extend(batch_score)
            batch_predictions = sess.run(predictions, {input_x: x_test_batch, dropout_keep_prob: 1.0})
            all_predictions.extend(batch_predictions)
            y2score = sess.run(y2, {input_x: x_test_batch, dropout_keep_prob: 1.0})
            all_y2.extend(np.transpose(y2score))

predicationGS = np.argmax(predicationGS, axis=1)

predictions_score_human_readable= np.column_stack(np.transpose(all_score))
yscore = predictions_score_human_readable[:,1]

ytrue=predicationGS
print("-------------------")
print("AUC score")
print(metrics.roc_auc_score(np.array(ytrue), np.array(yscore)))


text:
import numpy as np
import PIL.Image
import cv2, pickle, sys, os
from tqdm import trange
sys.path.append("/home/zhangtianyi/github/hmd/src/")
from predictor import joint_predictor
from predictor import anchor_predictor
from data_loader import dataloader_pred
from mesh_edit import fast_deform_dja
from mesh_edit import fast_deform_dsa
from renderer import SMPLRenderer
from utility import sil_iou
from utility import show_img_arr
from utility import center_crop
from utility import get_anchor_posi
from utility import get_joint_posi

# pdt_j = joint_predictor("../demo/pretrained_model/pretrained_joint.pth")
# pdt_a = anchor_predictor("../demo/pretrained_model/pretrained_anchor.pth")
pdt_j = joint_predictor("/home/zhangtianyi/github/hmd/model/snapshots_j_2019-11-18-11:03:06/pretrained_joint.pth")
pdt_a = anchor_predictor("/home/zhangtianyi/github/hmd/model/snapshots_a_2019-11-18-17:46:53/anchor_epoch_0_0.pth")

dataset = dataloader_pred(dataset_path = "./eval_data/wild_set/",
                          train = False, shuffle = False)
my_renderer = SMPLRenderer(face_path="../predef/smpl_faces.npy")

sta_num = 2699

# make verts for joint deform
with open ('../predef/mesh_joint_list.pkl', 'rb') as fp:
    item_dic = pickle.load(fp)
point_list = item_dic["point_list"]
index_map = item_dic["index_map"]

# make verts for anchor deform
with open ('../predef/dsa_achr.pkl', 'rb') as fp:
    dic_achr = pickle.load(fp)
achr_id = dic_achr['achr_id']
achr_num = len(achr_id)
ori_iou_list = []
ja_iou_list = []
sa_iou_list = []
err_j_hmr_list = []
err_j_ja_list = []
err_j_sa_list = []

tr = trange(sta_num, desc='Bar desc', leave=True)
for test_num in tr:

    test_tuple = dataset[test_num]
    src_j = test_tuple[0]
    #src_a = test_tuple[1]
    src_img = test_tuple[2]
    verts = test_tuple[5]
    vert_norms = test_tuple[6]
    proc_para = test_tuple[7]
    all_sil = np.array(test_tuple[8])
    proj_sil = all_sil[:, :, 1]
    gt_sil = all_sil[:, :, 0]

    # for test joint
    joint_posi = np.array(test_tuple[9])
    joint_move = np.resize(np.array(test_tuple[3]), (10, 2))
    j_posi_gt = joint_posi+joint_move

    # predict joint
    joint_tsr = pdt_j.predict_batch(src_j)
    joint_para = np.array(joint_tsr.data.cpu())
    joint_para = np.concatenate((joint_para, np.zeros((10,1))),axis = 1)

    # apply scale
    joint_para = joint_para * 0.007# 0.007

    flat_point_list = [item for sublist in point_list for item in sublist]

    num_mj = len(point_list)
    j_list = []
    for i in range(num_mj):
        j_p_list = []
        for j in range(len(point_list[i])):
            j_p_list.append(verts[point_list[i][j]])
        j_list.append(sum(j_p_list)/len(j_p_list))

    new_jv = []
    ori_jv = []
    for i in range(len(j_list)):
        # make new joint verts
        for j in point_list[i]:
            new_jv.append(verts[j] + joint_para[i])
            ori_jv.append(verts[j])
    new_jv = np.array(new_jv)
    ori_jv = np.array(ori_jv)

    # joint deform
    fd_ja = fast_deform_dja(weight = 10.0)
    ja_verts = fd_ja.deform(np.asarray(verts), new_jv)

    # make src_a
    proj_sil_j = my_renderer.silhouette(verts = ja_verts)
    src_sil_j = np.zeros((224, 224, 2))
    src_a = np.zeros((200, 4, 32, 32))

    # make anchor posi
    anchor_verts = np.zeros((200, 3))
    for i in range(achr_num):
        anchor_verts[i, :] = ja_verts[achr_id[i], :]
    achr_posi = get_anchor_posi(anchor_verts)

    for i in range(len(achr_posi)):
        crop_sil = center_crop(proj_sil_j, achr_posi[i], 32)
        crop_img = center_crop(src_img, achr_posi[i], 32)
        crop_img = crop_img.astype(np.int)
        crop_img = crop_img - crop_img[15, 15, :]
        crop_img = np.absolute(crop_img)
        crop_img = crop_img.astype(np.float)/255.0
        src_a[i,0,:,:] = crop_sil
        src_a[i,1:4,:,:] = np.rollaxis(crop_img, 2, 0)

    # predict anchor
    achr_tsr = pdt_a.predict_batch(src_a)
    achr_para = np.array(achr_tsr.data.cpu())
    achr_para = achr_para * 0.003

    ori_av = []
    new_av = []
    for j in range(achr_num):
        ori_av.append(ja_verts[achr_id[j]])
        new_av.append(ja_verts[achr_id[j]] +
                      vert_norms[achr_id[j]] * achr_para[j])
    ori_av = np.array(ori_av)
    new_av = np.array(new_av)

    # build active list of anchor
    contours, _ = cv2.findContours(proj_sil_j, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
    cim = np.zeros_like(proj_sil_j)
    cv2.drawContours(cim, contours, -1, 255, 1)
    cim = cv2.dilate(cim, kernel=np.ones((6, 6)))
    active_index = np.ones(len(achr_posi))
    for j in range(len(achr_posi)):
        ay = np.int(np.round(achr_posi[j][1]))
        ax = np.int(np.round(achr_posi[j][0]))
        if cim[ay, ax] == 0:
            active_index[j] = 0

    # anchor deform
    fd_sa = fast_deform_dsa(weight=1.0)
    sa_verts = fd_sa.deform(np.asarray(ja_verts),
                            new_av,
                            active_index = active_index,
                           )

    # get sil after anchor deform
    sa_sil = my_renderer.silhouette(verts = sa_verts)

    # compute IoU and compare
    ori_iou = sil_iou(proj_sil, gt_sil)
    ja_iou = sil_iou(proj_sil_j, gt_sil)
    sa_iou = sil_iou(sa_sil, gt_sil)

    ori_iou_list.append(ori_iou)
    ja_iou_list.append(ja_iou)
    sa_iou_list.append(sa_iou)

    # compute error of joint
    j_posi_sa = get_joint_posi(sa_verts)
    err_j_sa = np.mean([np.linalg.norm(j_posi_sa[i]-j_posi_gt[i]) for i in range(10)])
    err_j_sa_list.append(err_j_sa)

    j_posi_ja = get_joint_posi(ja_verts)
    err_j_ja = np.mean([np.linalg.norm(j_posi_ja[i]-j_posi_gt[i]) for i in range(10)])
    err_j_ja_list.append(err_j_ja)

    j_posi_hmr = get_joint_posi(verts)
    err_j_hmr = np.mean([np.linalg.norm(j_posi_hmr[i]-j_posi_gt[i]) for i in range(10)])
    err_j_hmr_list.append(err_j_hmr)

# save eval values
output_dir = "./eval_report/"
if not os.path.exists(output_dir):
    os.makedirs(output_dir)

with open(output_dir + "wild_report.txt", "w") as f_txt:
    f_txt.write("# test_index, hmr_iou, hmd_j_iou, hmd_a(v)_iou, \r\n" +
                "# hmr_2d_joint_err, hmd_j_2d_joint_err, " +
                "hmd_a(v)_2d_joint_err\r\n")
    for i in range(sta_num):
        f_txt.write("%04d %f %f %f %f %f %f\r\n" \
                    % (i, ori_iou_list[i], ja_iou_list[i], sa_iou_list[i],
                       err_j_hmr_list[i], err_j_ja_list[i], err_j_sa_list[i]))
    f_txt.write("mean %f %f %f %f %f %f\r\n" \
                % (np.mean(ori_iou_list[:sta_num]),
                   np.mean(ja_iou_list[:sta_num]),
                   np.mean(sa_iou_list[:sta_num]),
                   np.mean(err_j_hmr_list[:sta_num]),
                   np.mean(err_j_ja_list[:sta_num]),
                   np.mean(err_j_sa_list[:sta_num])))

# print results
print("IoU - hmr: %f, j: %f, a: %f" % (np.mean(ori_iou_list[:sta_num]),
                                       np.mean(ja_iou_list[:sta_num]),
                                       np.mean(sa_iou_list[:sta_num])))
print("Joint - hmr: %f, j: %f, a: %f" % (np.mean(err_j_hmr_list[:sta_num]),
                                         np.mean(err_j_ja_list[:sta_num]),
                                         np.mean(err_j_sa_list[:sta_num])))


text:
# -*- coding: utf-8 -*-
"""
Authors: Tim Hessels and Gonzalo Espinoza
         UNESCO-IHE 2016
Contact: t.hessels@unesco-ihe.org
         g.espinoza@unesco-ihe.org
Repository: https://github.com/wateraccounting/watools
Module: Collect/ETmonitor

Restrictions:
The data and this python file may not be distributed to others without
permission of the WA+ team due data restriction of the ALEXI developers.

Description:
This script collects ALEXI data from the UNESCO-IHE FTP server. The data has a
monthly temporal resolution and a spatial resolution of 0.05 degree. The
resulting tiff files are in the WGS84 projection.
The data is available between 2003-01-01 till 2014-12-31.

Example:
from watools.Collect import ALEXI
ALEXI.monthly(Dir='C:/Temp/', Startdate='2003-02-24', Enddate='2003-03-09',
                     latlim=[50,54], lonlim=[3,7])

"""
from __future__ import print_function
# General modules
from builtins import str
import numpy as np
import os
import pandas as pd
from ftplib import FTP
import datetime
import math
import glob

# Water Accounting Modules
import watools.WebAccounts as WebAccounts
import watools.General.raster_conversions as RC
import watools.General.data_conversions as DC

def DownloadData(Dir, Startdate, Enddate, latlim, lonlim, TimeStep, Waitbar):

    """
    This scripts downloads ALEXI ET data from the UNESCO-IHE ftp server.
    The output files display the total ET in mm for a period of one week.
    The name of the file corresponds to the first day of the week.

    Keyword arguments:
    Dir -- 'C:/file/to/path/'
    Startdate -- 'yyyy-mm-dd'
    Enddate -- 'yyyy-mm-dd'
    TimeStep -- 'daily' or 'weekly'  (by using here monthly, an older dataset will be used)
    lonlim -- [ymin, ymax] (values must be between -60 and 70)
    latlim -- [xmin, xmax] (values must be between -180 and 180)
    """
    # Check the latitude and longitude and otherwise set lat or lon on greatest extent
    if latlim[0] < -60 or latlim[1] > 70:
        print('Latitude above 70N or below 60S is not possible. Value set to maximum')
        latlim[0] = np.max(latlim[0],-60)
        latlim[1] = np.min(latlim[1],70)
    if lonlim[0] < -180 or lonlim[1] > 180:
        print ('Longitude must be between 180E and 180W. Now value is set to maximum')
        lonlim[0] = np.max(lonlim[0],-180)
        lonlim[1] = np.min(lonlim[1],180)

	# Check Startdate and Enddate
    if not Startdate:
        if TimeStep == 'weekly':
            Startdate = pd.Timestamp('2003-01-01')
        if TimeStep == 'daily':
            Startdate = pd.Timestamp('2005-01-01')
    if not Enddate:
        if TimeStep == 'weekly':
            Enddate = pd.Timestamp('2015-12-31')
        if TimeStep == 'daily':
            Enddate = pd.Timestamp('2016-12-31')

	# Make a panda timestamp of the date
    try:
        Enddate = pd.Timestamp(Enddate)
    except:
        Enddate = Enddate

    if TimeStep == 'weekly':

        # Define the Startdate of ALEXI
        DOY = datetime.datetime.strptime(Startdate,
                                         '%Y-%m-%d').timetuple().tm_yday
        Year = datetime.datetime.strptime(Startdate,
                                          '%Y-%m-%d').timetuple().tm_year

      	# Change the startdate so it includes an ALEXI date
        DOYstart = int(math.ceil(DOY/7.0)*7+1)
        DOYstart = str('%s-%s' %(DOYstart, Year))
        Day = datetime.datetime.strptime(DOYstart, '%j-%Y')
        Month = '%02d' % Day.month
        Day = '%02d' % Day.day
        Date = (str(Year) + '-' + str(Month) + '-' + str(Day))
        DOY = datetime.datetime.strptime(Date,
                                         '%Y-%m-%d').timetuple().tm_yday
        # The new Startdate
        Date = pd.Timestamp(Date)

        # amount of Dates weekly
        Dates = pd.date_range(Date, Enddate, freq = '7D')

        # Define directory and create it if not exists
        output_folder = os.path.join(Dir, 'Evaporation', 'ALEXI', 'Weekly')
        if not os.path.exists(output_folder):
            os.makedirs(output_folder)

    if TimeStep == 'daily':

        # Define Dates
        Dates = pd.date_range(Startdate, Enddate, freq = 'D')

        # Define directory and create it if not exists
        output_folder = os.path.join(Dir, 'Evaporation', 'ALEXI', 'Daily')
        if not os.path.exists(output_folder):
            os.makedirs(output_folder)

    # Create Waitbar
    total_amount = len(Dates)
    if Waitbar == 1:
        import watools.Functions.Start.WaitbarConsole as WaitbarConsole
        amount = 0
        WaitbarConsole.printWaitBar(amount, total_amount, prefix = 'Progress:', suffix = 'Complete', length = 50)

    if TimeStep == 'weekly':
        ALEXI_weekly(Date, Enddate, output_folder, latlim, lonlim, Year, Waitbar, total_amount, TimeStep)

    if TimeStep == 'daily':
        ALEXI_daily(Dates, output_folder, latlim, lonlim, Waitbar, total_amount, TimeStep)

def ALEXI_weekly(Date, Enddate, output_folder, latlim, lonlim, Year, Waitbar, total_amount, TimeStep):

    # Define the stop conditions
    Stop = Enddate.toordinal()
    End_date=0
    amount = 0
    while End_date == 0:

        # Date as printed in filename
        Datesname=Date+pd.DateOffset(days=-7)
        DirFile= os.path.join(output_folder,'ETa_ALEXI_CSFR_mm-week-1_weekly_%s.%02s.%02s.tif' %(Datesname.strftime('%Y'), Datesname.strftime('%m'), Datesname.strftime('%d')))

        # Define end filename
        filename = "ALEXI_weekly_mm_%s_%s.tif" %(Date.strftime('%j'), Date.strftime('%Y'))

		 # Temporary filename for the downloaded global file
        local_filename = os.path.join(output_folder, filename)

	    # Create the new date for the next download
        Datename = (str(Date.strftime('%Y')) + '-' + str(Date.strftime('%m')) + '-' + str(Date.strftime('%d')))

        # Define IDs
        yID = 3000 - np.int16(np.array([np.ceil((latlim[1]+60)*20),np.floor((latlim[0]+60)*20)]))
        xID = np.int16(np.array([np.floor((lonlim[0])*20),np.ceil((lonlim[1])*20)])+3600)

        # Download the data from FTP server if the file not exists
        if not os.path.exists(DirFile):
            try:
                Download_ALEXI_from_WA_FTP(local_filename, DirFile, filename, lonlim, latlim, yID, xID, TimeStep)
            except:
                print("Was not able to download file with date %s" %Date)

        # Current DOY
        DOY = datetime.datetime.strptime(Datename,
                                     '%Y-%m-%d').timetuple().tm_yday

        # Define next day
        DOY_next = int(DOY + 7)
        if DOY_next >= 366:
            DOY_next = 8
            Year += 1
        DOYnext = str('%s-%s' %(DOY_next, Year))
        DayNext = datetime.datetime.strptime(DOYnext, '%j-%Y')
        Month = '%02d' % DayNext.month
        Day = '%02d' % DayNext.day
        Date = (str(Year) + '-' + str(Month) + '-' + str(Day))

        # Adjust waitbar
        if Waitbar == 1:
            import watools.Functions.Start.WaitbarConsole as WaitbarConsole
            amount += 1
            WaitbarConsole.printWaitBar(amount, total_amount, prefix = 'Progress:', suffix = 'Complete', length = 50)

        # Check if this file must be downloaded
        Date = pd.Timestamp(Date)
        if Date.toordinal() > Stop:
            End_date = 1

def ALEXI_daily(Dates, output_folder, latlim, lonlim, Waitbar, total_amount, TimeStep):

    amount = 0
    for Date in Dates:

        # Date as printed in filename
        DirFile= os.path.join(output_folder,'ETa_ALEXI_CSFR_mm-day-1_daily_%d.%02d.%02d.tif' %(Date.year, Date.month, Date.day))
        DOY = Date.timetuple().tm_yday

        # Define end filename
        filename = "EDAY_CERES_%d%03d.dat.gz" %(Date.year, DOY)

		 # Temporary filename for the downloaded global file
        local_filename = os.path.join(output_folder, filename)

        # Define IDs
        yID = 3000 - np.int16(np.array([np.ceil((latlim[1]+60)*20),np.floor((latlim[0]+60)*20)]))
        xID = np.int16(np.array([np.floor((lonlim[0])*20),np.ceil((lonlim[1])*20)])+3600)

        # Download the data from FTP server if the file not exists
        if not os.path.exists(DirFile):
            try:
                Download_ALEXI_from_WA_FTP(local_filename, DirFile, filename, lonlim, latlim, yID, xID, TimeStep)
            except:
                print("Was not able to download file with date %s" %Date)

        # Adjust waitbar
        if Waitbar == 1:
            import watools.Functions.Start.WaitbarConsole as WaitbarConsole
            amount += 1
            WaitbarConsole.printWaitBar(amount, total_amount, prefix = 'Progress:', suffix = 'Complete', length = 50)

    os.chdir(output_folder)
    re = glob.glob("*.dat")
    for f in re:
        os.remove(os.path.join(output_folder, f))

def Download_ALEXI_from_WA_FTP(local_filename, DirFile, filename, lonlim, latlim, yID, xID, TimeStep):
    """
    This function retrieves ALEXI data for a given date from the
    ftp.wateraccounting.unesco-ihe.org server.

    Restrictions:
    The data and this python file may not be distributed to others without
    permission of the WA+ team due data restriction of the ALEXI developers.

    Keyword arguments:
	local_filename -- name of the temporary file which contains global ALEXI data
    DirFile -- name of the end file with the weekly ALEXI data
    filename -- name of the end file
    lonlim -- [ymin, ymax] (values must be between -60 and 70)
    latlim -- [xmin, xmax] (values must be between -180 and 180)
    """

    # Collect account and FTP information
    username, password = WebAccounts.Accounts(Type = 'FTP_WA')
    ftpserver = "ftp.wateraccounting.unesco-ihe.org"

    # Download data from FTP
    ftp=FTP(ftpserver)
    ftp.login(username,password)
    if TimeStep is "weekly":
        directory="/WaterAccounting/Data_Satellite/Evaporation/ALEXI/World/"
    if TimeStep is "daily":
        directory="/WaterAccounting/Data_Satellite/Evaporation/ALEXI/World_05182018/"
    ftp.cwd(directory)
    lf = open(local_filename, "wb")
    ftp.retrbinary("RETR " + filename, lf.write)
    lf.close()

    if TimeStep is "weekly":

        # Open global ALEXI data
        dataset = RC.Open_tiff_array(local_filename)

        # Clip extend out of world data
        data = dataset[yID[0]:yID[1],xID[0]:xID[1]]
        data[data < 0] = -9999

    if TimeStep is "daily":

        DC.Extract_Data_gz(local_filename, os.path.splitext(local_filename)[0])

        raw_data = np.fromfile(os.path.splitext(local_filename)[0], dtype="<f4")
        dataset = np.flipud(np.resize(raw_data, [3000,7200]))
        data = dataset[yID[0]:yID[1],xID[0]:xID[1]] / 2.45                      # Values are in MJ/m2d so convert to mm/d
        data[data < 0] = -9999

    # make geotiff file
    geo = [lonlim[0],0.05,0,latlim[1],0,-0.05]
    DC.Save_as_tiff(name = DirFile, data = data, geo = geo, projection = "WGS84")
    return

max_stars_repo_name: CMicha/watools

max_issues_repo_name: CMicha/watools

max_forks_repo_name: CMicha/watools