#!/usr/bin/env python
import pika
import pymongo
import uuid
import requests
from pymongo import MongoClient

connection = pika.BlockingConnection(pika.ConnectionParameters('localhost'))
channel = connection.channel()

client = MongoClient('mongodb://localhost:27017/')

db = client['imdb_']

collection = db['imdb_collection2']

channel.queue_declare(queue='names')

def callback(ch, method, properties, body):
    bdy = body.decode()
    bdy = bdy.strip('[]"')
    print(" [x] Received %r" % bdy)
    url = bdy
    new_name = uuid.uuid5(uuid.NAMESPACE_URL,url)

    HEADERS = {
   'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
   'accept-encoding': 'gzip, deflate, br',
   'accept-language': 'en-GB,en-US;q=0.8,en;q=0.6',
   'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36',
}

    r = requests.get(url,headers=HEADERS)
    response = r.content
    f = open('/home/user/mysys/jb6/dumps/%s.html' %new_name, 'w')
    f.write(str(response))
    f.close
    db.collection.insert({'url':url})

channel.basic_consume(callback,
                      queue='names',
                      no_ack=True)

print(' [*] Waiting for messages. To exit press CTRL+C')
channel.start_consuming()


from flask import request
from flask_restx import Resource

from ...util.dto import CustomerDto
from ..service.customers_service import Customer, CustomerById

api = CustomerDto.api
_customer = CustomerDto.customer

@api.route('/')
class CustomerList(Resource):
    @api.doc('list_of_registered_customer')
    @api.marshal_list_with(_customer)
    def get(self):
        return Customer.get()

    @api.response(201, 'Customer successfully created.')
    @api.doc('create a new customer')
    @api.expect(_customer, validate=True)
    def post(self):
        data = request.json
        return Customer.post(data=data)

    @api.response(201, 'Customer successfully updated.')
    @api.doc('update customer')
    @api.expect(_customer, validate=True)
    def put(self):
        data = request.json
        return Customer.put(data=data)

@api.route('/<public_id>')
@api.param('public_id', 'The Customer identifier')
@api.response(404, 'Customer not found.')
class CustomerByParam(Resource):
    @api.doc('get a customer')
    @api.marshal_with(_customer)
    def get(self, public_id):
        id = int(public_id)
        customer = CustomerById.get(id)
        # return {'message': 'Ok', 'status': 'success'}
        if not customer:
            api.abort(404)
        else:
            return customer

    def delete(self, public_id):
        id = int(public_id)
        customer = CustomerById.delete(id)
        return customer


#!/usr/bin/env python

def find_Rscript():
    import os

    search_paths = [
        "/opt/local/bin/Rscript",
        "/usr/local/bin/Rscript",
        "/usr/bin/Rscript",
        ]
    path = None
    for spath in search_paths:
        if spath is None or not os.path.exists(spath):
            continue
        path = spath
        break
    assert path is not None, "I could not find: Rscript"
    return path

def run_R(script):
    # Run an R script in the background.  Makes sure nothing prints to
    # the screen.
    import os
    import tempfile
    import subprocess
    from genomicode.dwdnorm import _safe_unlink

    # Bug: assumes that Rscript can be executed.
    assert type(script) is type("")

    rscript = find_Rscript()
    temp_path = "."
    script_file = None
    try:
        x, script_file = tempfile.mkstemp(dir=temp_path); os.close(x)
        open(script_file, 'w').write(script)

        command = [rscript, script_file]
        p = subprocess.Popen(
            command, bufsize=0, stdin=subprocess.PIPE,
            stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
        w, r, e = p.stdin, p.stdout, p.stderr
        w.close()
        output = r.read()
        err = e.read()
        assert not err.strip(), err
    finally:
        _safe_unlink(script_file)
    return output

def parse_indexes(MATRIX, indexes, indexes_include_headers):
    from genomicode import parselib

    max_index = MATRIX.ncol()
    num_headers = len(MATRIX._row_names)
    assert max_index, "empty matrix"

    I = []
    for s, e in parselib.parse_ranges(indexes):
        if indexes_include_headers:
            s, e = s-num_headers, e-num_headers
        assert s >= 1, "Index out of range: %s" % s
        assert e <= max_index, "Index out of range: %s" % e
        s, e = s - 1, min(e, max_index)
        I.extend(range(s, e))
    return I

def make_sample_names(MATRIX):
    num = MATRIX.ncol()
    names = ['Sample%d' % (i+1) for i in range(num)]
    return names

def make_array_names(MATRIX):
    num = MATRIX.ncol()
    names = ['Array%d' % (i+1) for i in range(num)]
    return names

def write_EIF_file(MATRIX, filename):
    # write EIF file
    array_names = make_array_names(MATRIX)

    handle = open(filename, 'w')
    print >>handle, "\t".join(array_names)
    for row in MATRIX._X:
        for i in range(len(row)):
            if row[i] < 1:
                row[i] = 1
        print >>handle, "\t".join(map(str, row))

def write_SIF_file(MATRIX, filename, batches):
    sample_names = make_sample_names(MATRIX)
    array_names = make_array_names(MATRIX)

    assert len(sample_names) == len(array_names)
    assert len(sample_names) == len(batches)

    handle = open(filename, 'w')
    header = 'Array name', 'Sample name', 'Batch'
    print >>handle, "\t".join(header)
    for x in zip(array_names, sample_names, batches):
        print >>handle, "\t".join(map(str, x))

def main():
    import os
    import sys
    import argparse
    import tempfile
    import shutil

    import arrayio
    from genomicode import filelib
    from genomicode import config

    usage = "combatnorm.py [options] expression_file"
    parser = argparse.ArgumentParser(usage=usage)
    parser.add_argument("expression_file", help="Matrix to normalize.")
    #parser.add_argument("normalized_file", help="Normalized matrix.")
    parser.add_argument(
        "--indexes1",
        help="Which columns in batch 1, E.g. 1-5,8 (1-based, "
        "inclusive).  Columns leftover will be in the remaining batches.")
    parser.add_argument(
        "--indexes2",
        help="If three or more batches, then specify the samples for the "
        "second batch.")
    # Allow up to 4 groups for now.
    parser.add_argument("--indexes3")
    parser.add_argument(
        "--indexes_include_headers", "--iih", action="store_true",
        help="If not given (default), then column 1 is the first column "
        "with data.  If given, then column 1 is the very first column in "
        "the file, including the headers.")

    args = parser.parse_args()
    assert os.path.exists(args.expression_file), \
        "File not found: %s" % args.expression_file
    assert args.indexes1
    if args.indexes2:
        assert args.indexes1
    if args.indexes3:
        assert args.indexes2

    MATRIX = arrayio.read(args.expression_file)

    # Figure out the batch for each sample.
    assert MATRIX.ncol(), "empty matrix"
    # Should be 1 to <num_groups>.
    batches = [None] * MATRIX.ncol()
    all_indexes = [args.indexes1, args.indexes2, args.indexes3]
    for i, indexes in enumerate(all_indexes):
        if not indexes:
            continue
        I = parse_indexes(MATRIX, indexes, args.indexes_include_headers)
        for j in I:
            assert j < len(batches)
            assert batches[j] is None, "overlapping: %d" % j
            batches[j] = i+1
    # The remaining samples are in the next batch.
    x = batches
    x = [x for x in batches if x]
    max_batch = max(x)
    for i in range(len(batches)):
        if batches[i] == None:
            batches[i] = max_batch + 1

    temp_path = None
    cwd = os.getcwd()
    try:
        temp_path = tempfile.mkdtemp(dir=".")
        os.chdir(temp_path)

        EIF_file = "EIF.dat"
        SIF_file = "SIF.dat"

        # Write the input files.
        write_EIF_file(MATRIX, EIF_file)
        write_SIF_file(MATRIX, SIF_file, batches)

        # Run combat.
        ComBat_R = config.combat
        assert os.path.exists(ComBat_R), 'File not found: %s' % ComBat_R

        assert os.path.exists(EIF_file)
        assert os.path.exists(SIF_file)
        x = [
            'source("%s")' % ComBat_R,
            'ComBat("%s", "%s", write=T)' % (EIF_file, SIF_file),
            ]
        script = "\n".join(x)
        run_R(script)
        #R = jmath.start_R()
        #R('source("%s")' % ComBat_R)
        #R('ComBat("%s", "%s", write=T)' % (EIF_file, SIF_file))
        outfile = "Adjusted_EIF.dat_.xls"
        assert filelib.exists_nz(outfile), "File not found: %s" % outfile

        x = open(outfile).readlines()
        x = [x.rstrip("\r\n") for x in x]
        x = [x for x in x if x]
        x = [x.split("\t") for x in x]
        #header = x[0]
        norm = x[1:]
        MATRIX_norm = MATRIX.matrix()
        assert len(norm) == MATRIX_norm.nrow()
        for x in norm:
            assert len(x) == MATRIX_norm.ncol()
        MATRIX_norm._X = norm
    finally:
        os.chdir(cwd)
        if temp_path is not None and os.path.exists(temp_path):
            shutil.rmtree(temp_path)

    arrayio.tab_delimited_format.write(MATRIX_norm, sys.stdout)

if __name__=='__main__':
    main()


#你想得到一个由迭代器生成的切片对象，但是标准切片操作并不能做到。
def increase(n):
    i = n
    while True:
        yield i
        i += 1

c = increase(0)
from itertools import islice
for i in islice(c,20,30):
    print(i)
#islice() 会消耗掉传入的迭代器中的数据。 必须考虑到迭代器是不可逆的这个事实。


from django.db import models

class Click(models.Model):
    campaign = models.IntegerField(db_index=True)
    timestamp = models.DateTimeField()

    class Meta:
        indexes = [
            models.Index(fields=["campaign", "timestamp"])
        ]

    @classmethod
    def count_campaign_clicks(cls,
                              campaign: int,
                              after_date: str = None,
                              before_date: str = None) -> int:
        """
        SQL:
            SELECT count(*)
            FROM click_logs_click
            WHERE campaign = '4510461'
              AND timestamp BETWEEN '2021-11-07 03:10:00'
              AND '2021-11-07 03:30:00';
        """
        qs = cls.objects.filter(campaign=campaign)
        if after_date:
            qs = qs.filter(timestamp__gt=after_date)
        if before_date:
            qs = qs.filter(timestamp__lt=before_date)
        return qs.count()

    def __repr__(self):
        return f'Click(campaign={self.campaign}, timestamp={self.timestamp})'

    def __str__(self):
        return repr(self)


# -*- coding: utf8 -*-
import logging
from datetime import datetime

# enum
from console.enum.app import GovernanceModeEnum
from .enum import ActionType
# exception
from console.exception.main import EnvAlreadyExist, InvalidEnvName
# repository
from console.repositories.app_config import port_repo
# service
from console.services.app_config import env_var_service
# model
from www.models.main import (ServiceProbe, TenantServiceConfigurationFile, TenantServiceEnvVar, TenantServicesPort,
                             TenantServiceVolume)
from www.models.label import ServiceLabels
from console.models.main import ComponentGraph, ServiceMonitor
from console.models.main import ServiceSourceInfo, ComponentK8sAttributes
# util
from www.utils.crypt import make_uuid

logger = logging.getLogger("default")

class Component(object):
    def __init__(self,
                 component,
                 component_source: ServiceSourceInfo,
                 envs,
                 ports,
                 volumes,
                 config_files,
                 probes: [ServiceProbe],
                 extend_info,
                 monitors,
                 graphs,
                 plugin_deps,
                 http_rules=None,
                 http_rule_configs=None,
                 tcp_rules=None,
                 service_group_rel=None,
                 labels=None,
                 support_labels=None,
                 k8s_attributes=None):
        self.component = component
        self.component_source = component_source
        self.envs = list(envs)
        self.ports = list(ports)
        self.http_rules = list(http_rules) if http_rules else []
        self.http_rule_configs = list(http_rule_configs) if http_rule_configs else []
        self.tcp_rules = list(tcp_rules) if tcp_rules else []
        self.volumes = list(volumes)
        self.config_files = list(config_files)
        self.probes = list(probes) if probes else []
        self.extend_info = extend_info
        self.monitors = list(monitors)
        self.graphs = list(graphs)
        self.component_deps = []
        self.volume_deps = []
        self.plugin_deps = list(plugin_deps)
        self.app_config_groups = []
        self.labels = list(labels) if labels else []
        self.service_group_rel = service_group_rel
        self.support_labels = {label.label_name: label for label in support_labels}
        self.k8s_attributes = list(k8s_attributes) if k8s_attributes else []
        self.action_type = ActionType.NOTHING.value

    def set_changes(self, tenant, region, changes, governance_mode):
        """
        Set changes to the component
        """
        update_funcs = self._create_update_funcs()
        for key in update_funcs:
            if not changes.get(key):
                continue
            update_func = update_funcs[key]
            update_func(changes.get(key))

        self.ensure_port_envs(governance_mode)

    def ensure_port_envs(self, governance_mode):
        # filter out the old port envs
        envs = [env for env in self.envs if env.container_port == 0]
        # create outer envs for every port
        for port in self.ports:
            envs.extend(self._create_envs_4_ports(port, governance_mode))
        self.envs = envs

    def _create_envs_4_ports(self, port: TenantServicesPort, governance_mode):
        attr_name_prefix = port.port_alias.upper() if port.port_alias else self._create_default_attr_name_prefix(port)
        host_value = "127.0.0.1" if governance_mode == GovernanceModeEnum.BUILD_IN_SERVICE_MESH.name else port.k8s_service_name
        host_env = self._create_port_env(port, "连接地址", attr_name_prefix + "_HOST", host_value)
        port_env = self._create_port_env(port, "端口", attr_name_prefix + "_PORT", str(port.container_port))
        return [host_env, port_env]

    def _create_default_attr_name_prefix(self, port: TenantServicesPort):
        port_alias = self.component.service_alias.upper()
        return port_alias + str(port.container_port)

    def _create_port_env(self, port: TenantServicesPort, name, attr_name, attr_value):
        return TenantServiceEnvVar(
            tenant_id=self.component.tenant_id,
            service_id=self.component.component_id,
            container_port=port.container_port,
            name=name,
            attr_name=attr_name,
            attr_value=attr_value,
            is_change=False,
            scope="outer",
            create_time=datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
        )

    def _create_update_funcs(self):
        return {
            "deploy_version": self._update_deploy_version,
            "app_version": self._update_version,
            "envs": self._update_inner_envs,
            "connect_infos": self._update_outer_envs,
            "ports": self._update_ports,
            "volumes": self._update_volumes,
            "probes": self._update_probe,
            "labels": self._update_labels,
            "component_graphs": self._update_component_graphs,
            "component_monitors": self._update_component_monitors,
            "plugin_deps": self._update_plugin_deps,
            "component_k8s_attributes": self._update_component_k8s_attributes,
        }

    def _update_plugin_deps(self, plugin_deps):
        if not plugin_deps.get("add"):
            return
        self.update_action_type(ActionType.UPDATE.value)

    def _update_deploy_version(self, dv):
        if not dv["is_change"]:
            return
        self.component.deploy_version = dv["new"]
        self.update_action_type(ActionType.BUILD.value)

    def _update_version(self, v):
        if not v["is_change"]:
            return
        self.component.version = v["new"]

    def _update_inner_envs(self, envs):
        self._update_envs(envs, "inner")
        self.update_action_type(ActionType.UPDATE.value)

    def _update_outer_envs(self, envs):
        self._update_envs(envs, "outer")
        self.update_action_type(ActionType.UPDATE.value)

    def _update_envs(self, envs, scope):
        if envs is None:
            return
        # create envs
        add = envs.get("add", [])
        for env in add:
            container_port = env.get("container_port", 0)
            value = env.get("attr_value", "")
            name = env.get("name", "")
            attr_name = env.get("attr_name", "")
            if not attr_name:
                continue
            if container_port == 0 and value == "**None**":
                value = self.component.service_id[:8]
            try:
                env_var_service.check_env(self.component, attr_name, value)
            except (EnvAlreadyExist, InvalidEnvName) as e:
                logger.warning("failed to create env: {}; will ignore this env".format(e))
                continue
            self.envs.append(
                TenantServiceEnvVar(
                    tenant_id=self.component.tenant_id,
                    service_id=self.component.service_id,
                    container_port=container_port,
                    name=name,
                    attr_name=attr_name,
                    attr_value=value,
                    scope=scope,
                ))

    def _update_ports(self, ports):
        if ports is None:
            return

        add = ports.get("add", [])
        for port in add:
            # Optimization: do not update port data iteratively
            self._update_port_data(port)
            new_port = TenantServicesPort(**port)
            new_port.service_id = self.component.component_id
            self.ports.append(new_port)

        old_ports = {port.container_port: port for port in self.ports}
        upd = ports.get("upd", [])
        for port in upd:
            old_port = old_ports.get(port["container_port"])
            old_port.protocol = port["protocol"]
            old_port.port_alias = port["port_alias"]
            if not old_port.is_inner_service:
                old_port.is_inner_service = port["is_inner_service"]
            if not old_port.is_outer_service:
                old_port.is_outer_service = port["is_outer_service"]
        self.update_action_type(ActionType.UPDATE.value)

    def _update_component_graphs(self, component_graphs):
        if not component_graphs:
            return
        graphs = component_graphs.get("add", [])
        for graph in graphs:
            new_graph = ComponentGraph(**graph)
            new_graph.graph_id = make_uuid()
            new_graph.component_id = self.component.component_id
            self.graphs.append(new_graph)

        graphs = component_graphs.get("upd", [])
        old_graphs = {graph.title: graph for graph in self.graphs}
        for graph in graphs:
            old_graph = old_graphs.get(graph.get("title"))
            if not old_graph:
                continue
            old_graph.promql = graph.get("promql", "")
            old_graph.sequence = graph.get("sequence", 99)
        self.update_action_type(ActionType.UPDATE.value)

    def _update_component_monitors(self, component_monitors):
        if not component_monitors:
            return
        monitors = component_monitors.get("add", [])
        for monitor in monitors:
            new_monitor = ServiceMonitor(**monitor)
            new_monitor.service_id = self.component.component_id
            new_monitor.tenant_id = self.component.tenant_id
            self.monitors.append(new_monitor)
        self.update_action_type(ActionType.UPDATE.value)

    def _update_labels(self, labels):
        if not labels:
            return
        labels = labels.get("add", [])
        for key in labels:
            label = self.support_labels.get(key)
            if not label:
                continue
            self.labels.append(
                ServiceLabels(
                    tenant_id=self.component.tenant_id,
                    service_id=self.component.component_id,
                    label_id=label.label_id,
                    region=self.component.service_region,
                    create_time=datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
                ))
        self.update_action_type(ActionType.UPDATE.value)

    def _update_port_data(self, port):
        container_port = int(port["container_port"])
        port_alias = self.component.service_alias.upper()
        k8s_service_name = port.get("k8s_service_name", self.component.service_alias)
        if k8s_service_name:
            filter_port = port_repo.get_by_k8s_service_name(self.component.tenant_id, k8s_service_name)
            if filter_port and self.component.service_id:
                k8s_service_name = k8s_service_name + "-" + make_uuid(
                )[-4:] if self.component.service_id != filter_port.service_id else k8s_service_name
            port["k8s_service_name"] = k8s_service_name
        port["tenant_id"] = self.component.tenant_id
        port["service_id"] = self.component.service_id
        port["mapping_port"] = container_port
        port["port_alias"] = port_alias

    def _update_volumes(self, volumes):
        old_volumes = {volume.volume_name: volume for volume in self.volumes}
        for volume in volumes.get("add"):
            volume["service_id"] = self.component.service_id
            host_path = "/grdata/tenant/{0}/service/{1}{2}".format(self.component.tenant_id, self.component.service_id,
                                                                   volume["volume_path"])
            volume["host_path"] = host_path
            file_content = volume.get("file_content")
            if file_content:
                self.config_files.append(
                    TenantServiceConfigurationFile(
                        service_id=self.component.component_id,
                        volume_name=volume["volume_name"],
                        file_content=file_content,
                    ))
            if "file_content" in volume.keys():
                volume.pop("file_content")
            self.volumes.append(TenantServiceVolume(**volume))

        old_config_files = {config_file.volume_name: config_file for config_file in self.config_files}
        for volume in volumes.get("upd"):
            old_volume = old_volumes.get(volume["volume_name"])
            old_volume.mode = volume.get("mode")
            old_config_file = old_config_files.get(volume.get("volume_name"))
            if not old_config_file:
                continue
            old_config_file.file_content = volume.get("file_content")

        self.config_files = old_config_files.values()
        self.update_action_type(ActionType.UPDATE.value)

    def _update_probe(self, probe):
        old_probes = {probe.mode: probe for probe in self.probes}

        new_probes = []
        add = probe.get("add")
        if add:
            new_probes.extend(add)
        upd = probe.get("upd")
        if upd:
            new_probes.extend(upd)
        # There can only be one probe of the same mode
        # Dedup new probes based on mode
        new_probes = {probe["mode"]: probe for probe in new_probes}
        if not new_probes:
            return
        probes = []
        for key in new_probes:
            probe = new_probes[key]
            old_probe = old_probes.get(probe["mode"])
            if not old_probe:
                # create new probe
                probe["probe_id"] = make_uuid()
                probe["service_id"] = self.component.component_id
                probes.append(ServiceProbe(**probe))
                continue
            # update probe
            probe = ServiceProbe(**probe)
            probe.ID = old_probe.ID
            probe.service_id = self.component.component_id
            probes.append(probe)
        self.probes = probes
        self.update_action_type(ActionType.UPDATE.value)

    def _update_component_k8s_attributes(self, component_k8s_attributes):
        if not component_k8s_attributes:
            return
        old_k8s_attributes = {attr.name: attr for attr in self.k8s_attributes}
        new_k8s_attributes = []
        if component_k8s_attributes.get("add", []):
            new_k8s_attributes.extend(component_k8s_attributes.get("add"))
        if component_k8s_attributes.get("upd", []):
            new_k8s_attributes.extend(component_k8s_attributes.get("upd"))

        k8s_attributes = {}
        for attribute in new_k8s_attributes:
            attr_name = attribute.get("name")
            if not attr_name:
                continue
            attr = old_k8s_attributes.get(attr_name)
            if not attr:
                self.k8s_attributes.append(
                    ComponentK8sAttributes(
                        tenant_id=self.component.tenant_id,
                        component_id=self.component.component_id,
                        name=attribute.get("name"),
                        save_type=attribute.get("save_type"),
                        attribute_value=attribute.get("attribute_value"),
                    ))
                continue
            # update attrs
            k8s_attr = ComponentK8sAttributes(**attribute)
            k8s_attr.tenant_id = self.component.tenant_id
            k8s_attr.component_id = self.component.component_id
            k8s_attributes[attr_name] = k8s_attr

        for original_attr in self.k8s_attributes:
            if k8s_attributes.get(original_attr.name):
                original_attr.attribute_value = k8s_attributes[original_attr.name].attribute_value

        self.update_action_type(ActionType.UPDATE.value)

    def update_action_type(self, action_type):
        if action_type > self.action_type:
            self.action_type = action_type


from data import *
from utils.augmentations import SSDAugmentation
from layers.modules import MultiBoxLoss
from ssd import build_ssd
from os.path import join
import os
import sys
import time
import torch
from pathlib import Path
from torch.autograd import Variable
import torch.nn as nn
import torch.optim as optim
import torch.backends.cudnn as cudnn
import torch.nn.init as init
import torch.utils.data as data
import torchvision

import numpy as np
import argparse
import tqdm

sys.path.append(os.path.abspath('../../active_learning'))
from active_learning import ActiveLearning
from active_loss import LossPredictionLoss
from active_learning_utils import *

def str2bool(v):
    return v.lower() in ("yes", "true", "t", "1")

def create_dir_if_doesnt_exist(path):
    # Create working dir if not exists.
    directory=Path(path)
    if not directory.exists():
        directory.mkdir(parents=True)

parser = argparse.ArgumentParser(
    description='Single Shot MultiBox Detector Training With Pytorch'
)
train_set = parser.add_mutually_exclusive_group()
parser.add_argument(
    '--dataset',
    default='VOC',
    choices=['VOC', 'COCO'],
    type=str,
    help='VOC or COCO'
)
parser.add_argument(
    '--dataset_root', default=VOC_ROOT, help='Dataset root directory path'
)
parser.add_argument(
    '--basenet', default='vgg16_reducedfc.pth', help='Pretrained base model'
)
parser.add_argument(
    '--batch_size', default=32, type=int, help='Batch size for training'
)
parser.add_argument(
    '--resume',
    default=None,
    type=str,
    help='Checkpoint state_dict file to resume training from'
)
parser.add_argument(
    '--start_iter', default=0, type=int, help='Resume training at this iter'
)
parser.add_argument(
    '--num_workers',
    default=4,
    type=int,
    help='Number of workers used in dataloading'
)
parser.add_argument(
    '--cuda', default=True, type=str2bool, help='Use CUDA to train model'
)
parser.add_argument(
    '--lr',
    '--learning-rate',
    default=1e-3,
    type=float,
    help='initial learning rate'
)
parser.add_argument(
    '--momentum', default=0.9, type=float, help='Momentum value for optim'
)
parser.add_argument(
    '--weight_decay', default=5e-4, type=float, help='Weight decay for SGD'
)
parser.add_argument(
    '--gamma', default=0.1, type=float, help='Gamma update for SGD'
)
parser.add_argument(
    '--use_active_learning', default=False, type=bool, help='If True, will use active learning'
)
parser.add_argument(
    '--lamda', default=1, type=int, help='Active learning loss weight'
)

parser.add_argument(
    '--visdom',
    default=False,
    type=str2bool,
    help='Use visdom for loss visualization'
)
parser.add_argument(
    '--save_folder',
    default='weights/',
    help='Directory for saving checkpoint models'
)
parser.add_argument(
    '--output_superannotate_csv_file',
    required=False,
    type=str,
    default=None,
    help='Path to the output csv file with the selected indices. Can be uploaded to annotate.online.')

args = parser.parse_args()
COCO_ROOT=''
if torch.cuda.is_available():
    if args.cuda:
        torch.set_default_tensor_type('torch.cuda.FloatTensor')
    if not args.cuda:
        print(
            "WARNING: It looks like you have a CUDA device, but aren't " +
            "using CUDA.\nRun with --cuda for optimal training speed."
        )
        torch.set_default_tensor_type('torch.FloatTensor')
else:
    torch.set_default_tensor_type('torch.FloatTensor')

if not os.path.exists(args.save_folder):
    os.mkdir(args.save_folder)

def train():
    if args.dataset == 'COCO':
        if args.dataset_root == VOC_ROOT:
            if not os.path.exists(COCO_ROOT):
                parser.error('Must specify dataset_root if specifying dataset')
            print(
                "WARNING: Using default COCO dataset_root because " +
                "--dataset_root was not specified."
            )
            args.dataset_root = COCO_ROOT
        cfg = coco
        dataset = COCODetection(
            root=args.dataset_root,
            transform=SSDAugmentation(cfg['min_dim'], MEANS)
        )
    elif args.dataset == 'VOC':
        if args.dataset_root == COCO_ROOT:
            parser.error('Must specify dataset if specifying dataset_root')
        cfg = voc
        dataset = VOCDetection(
            root=args.dataset_root,
            transform=SSDAugmentation(cfg['min_dim'], MEANS)
        )
    IMG_CNT = len(dataset)
    if args.visdom:
        import visdom
        viz = visdom.Visdom()

    criterion = MultiBoxLoss(
        cfg['num_classes'], 0.5, True, 0, True, 3, 0.5, False, args.cuda
    )

    # loss counters
    loc_loss = 0
    conf_loss = 0
    epoch = 0
    print('Loading the dataset...')

    rand_state = np.random.RandomState(1311)
    epoch_size = len(dataset) // args.batch_size
    global pool_idx
    pool_idx = list(range(IMG_CNT))
    print('Training SSD on:', args.dataset)
    print('Using the specified args:')
    print(args)

    step_index = 0

    if args.visdom:
        vis_title = 'SSD.PyTorch on ' + dataset.name
        vis_legend = ['Loc Loss', 'Conf Loss', 'Total Loss']
        iter_plot = create_vis_plot('Iteration', 'Loss', vis_title, vis_legend)
        epoch_plot = create_vis_plot('Epoch', 'Loss', vis_title, vis_legend)

    #random image choice
    train_idx = []
    progress = tqdm.tqdm(range(10))
    for cycle in progress:
        net = build_ssd('train', cfg['min_dim'], cfg['num_classes'])

        if args.use_active_learning:
            net = ActiveLearning(net)

        if args.cuda:
            net = torch.nn.DataParallel(net)
            cudnn.benchmark = True

        if args.resume:
            print('Resuming training, loading {}...'.format(args.resume))
            net.load_weights(args.resume)
        else:
            vgg_weights = torch.load(args.save_folder + args.basenet)
            print('Loading base network...')
            if args.use_active_learning:
                if args.cuda:
                    net.module.base_model.vgg.load_state_dict(vgg_weights)
                else:
                    net.base_model.vgg.load_state_dict(vgg_weights)
            else:
                if args.cuda:
                    net.module.vgg.load_state_dict(vgg_weights)
                else:
                    net.vgg.load_state_dict(vgg_weights)

        if args.cuda:
            net = net.cuda()

        if not args.resume:
            print('Initializing weights...')
            # initialize newly added layers' weights with xavier method
            if args.use_active_learning:
                if args.cuda:
                    model = net.module.base_model
                else:
                   model = net.base_model
            else:
                if args.cuda:
                    model = net.module
                else:
                   model = net
            model.extras.apply(weights_init)
            model.loc.apply(weights_init)
            model.conf.apply(weights_init)

        net.train()

        if args.use_active_learning and not cycle == 0:
            cfg = voc
            dataset = VOCDetection(
                root=args.dataset_root,
                transform=BaseTransform(300, MEANS)
            )
            device = 'cuda' if (torch.cuda.is_available() and args.cuda) else 'cpu'
            indices, losses = choose_indices_loss_prediction_active_learning(
                net, cycle, rand_state, pool_idx, dataset, device)
            train_idx.extend(indices)

            if args.output_superannotate_csv_file is not None:
                # Write image paths to csv file which can be uploaded to annotate.online.
                write_entropies_csv(
                    dataset, indices,losses, args.output_superannotate_csv_file)
        else:
            train_idx.extend(random_indices(pool_idx, rand_state, count=1000))
        cfg = voc
        dataset = data.Subset(VOCDetection(
            root=args.dataset_root,
            transform=SSDAugmentation(cfg['min_dim'], MEANS)
            ), train_idx
        )
        data_loader = data.DataLoader(
            dataset,
            args.batch_size,
            num_workers=args.num_workers,
            shuffle=True,
            collate_fn=detection_collate,
            pin_memory=True
        )
        optimizer = optim.SGD(
            net.parameters(),
            lr=args.lr,
            momentum=args.momentum,
            weight_decay=args.weight_decay
        )
        for epoch in range(300):
            if epoch == 240:
                adjust_learning_rate(optimizer)
            for iteration, (images, targets) in enumerate(data_loader):
                if args.cuda:
                    images = Variable(images.cuda())
                    targets = [Variable(ann.cuda(), requires_grad=False) for ann in targets]
                else:
                    images = Variable(images)
                    targets = [Variable(ann, requires_grad=False) for ann in targets]
                # forward
                t0 = time.time()
                if args.use_active_learning:
                    out, loss_pred = net(images)
                else:
                    out =  net(images)
                # backprop
                optimizer.zero_grad()
                loss_l, loss_c, N = criterion(out, targets)
                # print("loss_l {} loss_c {} N {}".format(loss_l, loss_c, N))
                loss = (loss_l + loss_c).sum() / N
                # print("loss = {}".format(loss))
                if args.use_active_learning:
                    criterion_lp = LossPredictionLoss()
                    loss_prediction_loss = criterion_lp(loss_pred, loss_l + loss_c)
                    loss += args.lamda * loss_prediction_loss
                loss.backward()
                optimizer.step()
                t1 = time.time()
                loc_loss += loss_l.mean().item()
                conf_loss += loss_c.mean().item()

                if iteration % 10 == 0:
                    progress.set_description('timer: %.4f sec, ' % (t1 - t0) +
                        'iter ' + repr(iteration) + (', Loss: %.4f' %
                        (loss.item())))
        folder = join(args.save_folder,
                      'active_learning' if args.use_active_learning else 'random')
        create_dir_if_doesnt_exist(folder)
        torch.save(
            net.state_dict(),
            join(folder, 'cycle_'+str(cycle+1)+'k.pth')
        )

def adjust_learning_rate(optimizer):
    """Sets the learning rate to the initial LR decayed by 10 at every
        specified step
    # Adapted from PyTorch Imagenet example:
    # https://github.com/pytorch/examples/blob/master/imagenet/main.py
    """
    lr = args.lr * 0.1
    for param_group in optimizer.param_groups:
        param_group['lr'] = lr

def xavier(param):
    init.xavier_uniform_(param)

def weights_init(m):
    if isinstance(m, nn.Conv2d):
        xavier(m.weight.data)
        m.bias.data.zero_()

def create_vis_plot(_xlabel, _ylabel, _title, _legend):
    return viz.line(
        X=torch.zeros((1, )).cpu(),
        Y=torch.zeros((1, 3)).cpu(),
        opts=dict(xlabel=_xlabel, ylabel=_ylabel, title=_title, legend=_legend)
    )

def update_vis_plot(
    iteration, loc, conf, window1, window2, update_type, epoch_size=1
):
    viz.line(
        X=torch.ones((1, 3)).cpu() * iteration,
        Y=torch.Tensor([loc, conf, loc + conf]).unsqueeze(0).cpu() / epoch_size,
        win=window1,
        update=update_type
    )
    # initialize epoch plot on first iteration
    if iteration == 0:
        viz.line(
            X=torch.zeros((1, 3)).cpu(),
            Y=torch.Tensor([loc, conf, loc + conf]).unsqueeze(0).cpu(),
            win=window2,
            update=True
        )

if __name__ == '__main__':
    train()


# -*- coding: utf-8 -*-
from gensim import models
def main():
	model = models.Word2Vec.load('word2vec.model')

	try:

		for word in ['月','蟾','嫦娥','兔','中秋','赏月','桂花','花灯','饼','桂花']:
			q_list = word.split()
			if len(q_list) == 1:
				topn = 5
				print("相似詞前 %d 排序"%topn)
				res = model.wv.most_similar(q_list[0],topn = topn)

				for item in res:
					print(word,'-->',item[0]+","+str(item[1]))

	except Exception as e:
		print(repr(e))

if __name__ == "__main__":
	main()


# coding=utf-8

import logging

from abc import ABC, abstractmethod
from app.repositories import redis_client

__author__ = 'LongHB'
_logger = logging.getLogger(__name__)

KEY_SPLITTER = ":"
NONE_VALUE_TO_CACHE = b''

class RedisCacheBase(ABC):
    """
    ABC means this class is abstract class
    """

    def __init__(self, name: str, ttl: int = None):
        """
        :param name: The name of the data is cached
        :param ttl: time-to-live in second of caching data. Default to None -> No expired time
        """

        self.name = name
        self.ttl = ttl
        self.redis_client = redis_client

    def _get_full_key(self, key: str) -> str:
        return self.name + KEY_SPLITTER + key

    def get(self, key: str):
        if key is None:
            return None
        full_key = self._get_full_key(key)
        value = self.redis_client.get(full_key)
        if value == NONE_VALUE_TO_CACHE: return None
        value = value.decode("utf-8") if value else value
        if not value:
            value = self.fetch_data(key)
            redis_client.set(full_key, NONE_VALUE_TO_CACHE, ex=self.ttl) if not value else \
                redis_client.set(full_key, value, ex=self.ttl)
        return value

    @abstractmethod
    def fetch_data(self, key: str):
        raise NotImplemented


#!/usr/bin/env python
# coding: utf-8

# In[82]:

# import torch
# import torch.nn
# from torch.nn import functional as f
# import pandas as pd
# import json
# import numpy as np
# import os

# DIRECTORY = 'data'
# paths = []

# for root, dirs, files in os.walk(DIRECTORY):
#     for name in files:
#         paths.append(os.path.join(root, name))

# names = [i.split('/')[-1] for i in paths][1:]
# data_dict = dict(zip([i[:-4] for i in names], paths[1:]))

# In[83]:

# import urllib
# from newspaper import Article
# from data import *

# def meta_extract(url):
#     article = Article(url)
#     article.download()
#     article.parse()
#     article.download('punkt')
#     article.nlp()

#     return article.authors, article.publish_date, article.top_image, article.images, article.title, article.summary

# # meta_extract(url)

# # In[84]:

# from nltk.classify import NaiveBayesClassifier
# from nltk.corpus import subjectivity
# from nltk.sentiment import SentimentAnalyzer, SentimentIntensityAnalyzer
# from nltk.sentiment.util import *
# import nltk

# def sentiment(inp_text):
#     sia = SentimentIntensityAnalyzer()
#     return sia.polarity_scores(inp_text)

# In[104]:

from data import *
from meta import *
from net import *

# class PreprocessingDataset(Dataset):
#     def __init__(self, file, root, x_col, y_col, meta_columns, label_idx = -1):
#         self.x_col = x_col
#         self.y_col = y_col
#         self.data = pd.read_csv(file)
#         self.data = self.data.sample(frac=1).reset_index(drop=True)
#         self.data = self.data.drop(meta_columns, axis=1)

#         # self.data, self.base_ref = self.tokenizer(self.data, [x_col])
#         self.x_data = self.data[x_col]
#         self.max_len = max([len(i) for i in self.x_data])
#         self.max_len = 600

#         self.x_data, self.token = self.word_vector(self.x_data)
#         self.data[x_col] = [torch.FloatTensor(i) for i in self.x_data]
#         self.data = self.vectorize(self.data, [y_col])
#         self.df_data = self.data
#         self.data = self.data.to_numpy()

#         self.root = root
#         self.transform = transforms.Compose([transforms.ToTensor()])

#     def format_text(self, token):
#         clean_token = ''.join(chr for chr in token if chr.isalnum() and chr.isalpha())
#         return clean_token

#     def word_vector(self, data):
#         x_data = data
#         x_data = list(x_data)
#         maximum_length = 0
#         max_idx = 0
#         for idx, i in enumerate(x_data):

#             if len(i) > maximum_length:
#                 maximum_length = len(i)
#                 max_idx = idx
#         maximum_length = 600
#         t = Tokenizer(num_words=600, filters='\n.,:!"#$()&@%^()-_`~[];.,{|}')
#         t.fit_on_texts(x_data)
#         sequences = t.texts_to_sequences(x_data)
#         sequences = keras.preprocessing.sequence.pad_sequences(sequences, maxlen=maximum_length)
#         print(x_data[0])
#         print(len(x_data[0]))
#         print(sequences[0])

#         return sequences, t

#     def vectorize(self, data_inp, columns):
#         data = data_inp
#         for column in columns:
#             labels = list(data[column].unique())
#             ref = dict(zip(data[column].unique(), [i for i in range(len(labels))]))
#             print(ref)
#             for idx, val in enumerate(data[column]):
#                 vectorized = ref[data[column][idx]]
#                 data[column][idx] = torch.tensor(vectorized, dtype=float)
#         return data

#     def __len__ (self):
#         return len(self.data)

#     def __getitem__ (self, idx):

#         self.transpose_data = self.data
#         self.transpose_data = self.transpose_data.transpose()
#         x_data = self.transpose_data[0]
#         y_data = self.transpose_data[1]

#         return x_data[idx], y_data[idx]

DIRECTORY = 'data'
data_dict = {'politifact': 'data/truth-detectiondeception-detectionlie-detection/politifact.csv', 'politifact_clean': 'data/truth-detectiondeception-detectionlie-detection/politifact_clean.csv', 'politifact_clean_binarized': 'data/truth-detectiondeception-detectionlie-detection/politifact_clean_binarized.csv'}
clean_truth_data = PreprocessingDataset(data_dict['politifact_clean_binarized'], DIRECTORY, 'statement', 'veracity', ['source', 'link'])

BATCH_SIZE = 64

primary_data = clean_truth_data #secondary option of truth_data

train_len = int(len(primary_data)*0.8)
test_len = len(primary_data) - train_len

train_set, test_set = torch.utils.data.random_split(primary_data, [train_len, test_len])

train_loader = DataLoader(train_set, batch_size=BATCH_SIZE, shuffle=True)
test_loader = DataLoader(test_set, batch_size=BATCH_SIZE, shuffle=True)

# print(len(train_set))
# print(len(test_set))

num_feats = np.array([train_set[i][0].numpy() for i in range(len(train_set))])
num_labels = np.array([train_set[i][1] for i in range(len(train_set))])

a = iter(train_loader)
b = next(a)
b = np.asarray(b[0])
# print(b.shape)
inp_size = (b.shape)[1]
# print(inp_size)
# print(inp_size)

# In[87]:

import itertools
ab = list(itertools.chain(*[i[0] for i in clean_truth_data]))
# print(len(ab))
ab = set([int(i) for i in ab])
emb_dim = len(ab)

# In[105]:

# import torch.nn as nn
# import torch.nn.functional as F

# device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

# class FeedForward(nn.Module):
#     def __init__(self, num_classes, input_size, kernel_size=4):
#         super(FeedForward, self).__init__()
#         self.fc1 = nn.Linear(input_size, 200)
#         self.fc3 = nn.Linear(200, 100)
#         self.fc4 = nn.Linear(100, 100)
#         self.fc5 = nn.Linear(100, 50)
#         self.fc6 = nn.Linear(50, 20)
#         self.fc7 = nn.Linear(20, 1)
#         self.dropout = nn.Dropout(0.3)

#     def forward(self, x):
#         x = self.dropout(F.relu(self.fc1(x)))
#         x = self.dropout(F.relu(self.fc3(x)))
#         x = self.dropout(F.relu(self.fc4(x)))
#         x = self.dropout(F.relu(self.fc5(x)))
#         x = self.dropout(F.relu(self.fc6(x)))
#         x = self.dropout(F.relu(self.fc7(x)))

#         return x

# class RecurrentClassifier(nn.Module):
#     def __init__(self, embedding_dim, input_size, hidden_size, output_size, num_layers, dropout=0.3):
#         super(RecurrentClassifier, self).__init__()

#         self.embedding = nn.Embedding(embedding_dim, input_size)
#         self.rnn = nn.LSTM(input_size,
#                             hidden_size,
#                             num_layers,
#                             batch_first = True,
#                             dropout=dropout)
#         self.fc1 = nn.Linear(hidden_size*2, hidden_size)
#         self.fc2 = nn.Linear(hidden_size, output_size)
#         self.dropout = nn.Dropout(dropout)

#     def forward(self, x):
#         x = self.embedding(x)
#         x, (hidden, cell) = self.rnn(x)
#         print(hidden.shape)
#         hidden = self.dropout(torch.cat((hidden[-2,:,:], hidden[-1, :, :]), dim=1))
#         x = self.fc1(hidden)
#         x = self.dropout(self.fc2(x))

#         return x

max_len = len(train_set[1][0])
ref_check = 1

feedforward = FeedForward(ref_check, inp_size).to(device)
recurrent = RecurrentClassifier(emb_dim, inp_size, 50, ref_check, 2, dropout=0.2).to(device)

# print(feedforward, recurrent)

# In[114]:

def train(net, train_loader, LR, DECAY, EPOCHS):
    optimizer = torch.optim.Adam(net.parameters(), lr=LR, weight_decay=DECAY)
    loss_func = torch.nn.BCEWithLogitsLoss()

    epochs = EPOCHS
    losses = []

    for step in range(epochs):
        running_loss = 0.0
        for i, data in enumerate(train_loader):
            inp, labels = data
            if net == recurrent:
                inp, labels = inp.long().to(device), labels.float().to(device)
                optimizer.zero_grad()
                outputs = net(inp)
                cost = loss_func(torch.squeeze(outputs), torch.squeeze(labels))
            elif net == feedforward:
                inp, labels = inp.float().to(device), labels.float().to(device)
                optimizer.zero_grad()
                outputs = net(inp)
                cost = loss_func(torch.squeeze(outputs), labels)
            cost.backward()
            optimizer.step()

            running_loss += cost.item()
        print(f'Epoch: {step}   Training Loss: {running_loss/len(train_loader)}')
    print('Training Complete')

    return losses

def eval(net, test_loader):
    total = 0
    acc = 0
    optimizer = torch.optim.Adam(net.parameters(), lr=LR, weight_decay=DECAY)

    for i, data in enumerate(test_loader):
        inp, labels = data
        optimizer.zero_grad()
        output = net(inp.float())
        output = output.detach().numpy()
        output = list(output)
        output = [list(i).index(max(i)) for i in output]

        for idx, item in enumerate(torch.tensor(output)):
            total += 1
            if item == labels[idx]:
                acc += 1
    print(f'{acc/total*100}%')

# In[116]:

def model_load(net, PATH, name, export=True):
    if export:
        torch.save(net.state_dict(), PATH+name+'.pth')
        return PATH+name+'.pth'
    else:
        net.torch.load_state_dict(torch.load(PATH + name + '.pth'))
        return net

# train(feedforward, train_loader, 1e-3, 5e-3, 200)
model_load(feedforward, 'model_parameters/', 'linear_politifact')

# train(recurrent, train_loader, 1e-3, 5e-3, 200)
model_load(recurrent, 'model_parameters/', 'lstm_politifact')

# In[138]:

# url = 'https://www.historyofvaccines.org/content/blog/defense-of-common-antivaxxer'

# _, _, _, _, _, summ = meta_extract(url)

token_basis = clean_truth_data.token

def tokenize_sequence(text_inp, tokenizer):
    text_inp = text_inp.lower().split('\n')
    tokenizer.fit_on_texts(text_inp)
    sequences = tokenizer.texts_to_sequences(text_inp)
    sequences = [i if i!=[] else [0] for i in tokenizer.texts_to_sequences(text_inp)]
    sequences = [i[0] for i in sequences]
    pad_len =  [0]*int(inp_size - len(sequences))
    sequences += pad_len
    return torch.FloatTensor(sequences)[:600]

# inp = tokenize_sequence(summ, token_basis)
# inp = inp[None, :]
# print(inp.shape)

def prediction(inp, model):
    output = model(inp)
    return output

# a = prediction(inp, feedforward)

# In[143]:

# feedforward_template = FeedForward(ref_check, inp_size).to(device)
# recurrent_template = RecurrentClassifier(emb_dim, inp_size, 50, ref_check, 2, dropout=0.2).to(device)
# model_load(feedforward_template, '/Users/devpatelio/Downloads/Coding/Global_Politics_EA/model_parameters/', 'linear_politifact')
# model_load(recurrent_template, '/Users/devpatelio/Downloads/Coding/Global_Politics_EA/model_parameters/', 'lstm_politifact')

# # feedforward_template.eval()
# # recurrent_template.eval()

# output_linear = '0'
# output_lstm = '1' #check for error without passing error

# output_linear = prediction(inp, feedforward_template)
# output_lstm = prediction(inp.long(), recurrent_template)

# if output_linear == 0:
#     output_linear = f"No Bias: Prediction = {output_linear}"
# elif output_linear == 1:
#     output_linear = f"Bias: Prediction = {output_linear}"

# if output_lstm == 0:
#     output_lstm = f"Limited Veracity: Prediction = {output_lstm}"
# elif output_lstm == 1:
#     output_lstm = f"Expressive Veracity: Prediction = {output_lstm}"

# In[179]:

from django.shortcuts import render
# from scrape import meta_extract
import flask
from flask import Flask, request, render_template, redirect, url_for
# from sentiment import sentiment
import numpy as np
import pandas as pd
import json
import random
import os
import html
import torch
# from linear_politifact_basis import token_basis, convert_word_to_token
from flask_ngrok import run_with_ngrok
from flask import Flask, render_template, redirect, url_for, request
app = Flask(__name__)
# run_with_ngrok(app)   #starts ngrok when the app is run

from flask import Flask, jsonify, request
from string import punctuation
from collections import Counter
import random

def model_load(net, PATH, name, export=True):
    if export:
        torch.save(net.state_dict(), PATH+name+'.pth')
        return PATH+name+'.pth'
    else:
        net.torch.load_state_dict(torch.load(PATH + name + '.pth'))
        net.eval()

@app.route('/')
def hello():
    return render_template("home.html")

@app.route('/link', methods=["POST", "GET"])
def link():
    if request.method == "POST":
        link_inp = request.form['linker']
        # print(type(link_inp))

        link_inp = link_inp.replace('.com', 'comkey')
        link_inp = link_inp.replace('https://', 'https')
        link_inp = link_inp.replace('www.', 'www')
        link_inp = link_inp.replace('/', 'slash')
        # print(link_inp)
        main = link_inp

        return redirect(url_for("preview_linker", linkage=main, tag='link_url'))
    else:
        return render_template("link.html")

@app.route('/text', methods=["POST", "GET"])
def pure_text():
    if request.method == "POST":
        inp_raw = request.form['raw_text']
        inp_raw = format_raw_text(inp_raw)
        return redirect(url_for('preview_linker', linkage=inp_raw, tag='pure_text'))
    else:
        return render_template("pure_text.html")

@app.route(f"/output/<tag>/<linkage>")
def preview_linker(linkage, tag):
    preview = linkage
    if tag == 'link_url':
        preview = preview.replace('https', 'https://')
        preview = preview.replace('www', 'www.')
        preview = preview.replace('slash', '/')
        preview = preview.replace('comkey', '.com')
        authart, publ, timg, allimg, tit, summ = meta_extract(preview)

    elif tag == 'pure_text':
        preview = preview.replace('uxd', ' ')
        summ = preview
        empty_msg = 'None'
        authart = empty_msg
        publ = empty_msg
        timg = empty_msg
        allimg = empty_msg
        tit = empty_msg

    sent = sentiment(summ)

    inp = tokenize_sequence(summ, token_basis)
    inp = inp[:600]
    inp = inp[None, :]
    # print(inp.shape)

    feedforward_template = FeedForward(ref_check, inp_size).to(device)
    recurrent_template = RecurrentClassifier(emb_dim, inp_size, 50, ref_check, 2, dropout=0.2).to(device)
    model_load(feedforward_template, 'model_parameters/', 'linear_politifact')
    model_load(recurrent_template, 'model_parameters/', 'lstm_politifact')

    # feedforward_template.eval()
    # recurrent_template.eval()

    output_linear = '0 ERROR'
    output_lstm = '1 ERROR' #check for error without passing error

    output_linear = F.sigmoid(prediction(inp, feedforward_template)).round()
    output_lstm = F.sigmoid(prediction(inp.long(), recurrent_template))

    all_types = list(pd.read_csv(data_dict['politifact_clean'])['veracity'].unique())

    if output_linear == 0:
        output_linear = f"Little Bias: Prediction = {output_linear}"
    elif output_linear == 1:
        output_linear = f"Substantial Bias: Prediction = {output_linear}"

    statement_type = ''
    if output_lstm <= 0.25:
        statement_type = 'True'
    elif 0.25 < output_lstm <= 0.5:
        statement_type = 'Mostly True'
    elif 0.5 < output_lstm <= 0.75:
        statement_type = 'Mostly False'
    elif 0.75 < output_lstm <= 1:
        statement_type = 'False'
    elif output_lstm > 1:
        statement_type = 'Pants on Fire!'

    output_lstm = f"Veracity -> {statement_type}: {output_lstm}"

    # if output_lstm == 0:
    #     output_lstm = f"Limited Veracity: Prediction = {output_lstm}"
    # elif output_lstm == 1:
    #     output_lstm = f"Expressive Veracity: Prediction = {output_lstm}"

    return render_template("preview.html", preview_link=preview,
                                            author_article=authart,
                                            published_article=publ,
                                            top_image = timg,
                                            all_image = allimg,
                                            title_article=tit,
                                            summary_article=summ,
                                            sentiment=sent,
                                            bias_point=output_linear,
                                            skew_point=output_lstm)


import math
import numpy as np
import cv2
import glob
import os.path as osp
import matplotlib.pyplot as plt

path_to_GT = "GT/*"

def PSNR(original, compressed):
    mse = np.mean((original - compressed) ** 2)
    if (mse == 0):
        return 100
    max_pixel = 255.0
    psnr = 20 * math.log10(max_pixel / math.sqrt(mse))
    return psnr

def ssim(img1, img2):
    C1 = (0.01 * 255)**2
    C2 = (0.03 * 255)**2

    img1 = img1.astype(np.float64)
    img2 = img2.astype(np.float64)
    kernel = cv2.getGaussianKernel(11, 1.5)
    window = np.outer(kernel, kernel.transpose())

    mu1 = cv2.filter2D(img1, -1, window)[5:-5, 5:-5]
    mu2 = cv2.filter2D(img2, -1, window)[5:-5, 5:-5]
    mu1_sq = mu1**2
    mu2_sq = mu2**2
    mu1_mu2 = mu1 * mu2
    sigma1_sq = cv2.filter2D(img1**2, -1, window)[5:-5, 5:-5] - mu1_sq
    sigma2_sq = cv2.filter2D(img2**2, -1, window)[5:-5, 5:-5] - mu2_sq
    sigma12 = cv2.filter2D(img1 * img2, -1, window)[5:-5, 5:-5] - mu1_mu2

    ssim_map = ((2 * mu1_mu2 + C1) * (2 * sigma12 + C2)) / ((mu1_sq + mu2_sq + C1) *
                                                            (sigma1_sq + sigma2_sq + C2))
    ssim_out = 1-ssim_map.mean()
    return ssim_out

def calc_ssim(img1, img2):
    if not img1.shape == img2.shape:
        raise ValueError('Input images must have the same dimensions.')
    if img1.ndim == 2:
        return ssim(img1, img2)
    elif img1.ndim == 3:
        if img1.shape[2] == 3:
            ssims = []
            for i in range(3):
                ssims.append(ssim(img1, img2))
            return np.array(ssims).mean()
        elif img1.shape[2] == 1:
            return ssim(np.squeeze(img1), np.squeeze(img2))
    else:
        raise ValueError('Wrong input image dimensions.')

def gt_finder(idx):
    idx1 = -1
    for path in glob.glob(path_to_GT):
        idx1 += 1
        if idx1 == idx:
            return np.array(cv2.imread(path))

path_to_ESRGAN = 'ESRGAN/*'

psnre = []
ssime = []

idx = 0
print("Calculating for ESRGAN")
for path in glob.glob(path_to_ESRGAN):
    idx += 1
    try:
        base = osp.splitext(osp.basename(path))[0]
        image1 = np.array(cv2.imread(path))
        image2 = gt_finder(idx)
        image1.resize(image2.shape)
        psnre.append((PSNR(image1, image2))*1000)
        ssime.append((calc_ssim(image1, image2)))
    except:
        continue
print(psnre)
print(ssime)

path_to_OURMODEL = 'Our_model/*'

psnro = []
ssimo = []

print("Calculating for Our_model")
idx = 0
for path in glob.glob(path_to_OURMODEL):
    idx += 1
    try:
        base = osp.splitext(osp.basename(path))[0]
        image1 = np.array(cv2.imread(path))
        image2 = gt_finder(idx)
        image1.resize(image2.shape)
        psnro.append((PSNR(image1, image2))*1000)
        ssimo.append((calc_ssim(image1, image2)))
    except:
        continue

print(psnro)
print(ssimo)

path_to_SRGAN = 'SRGAN/*'

psnrs = []
ssims = []

print("Calculating for SRGAN")
idx = 0
for path in glob.glob(path_to_SRGAN):
    idx += 1
    try:
        base = osp.splitext(osp.basename(path))[0]
        image1 = np.array(cv2.imread(path))
        image2 = gt_finder(idx)
        image1.resize(image2.shape)
        psnrs.append((PSNR(image1, image2))*1000)
        ssims.append((calc_ssim(image1, image2)))
    except:
        continue

print(psnrs)
print(ssims)

path_to_g = 'Ground/*'
print("Calculating for Ground Truth")

psnrg = []
ssimg = []
idx = 0
for path in glob.glob(path_to_g):
    idx += 1
    try:
        base = osp.splitext(osp.basename(path))[0]
        image1 = np.array(cv2.imread(path))
        image2 = gt_finder(idx)
        image1.resize(image2.shape)
        psnrg.append((PSNR(image1, image2))*1000)
        ssimg.append((calc_ssim(image1, image2)))
    except:
        continue

print(psnrg)
print(ssimg)

X = ['1','2','3','4','5','6','7','8','9']

X_axis = np.arange(len(X))

plt.bar(X_axis - 0.2 , psnrs , 0.2 , label = 'SRGAN')
plt.bar(X_axis  , psnre , 0.2 , label = 'ESRGAN')
plt.bar(X_axis + 0.2 , psnro , 0.2 , label = 'Our Model')

plt.xticks(X_axis, X)
plt.xlabel("Each Test Image")
plt.ylabel("PSNR Value")
plt.legend()
plt.show()

X = ['1','2','3','4','5','6','7','8','9']

X_axis = np.arange(len(X))

plt.bar(X_axis - 0.2 , ssims , 0.2 , label = 'SRGAN')
plt.bar(X_axis  , ssime , 0.2 , label = 'ESRGAN')
plt.bar(X_axis + 0.2 , ssimo , 0.2 , label = 'Our Model')

plt.xticks(X_axis, X)
plt.xlabel("Each Test Image")
plt.ylabel("SSIM Value")
plt.legend()
plt.show()


from setuptools import setup

install_requires = [
    "cmdtree>=0.0.3",
    "RPi.GPIO>=0.6.0",
]

entry_points = {
        'console_scripts': ['pi-fan-tuner=pi_fan_tuner:main'],
    }

setup(
    name='PIFanTuner',
    version='0.0.1',
    py_modules=["pi_fan_tuner", ],
    install_requires=install_requires,
    entry_points=entry_points,
    url='https://github.com/winkidney/PIFanTuner',
    license='MIT',
    author='winkidney',
    author_email='winkidney@gmail.com',
    description='RaspberryPI CPU fan tuner with a s8050 Triode',
)


#!/usr/bin/python
# coding: utf-8

import os
import ctypes
import cv2
from glob import iglob

this_dir = os.path.dirname(__file__)

DLL_PATH = os.path.join(this_dir,'lib', 'libtesseract.so' )
TESSDATA_PREFIX = os.path.join(this_dir,'tessdata' )
english_lang = 'eng'
chinese_lang = 'chi_sim'

tesseract_chinese = ctypes.cdll.LoadLibrary(DLL_PATH)
tesseract_chinese.TessBaseAPICreate.restype = ctypes.c_uint64   #由于系统为64位,这里类型需指明为c_uint64
api_chinese = tesseract_chinese.TessBaseAPICreate()

def tess_init_chinese():
    rc = tesseract_chinese.TessBaseAPIInit3(ctypes.c_uint64(api_chinese), TESSDATA_PREFIX, chinese_lang)
    if rc:
        tesseract_chinese.TessBaseAPIDelete(ctypes.c_uint64(api_chinese))
        print('Could not initialize tesseract.\n')
        exit(3)

def tess_recognition_from_file_chinese(img_name):
    tesseract_chinese.TessBaseAPIProcessPages(
        ctypes.c_uint64(api_chinese), img_name, None, 0, None)
    tesseract_chinese.TessBaseAPIGetUTF8Text.restype = ctypes.c_uint64
    text_out = tesseract_chinese.TessBaseAPIGetUTF8Text(ctypes.c_uint64(api_chinese))
    return ctypes.string_at(text_out)

tesseract_english = ctypes.cdll.LoadLibrary(DLL_PATH)
tesseract_english.TessBaseAPICreate.restype = ctypes.c_uint64   #由于系统为64位,这里类型需指明为c_uint64
api_english = tesseract_chinese.TessBaseAPICreate()

def tess_init_english():
    rc = tesseract_english.TessBaseAPIInit3(ctypes.c_uint64(api_english), TESSDATA_PREFIX, english_lang)
    if rc:
        tesseract_english.TessBaseAPIDelete(ctypes.c_uint64(api_english))
        print('Could not initialize tesseract.\n')
        exit(3)

def tess_recognition_from_file_english(img_name):
    tesseract_english.TessBaseAPIProcessPages(
        ctypes.c_uint64(api_english), img_name, None, 0, None)
    tesseract_english.TessBaseAPIGetUTF8Text.restype = ctypes.c_uint64
    text_out = tesseract_english.TessBaseAPIGetUTF8Text(ctypes.c_uint64(api_english))
    return ctypes.string_at(text_out)

def cleanup(temp_name):
    ''' Tries to remove files by filename wildcard path. '''
    for filename in iglob(temp_name + '*' if temp_name else temp_name):
        try:
            os.remove(filename)
        except OSError:
            pass

#Deprecated
def tessRecognition(img, tmp_dir, english = True):
    img_name = os.path.join(tmp_dir, 'img.png')
    cv2.imwrite(img_name, img)
    if english:
        text_out = tess_recognition_from_file_english(img_name)
    else:
        text_out = tess_recognition_from_file_chinese(img_name)
    cleanup(img_name)
    return text_out

def tessRecognitionByFullpath(imgpath, isenglish):
    if isenglish:
        text_out = tess_recognition_from_file_english(imgpath)
    else:
        text_out = tess_recognition_from_file_chinese(imgpath)
    cleanup(imgpath)
    return text_out

# def from_file(path):
#     tesseract.TessBaseAPIProcessPages(
#         ctypes.c_uint64(api), path, None, 0, None)
#     tesseract.TessBaseAPIGetUTF8Text.restype = ctypes.c_uint64
#     text_out = tesseract.TessBaseAPIGetUTF8Text(ctypes.c_uint64(api))
#     return ctypes.string_at(text_out)

# if __name__ == '__main__':
#     image_file_path = b'/home/cvrsg/JpHu/TestGround/tesseract/build/bin/pics/1.png'
#     # result = from_file(image_file_path)
#     tessInit()
#     result = tessRecognition(image_file_path)
#     print(result)


import numpy as np
import cv2 as cv
from time import time

def blur_face_avg(img):
    """ Return a (average) blurred version of the input image. """
    return cv.blur(img, (91, 91))

def show_face(img, window_name="Face"):
    """ Show the image. """
    if img is not None:
        cv.imshow(window_name, img)

def load_face_classifier():
    """ Load and return the haar cascade face classifier. """
    face_cascade = cv.CascadeClassifier('classifier/haarcascade_frontalface_default.xml')
    return face_cascade

def detect_face(face_cascade, gray):
    """ Find faces in the image. If faces are found, the detectMultiScale function returns the positions of
     detection faces as Rect(x, y, w, h). """
    faces = face_cascade.detectMultiScale(gray, 1.3, 5)
    return faces

def show_faces(faces, img):
    """ Show the detected faces with a rectangle on the original image. """
    roi, blur = None, None
    for (x, y, w, h) in faces:
        # Calculate center point and radius for circular masks.
        cx = w // 2
        cy = h // 2
        r = h // 2   # int(((w ** 2 + h ** 2) ** 0.5)/2)  # c² = a² + b², for a bigger circle, the roi itself needs to
        # be bigger which results in inconsistencies.

        # Draw a rectangle/circle on the image.
        # img = cv.rectangle(img, (x, y), (x + w, y + h), (255, 0, 0), 2)
        img = cv.circle(img, (x+cx, y+cy), r, (101, 201, 255), 2)

        # Create a ROI.
        roi = roi_image(x, y, w, h, img)

        # Blur the ROI.
        blur = blur_face_avg(roi)

        # Create masks (normal and inverted).
        mask = np.full((roi.shape[0], roi.shape[1], 1), 0, dtype=np.uint8)  # Init black mask.
        cv.circle(mask, (cx, cy), r, (255, 255, 255), -1)  # Fill in a white circle in the middle to create the mask.
        mask_inverted = 255 - mask

        # Apply the masks.
        fg = cv.bitwise_and(blur, blur, mask=mask)         # Foreground: blurred circle face.
        bg = cv.bitwise_and(roi, roi, mask=mask_inverted)  # Background: part outside the detected face.

        # Add the foreground & background together and insert it into the frame.
        img[y:y+h, x:x+w] = cv.add(fg, bg)

    show_face(img)

def roi_image(x, y, w, h, img):
    """ Return a ROI image based on the input rectangle (x, y, w, h) and the image. """
    return img[y:y+h, x:x+w]

def detect_blur_video():
    # State variable: to blur or not to blur.
    to_blur = True

    # Open a video capture.
    capture = cv.VideoCapture(0, cv.CAP_DSHOW)

    # Load the face classifier.
    face_classifier = load_face_classifier()

    # Get start time.
    time_start = time()

    # Init number of frames variable.
    n_frames = 0
    while True:
        key = 0xFF & cv.waitKey(1)

        # Exit the loop if 'q' is pressed.
        if key == ord('q'):
            break

        # Toggle blurring if 'b' is pressed.
        if key == ord('b'):
            to_blur = not to_blur

        # Get a frame from the video capture.
        ret, frame = capture.read()

        # Continue if can't get a frame from the video capture.
        if not ret:
            continue

        # Convert RGB frame to grayscale.
        gray = cv.cvtColor(frame, cv.COLOR_BGR2GRAY)

        if to_blur:
            # Detect faces.
            face_detections = detect_face(face_classifier, gray)

            # Show detected faces.
            show_faces(face_detections, frame)
        else:
            show_face(frame)

        n_frames += 1

    # Get end time.
    time_end = time()

    # Calculate elapsed time (seconds).
    time_elapsed = time_end - time_start
    # Calculate and print frames per second.
    fps = n_frames / time_elapsed
    print(f"Fps = {fps:.2f}; Time elapsed = {time_elapsed:.2f} seconds.")

    # When everything done, release the capture.
    capture.release()
    cv.destroyAllWindows()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
@author: Kai Borowiak
@summary: TestCase for testlink.objects.TestTestCase
"""

# IMPORTS
import random
import string
import unittest
from mock import patch, Mock

from testlink.objects.tl_testcase import TestCase

def randput(length=10): return "".join([random.choice(string.letters) for _ in xrange(random.randint(1, length))])

def randint(length=10): return int("".join([random.choice(string.digits) for _ in xrange(random.randint(1, length))]))

class TestCaseTests(unittest.TestCase):
    """TestCase Object Tests"""

    def __init__(self, *args, **kwargs):
        super(TestCaseTests, self).__init__(*args, **kwargs)
        self._testMethodDoc = "TestCase: " + self._testMethodDoc

    @patch('testlink.objects.TestCase.getTestProject')
    def test__str__(self, _patch):
        """String representation"""
        ext_id = randint()
        name = randput()

        project = Mock()
        project.prefix = randput()
        _patch.return_value = project

        obj = TestCase(name=name, external_id=ext_id)
        _string = str(obj)
        self.assertEqual(_string, "Testcase %s-%s: %s" % (project.prefix, ext_id, name))


import gzip
import json
import os
import re
import subprocess
import sys
import urllib.request
from urllib.error import HTTPError

import bs4
import filelock

# Global debug flag
DEBUG = 0

def convert_int(input_str, default=0):
    """Convert string to an int with default value if invalid string given

    @param str: String to convert
    @param default: Default value which is zero by default

    @return: an integer, always
    """
    # Try converting to integer
    try:
        i = int(input_str)
    # If it fails, use the default value
    except (ValueError, TypeError):
        i = default
    return i

def send_help():
    """Print help and exit"""
    # pylint: disable=line-too-long

    print("""
Usage: python3 hydrascrape.py <server> <project regexp> <jobset regexp> <handled builds file> <action> [options]

Tries to find builds to handle for specific projects and jobsets from a hydra server
Already handled builds will be read from handled builds file if it exists
Successfully handled builds (action returned 0) will be added to the handled builds file
action will be run with all the build information in the environment

For example, check environment variables starting with \"HYDRA_\":
python3 hydrascrape.py my.hydra.server myproject 'job.*' '.*' myhydra_handled_builds.txt 'env | egrep ^HYDRA_' -debug

Available options:
  -debug  Enable debugging (You can also set DEBUG environment variable to 1)
  -json   Enable JSON output, build info will be written in JSON format to <build ID>.json before running action
  -dp     Include disabled projects in search
  -hp     Include hidden projects in search
  -dj     Include disabled jobsets in search
  -hj     Include hidden jobsets in search
    """)
    sys.exit(0)

def get_page(context, url):
    """Fetches a page from given web site

    @param context: Connection context
    @param url: URL of the page

    @return: page data, on error returns empty page
    """
    if DEBUG:
        print(f"Fetching: {url}")

    req = urllib.request.Request(url, headers=context['headers'])
    try:
        response = urllib.request.urlopen(req)
    except HTTPError as error:
        print(f"HTTP error: {error}", file=sys.stderr)
        return ""

    # Decompress if gzipped content
    if response.info().get('Content-Encoding') == 'gzip':
        text = gzip.decompress(response.read())
    elif response.info().get('Content-Encoding') == 'deflate':
        text = response.read()
    elif response.info().get('Content-Encoding'):
        print('Encoding type unknown', file=sys.stderr)
        return ""
    else:
        text = response.read()

    return text

def get_builds(context, evaluation):
    """Fetches builds for given evaluation

    @param context: Connection context

    @return: a list of build numbers
    """
    text = get_page(context, f"{context['hydra_url']}eval/{evaluation}")
    soup = bs4.BeautifulSoup(text, features="html.parser")

    builds = []
    for link in reversed(soup.find_all("a", {"class": "row-link"})):
        build = convert_int(link.text.strip(), -1)
        if build != -1:
            if DEBUG:
                print(f"Found build {build}")
            builds.append(build)

    if DEBUG and len(builds) == 0:
        print("No builds found")

    return builds

def get_handled(filename):
    """Reads handled builds from handled builds file

    @param filename: Name of the handled builds file

    @return: a list of build numbers
    """
    handled = []
    try:
        with open(filename, "r", encoding="utf-8") as handled_file:
            file_lines = handled_file.read()
            build_ids = file_lines.split("\n")

        for bid in build_ids:
            if bid != "":
                as_int = convert_int(bid, -1)
                if as_int != -1:
                    handled.append(as_int)
                else:
                    print(f"Weird build number in handled builds file: {bid}",
                          file=sys.stderr)

        if DEBUG:
            print(f"handled = {handled}")

    except FileNotFoundError:
        if DEBUG:
            print(f"{filename} not found")

    except PermissionError as perm_error:
        print(f"Cannot read {filename}: {perm_error.strerror}",
              file=sys.stderr)
        sys.exit(1)

    return handled

def update_handled(filename, handled):
    """Writes handled builds into handled builds file

    @param filename: name of the handled builds file
    @param handled: list of handled build numbers
    """
    handled.sort()
    try:
        with open(filename, "w", encoding="utf-8") as handled_file:
            for build in handled:
                handled_file.write(f"{build}\n")

    except PermissionError as perm_error:
        print(
            f"Cannot update {filename}: {perm_error.strerror}", file=sys.stderr)
        sys.exit(1)

def get_postbuild_info(context: dict, log_hash: str, binfo: dict):
    """Get info provided by the Hydra postbuild script"""
    # Get run command log by hash
    text = get_page(context, f"{context['hydra_url']}runcommandlog/{log_hash}")
    # Convert bytes to string
    text = text.decode("utf-8")
    # Create regex pattern that matches simple variable="value" assignments
    pat = re.compile('^.*=".*"$')

    # Handle line by line
    for line in text.split('\n'):
        # Throw away extra spaces
        line = line.strip()
        # If pattern matches store the value in binfo
        if pat.match(line):
            varname = line.split("=")[0].lower().capitalize().replace('_', ' ')
            value = line.split('"')[1]
            binfo[varname] = value

    binfo["RunCommand status"] = "Succeeded"

# Constant data used by following get_build_info function
summarykeys = {
    "Build ID": 0,
    "Status": 1,
    "System": 0,
    "Nix name": 0,
    "Nr": -1,
}
detailkeys = [
    "Queued at",
    "Build started",
    "Build finished",
    "Short description",
    "License",
    "Homepage",
    "Maintainers",
    "Derivation store path",
    "Output store paths",
    "Closure size",
    "Output size",
]

def get_build_info(context, bnum):
    """Fetches information for given build number

    @param context: Connection context
    @param bnum: Build ID

    @return a dictionary with information of the build
    """
    text = get_page(context, f"{context['hydra_url']}build/{bnum}")
    soup = bs4.BeautifulSoup(text, features="html.parser")

    binfo = {}

    for div in soup.find_all("div", {"id": "tabs-summary"}):
        for header in div.find_all("th"):
            hdr = header.text.strip().rstrip(':')
            val = summarykeys.get(hdr)
            if val is not None:
                if val == -1:
                    break

                try:
                    data = header.find_next_sibling().contents[val]
                    if data.name == "time":
                        binfo[hdr] = data['data-timestamp']
                    else:
                        binfo[hdr] = data.text.strip()
                except (IndexError, AttributeError):
                    if DEBUG:
                        print(f"Trouble getting {hdr}")

    for div in soup.find_all("div", {"id": "tabs-details"}):
        for header in div.find_all("th"):
            hdr = header.text.strip().rstrip(':').replace('(', '').replace(')', '')
            if hdr in detailkeys:
                try:
                    data = header.find_next_sibling().contents[0]
                    if data.name == "time":
                        binfo[hdr] = data['data-timestamp']
                    else:
                        val = data.text.split('\n')[0].strip()
                        if val != "not given":
                            binfo[hdr] = val

                except (IndexError, AttributeError):
                    if DEBUG:
                        print(f"Trouble getting {hdr}")

    inputs = []
    for div in soup.find_all("div", {"id": "tabs-buildinputs"}):
        for tbody in div.find_all("tbody"):
            for row in tbody.find_all("tr"):
                tds = row.find_all("td")
                try:
                    input_name = tds[0].text.strip()
                    input_source = tds[2].text.strip()
                    input_hash = tds[3].text.strip()
                    inputs.append({"Name": input_name,
                                   "Hash": input_hash,
                                   "Source": input_source})
                except IndexError:
                    if DEBUG:
                        print("Trouble getting build inputs")
    if len(inputs) > 0:
        binfo['Inputs'] = inputs
    binfo['Output store paths'] = binfo['Output store paths'].split(' ')
    binfo['Job'] = soup.find(
        "div", {"class": "page-header"}).text.split(':')[-1].strip()

    # Find run command log hash entries for build
    loghash = None
    binfo["RunCommand status"] = None
    row = soup.find("div", {"class": "flex-row"})
    if row is not None:
        status = row.find("img", {"class": "build-status"})
        status_text = status.get("title") if status is not None else None
        if status_text != "Succeeded":
            binfo["RunCommand status"] = status_text
        else:
            for link in row.find_all("a", {"class": "btn btn-secondary btn-sm"}):
                href = link.get('href')
                if (
                    href is not None
                    and href.find("/runcommandlog/") != -1
                    and href.endswith("/raw")
                ):
                    loghash = href.split("/")[-2]
                    # break when raw is found we don't need anything else
                    break

    # If found, then process log for post build data
    if loghash is not None:
        get_postbuild_info(context, loghash, binfo)

    return binfo

def set_env(binfo):
    """Sets environment variables based on given dictionary

    @param binfo: Dictionary containing build information

    @return: environment dictionary
    """
    env = os.environ.copy()

    for key in binfo:
        if key == "Output store paths":
            # Set the plain hash of the first output separately
            env["HYDRA_OUTPUT_STORE_HASH"] = binfo[key][0].removeprefix(
                "/nix/store/").split('-', 1)[0]
            env["HYDRA_OUTPUT_STORE_PATHS"] = ' '.join(binfo[key])
            continue
        if key == "Derivation store path":
            # Set the plain hash of the derivation separately
            env["HYDRA_DERIVATION_STORE_HASH"] = binfo[key].removeprefix(
                "/nix/store/").split('-', 1)[0]
        if key == "Inputs":
            env["HYDRA_INPUTS"] = ""
            for i in binfo[key]:
                env["HYDRA_INPUTS"] += f"{i['Name'].upper()} "
                env[f"HYDRA_{i['Name'].upper()}_HASH"] = i['Hash']
                env[f"HYDRA_{i['Name'].upper()}_SOURCE"] = i['Source']
            env["HYDRA_INPUTS"] = env["HYDRA_INPUTS"].strip()
            continue
        env["HYDRA_" + key.upper().replace(' ', '_')] = binfo[key]

    return env

def save_json(binfo):
    """Saves given given build information dictionary in json format

    @param binfo: Dictionary containing build information
    """
    filename = f"{binfo['Build ID']}.json"
    if DEBUG:
        print(f"Writing json info into {filename}")

    json_obj = json.dumps(binfo, indent=2)

    try:
        with open(filename, "w", encoding="utf-8") as outf:
            outf.write(json_obj)

    except PermissionError as perm_error:
        print(f"Could not write {perm_error.filename}: {perm_error.strerror}",
              file=sys.stderr)
        sys.exit(1)

def get_projects(context):
    """Gets projects from a hydra site
    @param context: Connection context
    """
    text = get_page(context, context['hydra_url'])
    soup = bs4.BeautifulSoup(text, features="html.parser")

    projects = soup.find_all("tr", {"class": "project"})
    hiddens = soup.find_all("span", {"class": "hidden-project"})
    disableds = soup.find_all("span", {"class": "disabled-project"})

    plist = []
    for proj in projects:
        plist.append(
            proj.find("a", {"class": "row-link"})["href"].split("/")[-1])

    hlist = []
    for proj in hiddens:
        hlist.append(
            proj.find("a", {"class": "row-link"})["href"].split("/")[-1])

    dlist = []
    for proj in disableds:
        dlist.append(
            proj.find("a", {"class": "row-link"})["href"].split("/")[-1])

    if context['hid_proj'] is False:
        for proj in hlist:
            if proj in plist:
                plist.remove(proj)

    if context['dis_proj'] is False:
        for proj in dlist:
            if proj in plist:
                plist.remove(proj)

    if DEBUG:
        print("Found projects: ", plist)

    return plist

def get_jobsets(context, project):
    """Gets jobsets for a given hydra project

    @param context: Connection context
    """
    text = get_page(context, f"{context['hydra_url']}project/{project}")
    soup = bs4.BeautifulSoup(text, features="html.parser")

    jobsets = soup.find_all("tr", {"class": "jobset"})
    hiddens = soup.find_all("span", {"class": "hidden-jobset"})
    disableds = soup.find_all("span", {"class": "disabled-jobset"})

    jlist = []
    for jobset in jobsets:
        jlist.append(jobset.find(
            "a", {"class": "row-link"})["href"].split("/")[-1])

    hlist = []
    for jobset in hiddens:
        hlist.append(jobset.find(
            "a", {"class": "row-link"})["href"].split("/")[-1])

    dlist = []
    for jobset in disableds:
        dlist.append(jobset.find(
            "a", {"class": "row-link"})["href"].split("/")[-1])

    if context['hid_jobset'] is False:
        for jobset in hlist:
            if jobset in jlist:
                jlist.remove(jobset)

    if context['dis_jobset'] is False:
        for jobset in dlist:
            if jobset in jlist:
                jlist.remove(jobset)

    if DEBUG:
        print("Found jobsets: ", jlist)

    return jlist

def handle_jobset(context, project, jobset, handled):
    """Handles a given jobset of given project

    @param context: Connection context
    @param project: Project name
    @param jobset: Jobset name
    @param handled: list of handled builds
    """
    # It is assumed here that all evaluations containing builds that need processing
    # are found on the first page of evaluations listing
    text = get_page(
        context, f"{context['hydra_url']}jobset/{project}/{jobset}")

    soup = bs4.BeautifulSoup(text, features="html.parser")

    builds = []
    for link in reversed(soup.find_all("a", {"class": "row-link"})):
        # Could also use the link as-is, but just to be safe side we create new link
        builds += get_builds(context, link.text)

    # Remove duplicates
    builds = list(dict.fromkeys(builds))

    new_handled = []

    for i in builds:
        if i not in handled:
            binfo = get_build_info(context, i)
            buildid = convert_int(binfo.get('Build ID'), -1)

            if buildid != i:
                if DEBUG:
                    print(f"Build ID mismatch: build id on page = {buildid}, "
                          f"build id requested = {i}, skipping")
                continue

            status = binfo.get('Status', 'Unknown')

            if status in ("Scheduled to be built", "Build in progress"):
                if DEBUG:
                    print(f"Build {i} not finished yet, skipping")
                continue

            if status == "Success":

                # check postbuild status for succeeded builds only
                runcmd_status = binfo["RunCommand status"]
                if runcmd_status is None:
                    if DEBUG:
                        print(
                            f"RunCommands for build {i} are not finished yet, skipping")
                    continue

                elif runcmd_status != "Succeeded":
                    # if there is a status but it was not success,
                    # there is no need to retry later. just mark as handled.
                    if DEBUG:
                        print(f"RunCommands for build {i} {runcmd_status}, "
                              "marking as handled")
                    new_handled.append(i)
                    continue

                binfo['Server'] = context['server']
                binfo['Project'] = project
                binfo['Jobset'] = jobset
                del binfo['Status']
                env = set_env(binfo)
                if context['json_en']:
                    save_json(binfo)
                if DEBUG:
                    print(f"Handling {i} " + "-" * 60)

                # Run the user specified action with build info in environment
                result = subprocess.run(context['action'], shell=True,
                                        env=env, check=False)

                if result.returncode == 0:
                    new_handled.append(i)
                    if DEBUG:
                        print("Handling successful " + "-" * 55)
                else:
                    if DEBUG:
                        print(f"Action failed with code: {result.returncode}")
            else:
                if DEBUG:
                    print(f"Build {i} has failed, just marking as handled")
                new_handled.append(i)
        else:
            if DEBUG:
                print(f"Build {i} already handled")

    return new_handled

def main_locked(context):
    """locked main program, called only if lock was aqcuired successfully

    @param context: Connection context
    """
    context['hydra_url'] = f"https://{context['server']}/"
    context['headers'] = {
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'Accept-Encoding': 'gzip, deflate',
        'User-Agent': 'hydrascraper.py v1.0'
    }

    handled = get_handled(context['handled_file'])

    projects = get_projects(context)
    projects = list(filter(context['re_p'].match, projects))
    if DEBUG:
        print("Selected projects: ", projects)

    jobsets = {}
    for proj in projects:
        jobsets[proj] = get_jobsets(context, proj)

    for proj in projects:
        jobsets[proj] = list(filter(context['re_js'].match, jobsets[proj]))
        if DEBUG:
            print(f"{proj} selected jobsets: {jobsets[proj]}")

    for project in projects:
        for jobset in jobsets[project]:
            handled += handle_jobset(context, project, jobset, handled)

    update_handled(context['handled_file'], handled)

def json_e(context):
    """Enable JSON output in context

    @param context: Connection context
    """
    context['json_en'] = True
    if DEBUG:
        print("JSON enabled")

def debug_e(_):
    """Set global debug flag"""
    global DEBUG
    DEBUG = 1
    print("Debug enabled")

def hp_e(context):
    """Set hidden projects flag in context

    @param context: Connection context
    """
    context['hid_proj'] = True
    if DEBUG:
        print("Including hidden projects")

def dp_e(context):
    """Set disabled projects flag in context

    @param context: Connection context
    """
    context['dis_proj'] = True
    if DEBUG:
        print("Including disabled projects")

def hj_e(context):
    """Set hidden jobsets flag in context

    @param context: Connection context
    """
    context['hid_jobset'] = True
    if DEBUG:
        print("Including hidden jobsets")

def dj_e(context):
    """Set disabled jobsets flag in context

    @param context: Connection context
    """
    context['dis_jobset'] = True
    if DEBUG:
        print("Including disabled jobsets")

def main(argv):
    """Main function

    @param argv: Command line parameters
    """
    global DEBUG
    # Set debug if set in environment
    DEBUG = convert_int(os.getenv("DEBUG"))

    # Map options to functions setting the flags
    argfu = {"-json": json_e,
             "-debug": debug_e,
             "-hp": hp_e,
             "-dp": dp_e,
             "-hj": hj_e,
             "-dj": dj_e,
             }

    # Default settings in context
    context = {'json_en': False,
               'dis_proj': False,
               'hid_proj': False,
               'dis_jobset': False,
               'hid_jobset': False}

    # Help user, too few arguments given
    if len(argv) < 5:
        send_help()

    context['server'] = argv[0]

    regexes = []
    for i in [0, 1]:
        try:
            # Create regular expression objects of project and jobset strings
            regexes.append(re.compile(argv[i + 1]))
        except re.error as error:
            if error.pos is not None:
                print(argv[i + 1], file=sys.stderr)
                print(" " * error.pos + "^", file=sys.stderr)
            print(f"Regular expression error: {error.msg}", file=sys.stderr)
            sys.exit(1)

    context['re_p'] = regexes[0]
    context['re_js'] = regexes[1]
    context['handled_file'] = argv[3]
    context['action'] = argv[4]

    if len(argv) > 5:
        # Process options
        for i in range(5, len(argv)):
            func = argfu.get(argv[i], None)
            if func is not None:
                func(context)
            else:
                print(f"Invalid argument: {argv[i]}", file=sys.stderr)
                sys.exit(1)

    lock = filelock.FileLock(f"{argv[3]}.lock")
    try:
        lock.acquire(timeout=0)
        main_locked(context)

    except filelock.Timeout as timeout:
        if DEBUG:
            print(f"Unable to get lock {timeout.lock_file}")
            print("If no other hydrascrapers are running, "
                  "check permissions and/or delete the lock file")

    except PermissionError as perm_error:
        print(
            f"Could not aquire {argv[3]}.lock: {perm_error.strerror}", file=sys.stderr)

    finally:
        lock.release()

# Run main when executed from command line
if __name__ == "__main__":
    main(sys.argv[1:])


import os
import re
import math
import sys
from scipy.stats import wasserstein_distance

class CalcJS:
	def __init__(self, fichier_source):
		self.delta = 0.0005
		fp = open (fichier_source, "r")
		self.dict_tokens_source = {}
		cpt = 0
		self.nb_tokens_source = 0
		for ligne in fp:
			if cpt > 2:
				ligne = ligne.rstrip()
				tab_tokens = ligne.split("\t")
				for i in range(len(tab_tokens)):
					if i in self.dict_tokens_source:
						self.dict_tokens_source[i] += int(tab_tokens[i])
					else:
						self.dict_tokens_source[i] = int(tab_tokens[i])
					self.nb_tokens_source += int(tab_tokens[i])
			cpt += 1
		fp.close()

	def calcule_wd(self, indiv):
		dict_tokens_resume = indiv.tokens
		nb_tokens_resume = 0
		for t in dict_tokens_resume:
			nb_tokens_resume += dict_tokens_resume[t]

		taille_vocabulaire_source = len (self.dict_tokens_source)
		taille_vocabulaire_resume = len (dict_tokens_resume)
		#print(dict_tokens_resume)

		#Calcul des probabilites lissees
		self.probas_lissees_resume = self.probas_lissees (dict_tokens_resume, nb_tokens_resume, taille_vocabulaire_resume, self.delta)
		self.probas_lissees_source = self.probas_lissees (self.dict_tokens_source, self.nb_tokens_source, taille_vocabulaire_source, self.delta)
		#print self.probas_lissees_resume
		#print self.probas_lissees_source
		#return self.jensen_shannon (self.probas_lissees_resume, self.probas_lissees_source)
		return wasserstein_distance(list(self.probas_lissees_source.values()),list(self.probas_lissees_resume.values()))
		#return wasserstein_distance(list(self.dict_tokens_source.values()),list(dict_tokens_resume.values()))

	def calcule_js(self, indiv):
		dict_tokens_resume = indiv.tokens
		nb_tokens_resume = 0
		for t in dict_tokens_resume:
			nb_tokens_resume += dict_tokens_resume[t]

		taille_vocabulaire_source = len (self.dict_tokens_source)
		taille_vocabulaire_resume = len (dict_tokens_resume)
		#print(dict_tokens_resume)

		#Calcul des probabilites lissees
		self.probas_lissees_resume = self.probas_lissees (dict_tokens_resume, nb_tokens_resume, taille_vocabulaire_resume, self.delta)
		self.probas_lissees_source = self.probas_lissees (self.dict_tokens_source, self.nb_tokens_source, taille_vocabulaire_source, self.delta)
		#print self.probas_lissees_resume
		#print self.probas_lissees_source
		#self.wd=wasserstein_distance(self.probas_lissees_source.values(),self.probas_lissees_resume.values())
		#return self.jensen_shannon (self.probas_lissees_resume, self.probas_lissees_source)
		return self.jensen_shannon ()

	def calcule_monotonicity(self, indiv):
		dict_tokens_resume = indiv.tokens
		return self.pearson(self.dict_tokens_source,dict_tokens_resume)

	def calcule_monotonicity_w(self, indiv):
		dict_tokens_resume = indiv.tokens
		return self.pearson_weighted(self.dict_tokens_source,dict_tokens_resume)

	def kullback_leibler (self, probas1, probas2):
		kl = 0
		for token in probas1:
			kl += probas1[token] * math.log( probas1[token] / probas2[token] )
		return kl

	def jensen_shannon (self):
		probas_moyenne = {}
		for token in self.probas_lissees_source:
			#print("token"+str(token) + str(self.probas_lissees_source.keys()))
			probas_moyenne[token] = (self.probas_lissees_resume[token] + self.probas_lissees_source[token]) / 2
		#print("------"+str(self.probas_lissees_resume)+"----------")

		return self.kullback_leibler(self.probas_lissees_source, probas_moyenne) + self.kullback_leibler(self.probas_lissees_resume, probas_moyenne ) / 2

	def probas_lissees (self, dict_tokens, nb_tokens, taille_vocabulaire, delta ):
		probas = {}
		somme_probas = 0
		for token in dict_tokens:
			probas[token] = dict_tokens[token] + delta
			probas[token] = probas[token] / (nb_tokens + 1 *   delta * taille_vocabulaire)
			somme_probas += probas[token]
		#print "----------------Somme probas : "+str(somme_probas)+"----------------"
		return probas

	def pearson(self,dic1,dic2):
		ldec1 = sorted(dic1, key=dic1.get, reverse=True)
		#print(ldec1)
		ldec2 = sorted(dic2, key=dic2.get, reverse=True)
		#print(ldec2)
		#remplacer les valeurs de ldec2 par leur index dans ldec1
		indL1=[]
		for l2 in ldec2:
			indL1.append(ldec1.index(l2))
		#print(indL1)
		#compter le nombre d'inversions d'indexes
		count=0
		first = indL1.pop(0)
		index=0
		for l in indL1:
			#print("first:"+str(first)+",l:"+str(l))
			if first > l:
				count=count+1
			first = l
			index = index + 1
		return(count)

	def pearson_weighted(self,dic1,dic2):
		ldec1 = sorted(dic1, key=dic1.get, reverse=True)
		#print(ldec1)
		ldec2 = sorted(dic2, key=dic2.get, reverse=True)
		#print(ldec2)
		#remplacer les valeurs de ldec2 par leur index dans ldec1
		indL1=[]
		for l2 in ldec2:
			indL1.append(ldec1.index(l2))
		#print(indL1)
		#compter le nombre d'inversions d'indexes
		count=0
		first = indL1.pop(0)
		index=1
		for l in indL1:
			#print("first:"+str(first)+",l:"+str(l))
			if first > l:
				return l
				#count=count+index
			first = l
			index = index + 1
		return(count)

#out_ws.txt

#"python ../Genetic/algo_ge.py ../TAC/u08_corr/D0802/D0802-A/concat-D0802.txt out_ws.txt
#	python rouge.py out_ws.txt ../TAC/u08_corr/D0802 >> stat_ws.txt


from sklearn.linear_model import LogisticRegression
import numpy as np
import csv
from sklearn import preprocessing
from sklearn.externals import joblib

def read_csv(filename):
  """
  - params filename: filename of the file
  - params type: string
  - return: numpy ndarray
  """
  with open(filename, mode='r') as f:
    csv_reader = csv.reader(f)
    points = []
    labels = []
    for row in csv_reader:
      row = list(map(lambda x: float(x), row))
      points.append(row[:len(row) - 1])
      labels.append([row[-1]])
    points = np.array(points)
    labels = np.array(labels)
  return points, labels

def load_dataset(files):
  data, labels = None, None
  for filename in files:
    temp_point, temp_label = read_csv(filename)
    if data is None:
      data, labels = temp_point, temp_label
    else:
      data = np.vstack((data, temp_point))
      labels = np.vstack((labels, temp_label))
  return data, labels

def train_test():
  all_acc = []
  for idx in range(5):
    train_files = ['../data/fold-{}.csv'.format(i) for i in range(5)]
    train_files = list(filter(lambda x: x != '../data/fold-{}.csv'.format(idx), train_files))
    test_files = ['../data/fold-{}.csv'.format(idx)]
    train_data, train_labels = load_dataset(train_files)
    test_data, test_labels = load_dataset(test_files)
    clf = LogisticRegression(random_state=0, solver='lbfgs', multi_class='multinomial').fit(train_data, train_labels)
    result = clf.predict(test_data)
    result = result.reshape((-1, 1))
    acc = result == test_labels
    acc = acc.sum() / test_labels.shape[0]
    all_acc.append(acc)
  print(sum(all_acc) / len(all_acc))

def train_model():
  train_files = ['../data/fold-{}.csv'.format(i) for i in range(5)]
  train_data, train_labels = load_dataset(train_files)
  result = []
  idx = 0
  with open('../data/test set.csv', 'r') as f:
    csv_reader = csv.reader(f)
    for row in csv_reader:
      if idx > 0:
        row = list(map(lambda x: float(x), row))
        result.append(row)
      idx += 1
  result = np.array(result)
  result = preprocessing.scale(result)
  clf = LogisticRegression(random_state=0, solver='lbfgs', multi_class='multinomial').fit(train_data, train_labels)
  predicted = clf.predict(result)
  predicted = predicted.reshape((-1, 1))
  with open('../result/result.csv', 'w', newline='') as f:
    csv_writer = csv.writer(f)
    csv_writer.writerow(['id', 'predicted'])
    for i in range(predicted.shape[0]):
      csv_writer.writerow([i + 1, predicted[i, 0]])

if __name__ == "__main__":
  # train_test()
  train_model()