##############################################################################
# Copyright 2016-2017 Rigetti Computing
#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.
##############################################################################

import pyquil.quil as pq
from pyquil.gates import H
import numpy as np
from math import log
from grove.qft.fourier import qft

def controlled(m):
    """
    Make a one-qubit-controlled version of a matrix.

    :param m: (numpy.ndarray) A matrix.
    :return: A controlled version of that matrix.
    """
    rows, cols = m.shape
    assert rows == cols
    n = rows
    I = np.eye(n)
    Z = np.zeros((n, n))
    controlled_m = np.bmat([[I, Z],
                            [Z, m]])
    return controlled_m

def phase_estimation(U, accuracy, reg_offset=0):
    """
    Generate a circuit for quantum phase estimation.

    :param U: (numpy.ndarray) A unitary matrix.
    :param accuracy: (int) Number of bits of accuracy desired.
    :param reg_offset: (int) Where to start writing measurements (default 0).
    :return: A Quil program to perform phase estimation.
    """
    assert isinstance(accuracy, int)
    rows, cols = U.shape
    m = int(log(rows, 2))
    output_qubits = range(0, accuracy)
    U_qubits = range(accuracy, accuracy + m)
    p = pq.Program()

    # Hadamard initialization
    for i in output_qubits:
        p.inst(H(i))
    # Controlled unitaries
    for i in output_qubits:
        if i > 0:
            U = np.dot(U, U)
        cU = controlled(U)
        name = "CONTROLLED-U{0}".format(2 ** i)
        # define the gate
        p.defgate(name, cU)
        # apply it
        p.inst((name, i) + tuple(U_qubits))
    # Compute the QFT
    p = p + qft(output_qubits)
    # Perform the measurements
    for i in output_qubits:
        p.measure(i, reg_offset + i)

    return p

if __name__ == '__main__':
    import pyquil.api as api
    qvm = api.SyncConnection()
    X = np.asarray([[0.0, 1.0], [1.0, 0.0]])
    Y = np.asarray([[0.0, -1.0j], [1.0j, 0.0]])
    Rx = np.exp(X * np.pi / 8)
    Ry = np.exp(Y * np.pi / 16)
    U = np.kron(Rx, Ry)
    p = phase_estimation(U, 3)
    print(p)
    print(qvm.run(p, range(3+2), 10))
    print(qvm.wavefunction(p))


# uncompyle6 version 3.7.4
# Python bytecode 2.5 (62131)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: /Users/garethr/Documents/Projects/src/network/configs/common/settings.py
# Compiled at: 2009-06-25 09:59:29
import os, django
DJANGO_ROOT = os.path.dirname(os.path.realpath(django.__file__))
SITE_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
DEBUG = True
TEMPLATE_DEBUG = DEBUG
ADMINS = ()
MANAGERS = ADMINS
DATABASE_ENGINE = 'sqlite3'
DATABASE_NAME = os.path.join(SITE_ROOT, 'db') + '/development.db'
TIME_ZONE = 'Europe/London'
LANGUAGE_CODE = 'en-gb'
SITE_ID = 1
USE_I18N = False
MEDIA_ROOT = os.path.join(SITE_ROOT, 'assets')
MEDIA_URL = ''
ADMIN_MEDIA_PREFIX = '/media/'
SECRET_KEY = 'yqci=(=-#y#_=-!#rl_9!0z+^n=+c+gb-#w1i6s7!knoc9b1oy'
TEMPLATE_LOADERS = ('django.template.loaders.filesystem.load_template_source', 'django.template.loaders.app_directories.load_template_source',
                    'django.template.loaders.eggs.load_template_source')
MIDDLEWARE_CLASSES = ('django.middleware.common.CommonMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware',
                      'django.contrib.auth.middleware.AuthenticationMiddleware')
ROOT_URLCONF = 'network.configs.common.urls'
TEMPLATE_DIRS = os.path.join(SITE_ROOT, 'templates')
INSTALLED_APPS = ('django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions',
                  'django.contrib.sites', 'django.contrib.admin', 'django.contrib.admindocs',
                  'django.contrib.humanize', 'registration', 'clue', 'tagging', 'profiles',
                  'home')
AUTH_PROFILE_MODULE = 'profiles.profile'
ACCOUNT_ACTIVATION_DAYS = 7
LOGIN_REDIRECT_URL = '/'
EMAIL_HOST = 'localhost'
EMAIL_PORT = 1025
DEFAULT_FROM_EMAIL = 'webmaster@localhost'
try:
    from local_settings import *
except ImportError:
    pass


from cloupy.scraping import imgw
import pytest
import urllib.request
import urllib.error

def check_if_NOT_connected_to_the_internet(host='http://google.com'):
    try:
        urllib.request.urlopen(host)
        return False
    except urllib.error.URLError:
        return True

@pytest.mark.filterwarnings("ignore::pandas.errors.DtypeWarning")
@pytest.mark.skipif(check_if_NOT_connected_to_the_internet(), reason='internet connection required')
class TestDataDownloading:

    @pytest.fixture
    def intervals(self):
        return ['monthly', 'daily', 'prompt']

    @pytest.fixture
    def st_kinds(self):
        return ['synop', 'climat', 'fall']

    def test_if_column_2_is_always_year(
            self, intervals, st_kinds
    ):

        from os import listdir
        from os.path import isfile, join
        import shutil
        from random import shuffle

        y_range = range(2018, 2019)
        files_reading_dir_path = str(__file__).replace(
            join('test', 'test_integration', 'test_integration_imgw.py'),
            join('scraping', 'files_reading_folder')
        )

        for interval in intervals:
            for st_kind in st_kinds:

                if st_kind == 'fall' and interval == 'prompt':
                    continue

                urls = imgw.get_urls(interval, st_kind, y_range)
                imgw.download_data(urls)
                downloaded_files_names = [f for f in listdir(files_reading_dir_path) if
                                          isfile(join(files_reading_dir_path, f))]

                file_formats = imgw.get_file_formats(interval, st_kind, 'all')
                keywords = ['nazwa stacji', 'temperatura', 'rok', 'opad', 'wiatr']
                shuffle(keywords)

                for file in file_formats:

                    if isinstance(file_formats, str):
                        file = file_formats

                    df = imgw.concatenate_data(
                        downloaded_files_names=downloaded_files_names, file_formats=file, years_range=y_range,
                        keywords=keywords, specific_columns=None, optimize_memory_usage=False,
                        merge_splitted_stations=True
                    )

                    df = df[0][df[1]]

                    assert min(df[2]) == 2018

                shutil.rmtree(files_reading_dir_path)

    def test_data_downloading_for_years_before_2001(
            self, intervals, st_kinds
    ):
        years_range = range(1984, 1987)
        TestDataDownloading.download_and_test_data(intervals, st_kinds, years_range)

    def test_data_downloading_for_years_after_2000(
            self, intervals, st_kinds
    ):
        years_range = range(2011, 2013)
        TestDataDownloading.download_and_test_data(intervals, st_kinds, years_range)

    def test_data_downloading_for_years_between_2000_and_2001(
            self, intervals, st_kinds
    ):
        years_range = range(2000, 2002)
        TestDataDownloading.download_and_test_data(intervals, st_kinds, years_range)

    def test_adding_coordinates_to_dataframe(
            self, intervals, st_kinds
    ):
        years_range = range(2010, 2011)
        for interval in intervals:
            for st_kind in st_kinds:

                if st_kind == 'fall' and interval == 'prompt':
                    continue

                df = imgw.download_imgw_climatological_data(
                    interval, st_kind, years_range,
                    specific_columns=[0, 1, 2, 3],
                    optimize_memory_usage=True,
                    return_coordinates=True
                )

                assert 'lat' in df.columns
                assert 'lon' in df.columns
                assert 'elv' in df.columns

                assert not df['lat'].isnull().all()
                assert not df['lon'].isnull().all()
                assert not df['elv'].isnull().all()

    @staticmethod
    def download_and_test_data(
            intervals, st_kinds, years_range
    ):
        for interval in intervals:
            for st_kind in st_kinds:
                if interval == 'prompt' and st_kind == 'fall':
                    with pytest.raises(NotADirectoryError):
                        imgw.download_imgw_climatological_data(
                            interval, st_kind, years_range
                        )
                        continue
                else:
                    df = imgw.download_imgw_climatological_data(
                        interval, st_kind, years_range,
                        optimize_memory_usage=True,
                        specific_columns=[0, 1, 2, 3]
                    )

                assert not df.empty

{'listdir': 'os.listdir', 'isfile': 'os.path.isfile', 'join': 'os.path.join', 'shutil': 'shutil', 'shuffle': 'random.shuffle'}.download_and_test_data


"""Tests for the lastfm sensor."""
from unittest.mock import patch

from pylast import Track
import pytest

from homeassistant.components import sensor
from homeassistant.components.lastfm.const import STATE_NOT_SCROBBLING
from homeassistant.core import HomeAssistant
from homeassistant.setup import async_setup_component

class MockNetwork:
    """Mock _Network object for pylast."""

    def __init__(self, username: str):
        """Initialize the mock."""
        self.username = username

class MockUser:
    """Mock User object for pylast."""

    def __init__(self, now_playing_result):
        """Initialize the mock."""
        self._now_playing_result = now_playing_result
        self.name = "test"

    def get_playcount(self):
        """Get mock play count."""
        return 1

    def get_image(self):
        """Get mock image."""

    def get_recent_tracks(self, limit):
        """Get mock recent tracks."""
        return []

    def get_top_tracks(self, limit):
        """Get mock top tracks."""
        return []

    def get_now_playing(self):
        """Get mock now playing."""
        return self._now_playing_result

@pytest.fixture(name="lastfm_network")
def lastfm_network_fixture():
    """Create fixture for LastFMNetwork."""
    with patch(
        "homeassistant.components.lastfm.sensor.LastFMNetwork"
    ) as lastfm_network:
        yield lastfm_network

async def test_update_not_playing(hass: HomeAssistant, lastfm_network) -> None:
    """Test update when no playing song."""

    lastfm_network.return_value.get_user.return_value = MockUser(None)

    assert await async_setup_component(
        hass,
        sensor.DOMAIN,
        {"sensor": {"platform": "lastfm", "api_key": "secret-key", "users": ["test"]}},
    )
    await hass.async_block_till_done()

    entity_id = "sensor.test"

    state = hass.states.get(entity_id)

    assert state.state == STATE_NOT_SCROBBLING

async def test_update_playing(hass: HomeAssistant, lastfm_network) -> None:
    """Test update when song playing."""

    lastfm_network.return_value.get_user.return_value = MockUser(
        Track("artist", "title", MockNetwork("test"))
    )

    assert await async_setup_component(
        hass,
        sensor.DOMAIN,
        {"sensor": {"platform": "lastfm", "api_key": "secret-key", "users": ["test"]}},
    )
    await hass.async_block_till_done()

    entity_id = "sensor.test"

    state = hass.states.get(entity_id)

    assert state.state == "artist - title"


import networkx as nx
import pandas as pd
from node2vec import Node2Vec
from gensim.models import KeyedVectors
from collections import defaultdict
from EvalUtils import EvalUtils

#g = nx.read_weighted_edgelist("../datasets/redditdataset_75.txt", create_using=nx.DiGraph())
missed = 0
df = pd.read_csv('../datasets/collegedataset.txt', names = ['v1','v2','timestamp'],sep = '\t',lineterminator='\n',header = None, dtype = str)
sumpred = 0
summap = 0
avg_f1 = 0
for i in range(1,5):

    try:

        dftrain = df[:int(i*len(df)/5)]
        dftest =  df[int(i*len(df)/5 + 1):]

        graph = nx.from_pandas_edgelist(dftrain,source='v1',
                                           target='v2',edge_attr='timestamp',
                                           create_using=nx.DiGraph())

        testgraph = nx.from_pandas_edgelist(dftest, source='v1',
                                        target='v2', edge_attr='timestamp',
                                        create_using=nx.DiGraph())

        actual = set((edge[0], edge[1]) for edge in testgraph.edges())
        nodes = graph.nodes

        node2vec = Node2Vec(graph, num_walks=10, walk_length=8)

        model = node2vec.fit(window=10, min_count=1, batch_words=4)  # Any keywords acceptable by gensim.Word2Vec can be passed, `diemnsions` and `workers` are automatically passed (from the Node2Vec constructor)
        predset = set()
        for node in nodes:
            listobj = model.wv.most_similar(node)[0][0]
            pred_edge = (node, listobj)
            predset.add(pred_edge)

        precision = len(set(actual).intersection(predset)) / len(predset)
        recall = len(set(actual).intersection(predset)) / len(actual)

        f1_score = (2 * precision * recall) / (precision + recall)
        print("p",precision)
        print("r",recall)
        print ("f1",f1_score)

        avg_f1 += f1_score

    except KeyError:
        missed+=1
        continue

print("avg f1:", avg_f1 / 5)


# -*- Mode: Python -*-
# vi:si:et:sw=4:sts=4:ts=4
import sqlite3
import operator
import types

from zope.interface import implements
from twisted.enterprise import adbapi
from twisted.spread import pb
from twisted.internet import reactor
from twisted.python import log as twisted_log

from feat.common import (log, text_helper, error_handler, defer,
                         formatable, enum, decorator, time, manhole,
                         fiber, signal, )
from feat.agencies import common
from feat.common.serialization import banana
from feat.extern.log import log as flulog

from feat.interface.journal import *
from feat.interface.serialization import *
from feat.agencies.interface import *
from feat.interface.log import *

class State(enum.Enum):

    '''
    disconnected - there is no connection to database
    connected - connection is ready, entries can be insterted
    '''
    (disconnected, connected, ) = range(2)

class EntriesCache(object):
    '''
    Helper class storing the data and giving the back in transactional way.
    '''

    def __init__(self):
        self._cache = list()
        self._fetched = None

    def append(self, entry):
        self._cache.append(entry)

    def fetch(self):
        '''
        Gives all the data it has stored, and remembers what it has given.
        Later we need to call commit() to actually remove the data from the
        cache.
        '''
        if self._fetched is not None:
            raise RuntimeError('fetch() was called but the previous one has '
                               'not yet been applied. Not supported')
        if self._cache:
            self._fetched = len(self._cache)
        return self._cache[0:self._fetched]

    def commit(self):
        '''
        Actually remove data returned by fetch() from the cache.
        '''
        if self._fetched is None:
            raise RuntimeError('commit() was called but nothing was fetched')
        self._cache = self._cache[self._fetched:]
        self._fetched = None

    def rollback(self):
        if self._fetched is None:
            raise RuntimeError('rollback() was called but nothing was fetched')
        self._fetched = None

    def is_locked(self):
        '''
        Tells if we are currently in the locked state (need commit or rollback)
        '''
        return self._fetched is not None

    def __len__(self):
        return len(self._cache)

@decorator.parametrized_function
def in_state(func, *states):

    def wrapper(self, *args, **kwargs):
        d = defer.succeed(None)
        if not self._cmp_state(states):
            d.addCallback(defer.drop_param, self.wait_for_state, *states)
        d.addCallback(defer.drop_param, func, self, *args, **kwargs)
        return d

    return wrapper

class Journaler(log.Logger, common.StateMachineMixin):
    implements(IJournaler, ILogKeeper)

    log_category = 'journaler'

    _error_handler = error_handler

    # FIXME: at some point switch to False and remove this attribute
    should_keep_on_logging_to_flulog = True

    def __init__(self, logger):
        log.Logger.__init__(self, self)

        common.StateMachineMixin.__init__(self, State.disconnected)
        self._writer = None
        self._flush_task = None
        self._cache = EntriesCache()
        self._notifier = defer.Notifier()

    def configure_with(self, writer):
        self._ensure_state(State.disconnected)
        twisted_log.addObserver(self.on_twisted_log)
        self._writer = IJournalWriter(writer)
        self._set_state(State.connected)
        self._schedule_flush()

    def close(self, flush_writer=True):

        def set_disconnected():
            self._writer = None
            self._set_state(State.disconnected)

        try:
            twisted_log.removeObserver(self.on_twisted_log)
        except ValueError:
            # it should be safe to call close() multiple times,
            # in this case we are not registered as the observer anymore
            pass

        d = self._close_writer(flush_writer)
        d.addCallback(defer.drop_param, set_disconnected)
        return d

    ### IJournaler ###

    def get_connection(self, externalizer):
        externalizer = IExternalizer(externalizer)
        instance = JournalerConnection(self, externalizer)
        return instance

    def prepare_record(self):
        return Record(self)

    @in_state(State.connected)
    def get_histories(self):
        return self._writer.get_histories()

    @in_state(State.connected)
    def get_entries(self, history):
        return self._writer.get_entries(history)

    def insert_entry(self, **data):
        self._cache.append(data)
        self._schedule_flush()
        return self._notifier.wait('flush')

    @in_state(State.connected)
    def get_filename(self):
        return self._writer.get_filename()

    def is_idle(self):
        if len(self._cache) > 0:
            return False
        if self._writer:
            return self._writer.is_idle()
        return True

    ### ILogObserver provider ###

    def on_twisted_log(self, event_dict):
        edm = event_dict['message']
        if not edm:
            if event_dict['isError'] and 'failure' in event_dict:
                fail = event_dict['failure']
                self.error("A twisted traceback occurred. Exception: %r.",
                           fail.value)
                if flulog.getCategoryLevel("twisted") < flulog.WARN:
                    self.debug(
                        "Run with debug level >= 2 to see the traceback.")
                else:
                    self.error("%s", fail.getTraceback())

    ### ILogKeeper Methods ###

    def do_log(self, level, object, category, format, args,
               depth=-1, file_path=None, line_num=None):
        level = int(level)
        if category is None:
            category = 'feat'
        if level > flulog.getCategoryLevel(category):
            return

        if file_path is None and line_num is None:
            file_path, line_num = flulog.getFileLine(where=-depth-2)

        if args:
            message = format % args
        else:
            message = str(format)

        data = dict(
            entry_type='log',
            level=level,
            log_name=object,
            category=category,
            file_path=file_path,
            line_num=line_num,
            message=message)
        self.insert_entry(**data)

        if self.should_keep_on_logging_to_flulog:
            flulog.doLog(level, object, category, format, args,
                         where=depth, filePath=file_path, line=line_num)

    ### private ###

    def _schedule_flush(self):
        if self._flush_task is None:
            self._flush_task = time.callLater(0, self._flush)

    @in_state(State.connected)
    def _flush(self):
        entries = self._cache.fetch()
        if entries:
            d = self._writer.insert_entries(entries)
            d.addCallbacks(defer.drop_param, self._flush_error,
                           callbackArgs=(self._flush_complete, ))
            return d
        else:
            self._flush_complete()

    def _flush_complete(self):
        if self._cache.is_locked():
            self._cache.commit()
        self._flush_task = None
        self._notifier.callback('flush', None)
        if len(self._cache) > 0:
            self._schedule_flush()

    def _flush_error(self, fail):
        self._cache.rollback()
        fail.raiseException()

    def _close_writer(self, flush_writer=True):
        d = defer.succeed(None)
        if self._writer:
            d.addCallback(defer.drop_param, self._writer.close,
                          flush=flush_writer)
        return d

class BrokerProxyWriter(log.Logger, common.StateMachineMixin):
    implements(IJournalWriter)

    _error_handler = error_handler

    def __init__(self, broker):
        '''
        @param encoding: Optional encoding to be used for blob fields.
        @type encoding: Should be a valid parameter for str.encode() method.
        @param filename: File to use for entries. Defaults to :memory:
        @param logger: ILogger to use
        '''
        log.Logger.__init__(self, broker)
        common.StateMachineMixin.__init__(self, State.disconnected)

        self._broker = broker
        self._set_writer(None)
        self._cache = EntriesCache()
        self._semaphore = defer.DeferredSemaphore(1)

    def initiate(self):
        d = self._broker.get_journal_writer()
        d.addCallback(self._set_writer)
        d.addCallback(defer.drop_param, self._set_state, State.connected)
        return d

    def close(self, flush=True):
        d = defer.succeed(None)
        if flush:
            d.addCallback(defer.drop_param, self._flush_next)
        d.addCallback(defer.drop_param, self._set_state, State.disconnected)
        d.addCallback(defer.drop_param, self._set_writer, None)
        return d

    @in_state(State.connected)
    def get_histories(self):
        return self._writer.callRemote('get_histories')

    @in_state(State.connected)
    def get_entries(self, history):
        return self._writer.callRemote('get_entries', history)

    def insert_entries(self, entries):
        for data in entries:
            self._cache.append(data)
        return self._flush_next()

    @in_state(State.connected)
    def get_filename(self):
        return self._writer.callRemote('get_filename')

    def is_idle(self):
        if len(self._cache) > 0:
            return False
        return True

    ### private ###

    @in_state(State.connected)
    def _flush_next(self):
        if len(self._cache) == 0:
            return defer.succeed(None)
        else:
            d = self._semaphore.run(self._push_entries)
            d.addCallback(defer.drop_param, self._flush_next)
            return d

    def _push_entries(self):
        entries = self._cache.fetch()
        if entries:
            try:
                d = self._writer.callRemote('insert_entries', entries)
                d.addCallbacks(defer.drop_param, defer.drop_param,
                               callbackArgs=(self._cache.commit, ),
                               errbackArgs=(self._cache.rollback, ))
                return d
            except pb.DeadReferenceError:
                # for some reason callRemote raises this error
                # instead of giving failed Deferred
                self._cache.rollback()

    def _set_writer(self, writer):
        self._writer = writer
        if isinstance(self._writer, pb.RemoteReference):
            self._writer.notifyOnDisconnect(self._on_disconnect)

    def _on_disconnect(self, writer):
        self._set_state(State.disconnected)
        self._set_writer(None)

class SqliteWriter(log.Logger, log.LogProxy, common.StateMachineMixin,
                   manhole.Manhole):
    implements(IJournalWriter)

    _error_handler = error_handler

    def __init__(self, logger, filename=":memory:", encoding=None,
                 on_rotate=None):
        '''
        @param encoding: Optional encoding to be used for blob fields.
        @type encoding: Should be a valid parameter for str.encode() method.
        @param filename: File to use for entries. Defaults to :memory:
        @param logger: ILogger to use
        '''
        log.Logger.__init__(self, logger)
        log.LogProxy.__init__(self, logger)
        common.StateMachineMixin.__init__(self, State.disconnected)

        self._encoding = encoding
        self._db = None
        self._filename = filename
        self._reset_history_id_cache()
        self._cache = EntriesCache()
        # the semaphore is used to always have at most running
        # .perform_instert() method
        self._semaphore = defer.DeferredSemaphore(1)

        self._sighup_installed = False

        self._on_rotate_cb = on_rotate

    def initiate(self):
        self._db = adbapi.ConnectionPool('sqlite3', self._filename,
                                         cp_min=1, cp_max=1, cp_noisy=True,
                                         check_same_thread=False,
                                         timeout=10)
        self._install_sighup()
        return self._check_schema()

    ### IJournalWriter ###

    def close(self, flush=True):
        d = defer.succeed(None)
        if self._cmp_state(State.disconnected):
            return d
        if flush:
            d.addCallback(defer.drop_param, self._flush_next)
        d.addCallback(defer.drop_param, self._db.close)
        d.addCallback(defer.drop_param, self._uninstall_sighup)
        d.addCallback(defer.drop_param, self._set_state,
                      State.disconnected)
        return d

    @manhole.expose()
    @in_state(State.connected)
    def get_histories(self):
        return History.fetch(self._db)

    @manhole.expose()
    @in_state(State.connected)
    def get_entries(self, history, start_date=0, limit=None):
        '''
        Returns a list of journal entries  for the given history_id.
        '''
        if not isinstance(history, History):
            raise AttributeError(
                'First paremeter is expected to be History instance, got %r'
                % history)

        command = text_helper.format_block("""
        SELECT histories.agent_id,
               histories.instance_id,
               entries.journal_id,
               entries.function_id,
               entries.fiber_id,
               entries.fiber_depth,
               entries.args,
               entries.kwargs,
               entries.side_effects,
               entries.result,
               entries.timestamp
          FROM entries
          LEFT JOIN histories ON histories.id = entries.history_id
          WHERE entries.history_id = ?""")
        if start_date:
            command += " AND entries.timestamp >= %s" % (start_date, )
        command += " ORDER BY entries.rowid ASC"
        if limit:
            command += " LIMIT %s" % (limit, )
        d = self._db.runQuery(command, (history.history_id, ))
        d.addCallback(self._decode)
        return d

    @in_state(State.connected)
    def get_log_entries(self, start_date=None, end_date=None, filters=list()):
        '''
        @param start_date: epoch time to start search
        @param end_date: epoch time to end search
        @param filters: list of dictionaries with the following keys:
                        level - mandatory, display entries with lvl <= level
                        category - optional, limit to log_category
                        name - optional, limit to log_name
                        Leaving optional fields blank will match all the
                        entries. The entries in this list are combined with
                        OR operator.
        '''
        query = text_helper.format_block('''
        SELECT logs.message,
               logs.level,
               logs.category,
               logs.log_name,
               logs.file_path,
               logs.line_num,
               logs.timestamp
        FROM logs
        WHERE 1
        ''')
        query = self._add_timestamp_condition_sql(query, start_date, end_date)

        def transform_filter(filter):
            level = filter.get('level', None)
            category = filter.get('category', None)
            name = filter.get('name', None)
            if level is None:
                raise AttributeError("level is mandatory parameter.")
            resp = "(logs.level <= %d" % (int(level), )
            if category is not None:
                resp += " AND logs.category == '%s'" % (category, )
            if name is not None:
                resp += " AND logs.log_name == '%s'" % (name, )
            resp += ')'
            return resp

        filter_strings = map(transform_filter, filters)
        if filter_strings:
            query += " AND (%s)\n" % (' OR '.join(filter_strings), )
        d = self._db.runQuery(query)
        d.addCallback(self._decode)
        return d

    @in_state(State.connected)
    def get_log_categories(self, start_date=None, end_date=None):
        '''
        @param start_date: epoch time to start search
        @param end_date: epoch time to end search
        '''
        query = text_helper.format_block('''
        SELECT DISTINCT logs.category
        FROM logs
        WHERE 1
        ''')
        query = self._add_timestamp_condition_sql(query, start_date, end_date)
        d = self._db.runQuery(query)

        def unpack(res):
            return map(operator.itemgetter(0), res)

        d.addCallback(unpack)
        return d

    def get_log_names(self, category, start_date=None, end_date=None):
        '''
        Fetches log names for the given category.
        @param start_date: epoch time to start search
        @param end_date: epoch time to end search
        '''
        query = text_helper.format_block('''
        SELECT DISTINCT logs.log_name
        FROM logs
        WHERE category = ?
        ''')
        query = self._add_timestamp_condition_sql(query, start_date, end_date)
        d = self._db.runQuery(query, (category, ))

        def unpack(res):
            return map(operator.itemgetter(0), res)

        d.addCallback(unpack)
        return d

    def get_log_time_boundaries(self):
        '''
        @returns: a tuple of log entry timestaps (first, last) or None
        '''
        query = text_helper.format_block('''
        SELECT min(logs.timestamp),
               max(logs.timestamp)
        FROM logs''')

        def unpack(res):
            if res:
                return res[0]

        d = self._db.runQuery(query)
        d.addCallback(unpack)
        return d

    @manhole.expose()
    def insert_entries(self, entries):
        for data in entries:
            self._cache.append(data)
        return self._flush_next()

    @manhole.expose()
    def get_filename(self):
        return self._filename

    def is_idle(self):
        if len(self._cache) > 0:
            return False
        return True

    ### Private ###

    def _add_timestamp_condition_sql(self, query, start_date, end_date):
        if start_date is not None:
            query += "  AND logs.timestamp >= %d\n" % (int(start_date), )
        if end_date is not None:
            query += "  AND logs.timestamp <= %d\n" % (int(end_date), )
        return query

    def _reset_history_id_cache(self):
        # (agent_id, instance_id, ) -> history_id
        self._history_id_cache = dict()

    def _sighup_handler(self, signum, frame):
        self.log("Received SIGHUP, reopening the journal.")
        self.close()
        self.initiate()
        if callable(self._on_rotate_cb):
            self._on_rotate_cb()

    def _install_sighup(self):
        if self._sighup_installed:
            return
        if self._filename == ':memory:':
            return
        self.log('Installing SIGHUP handler.')
        handler = signal.signal(signal.SIGHUP, self._sighup_handler)
        self._sighup_installed = True

    def _uninstall_sighup(self):
        if not self._sighup_installed:
            return

        try:
            signal.unregister(signal.SIGHUP, self._sighup_handler)
            self.log("Uninstalled SIGHUP handler.")
        except ValueError:
            self.warning("Unregistering of sighup failed. Straaange!")
        self._sighup_installed = False

    def _decode(self, entries):
        '''
        Takes the list of rows returned by sqlite.
        Returns rows in readable format.
        '''

        def decode_blobs(row):
            row = list(row)
            for index, value in zip(range(len(row)), row):
                if isinstance(value, types.BufferType):
                    value = str(value)
                    if self._encoding:
                        value = value.decode(self._encoding)
                    row[index] = value
            return row

        return map(decode_blobs, entries)

    def _encode(self, data):
        result = dict()

        if data['entry_type'] == 'journal':
            to_copy = ('fiber_depth', 'instance_id', 'entry_type')
            to_decode = ('agent_id', 'function_id', 'fiber_id', )
            to_blob = ('journal_id', 'args', 'kwargs', 'side_effects',
                       'result', )
        elif data['entry_type'] == 'log':
            to_copy = ('level', 'log_name', 'category', 'line_num',
                       'entry_type')
            to_decode = ('file_path', )
            to_blob = ('message', )
        else:
            raise RuntimeError('Unknown entry type %r' % data['entry_type'])

        # just copy, caring open escapes
        for key in to_copy:
            result[key] = data[key]

        for key in to_decode:
            if data[key] is None:
                data[key] = ""
            result[key] = data[key].decode("utf-8")

        # encode the blobs
        for key in to_blob:
            safe = data[key]
            if self._encoding:
                safe = safe.encode(self._encoding)
            result[key] = sqlite3.Binary(safe)

        return result

    def _check_schema(self):
        d = self._db.runQuery(
            'SELECT value FROM metadata WHERE name = "encoding"')
        d.addCallbacks(self._got_encoding, self._create_schema)
        return d

    def _got_encoding(self, res):
        encoding = res[0][0]
        if encoding == 'None':
            encoding = None
        if self._encoding is not None and encoding != self._encoding:
            self.warning("Journaler created with encoding %r but the one "
                         "loaded from existing database is %r. Using "
                         "the value of: %r",
                         self._encoding, encoding, encoding)
        self._encoding = encoding
        self._initiated_ok()

    def _create_schema(self, fail):
        fail.trap(sqlite3.OperationalError)
        commands = [
            text_helper.format_block("""
            CREATE TABLE entries (
              history_id INTEGER NOT NULL,
              journal_id BLOB,
              function_id VARCHAR(200),
              fiber_id VARCHAR(36),
              fiber_depth INTEGER,
              args BLOB,
              kwargs BLOB,
              side_effects BLOB,
              result BLOB,
              timestamp INTEGER
            )
            """),
            text_helper.format_block("""
            CREATE TABLE logs (
              message BLOB,
              level INTEGER,
              category VARCHAR(36),
              log_name VARCHAR(36),
              file_path VARCHAR(200),
              line_num INTEGER,
              timestamp INTEGER
            )
            """),
            text_helper.format_block("""
            CREATE TABLE histories (
              id INTEGER PRIMARY KEY AUTOINCREMENT,
              agent_id VARCHAR(36),
              instance_id INTEGER
            )
            """),
            text_helper.format_block("""
            CREATE TABLE metadata (
              name VARCHAR(100),
              value VARCHAR(100)
            )
            """),
            text_helper.format_block("""
            CREATE INDEX history_idx ON entries(history_id)
            """),
            text_helper.format_block("""
            CREATE INDEX instance_idx ON histories(agent_id, instance_id)
            """)]

        def run_all(connection, commands):
            for command in commands:
                self.log('Executing command:\n %s', command)
                connection.execute(command)

        insert_meta = "INSERT INTO metadata VALUES('%s', '%s')"
        commands += [insert_meta % (u'encoding', self._encoding, )]

        self._reset_history_id_cache()
        # insert_history = "INSERT INTO histories VALUES(%d, '%s', %d)"
        # for (a_id, i_id), h_id in self._history_id_cache.iteritems():
        #     commands += [insert_history % (h_id, a_id, i_id)]

        d = self._db.runWithConnection(run_all, commands)
        d.addCallbacks(self._initiated_ok, self._error_handler)
        return d

    def _initiated_ok(self, *_):
        self.log('Journaler initiated correctly for the filename %r',
                 self._filename)
        self._set_state(State.connected)
        return self._flush_next()

    def _perform_inserts(self, cache):

        def do_insert_entry(connection, history_id, data):
            command = text_helper.format_block("""
            INSERT INTO entries VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?,
                                        strftime('%s', 'now'))
            """)
            connection.execute(
                command, (history_id,
                          data['journal_id'], data['function_id'],
                          data['fiber_id'], data['fiber_depth'],
                          data['args'], data['kwargs'],
                          data['side_effects'], data['result'], ))

        def do_insert_log(connection, data):
            command = text_helper.format_block("""
            INSERT INTO logs VALUES (?, ?, ?, ?, ?, ?, strftime('%s', 'now'))
            """)
            connection.execute(
                command, (data['message'], int(data['level']),
                          data['category'], data['log_name'],
                          data['file_path'], data['line_num']))

        def transaction(connection, cache):
            entries = cache.fetch()
            if not entries:
                return
            try:
                entries = map(self._encode, entries)
                for data in entries:
                    if data['entry_type'] == 'journal':
                        history_id = self._get_history_id(
                            connection, data['agent_id'], data['instance_id'])
                        do_insert_entry(connection, history_id, data)
                    elif data['entry_type'] == 'log':
                        do_insert_log(connection, data)
                cache.commit()
            except Exception:
                cache.rollback()
                raise

        return self._db.runWithConnection(transaction, cache)

    def _get_history_id(self, connection, agent_id, instance_id):
        '''
        Checks own cache for history_id for agent_id and instance_id.
        If information is missing fetch it from database. If it is not there
        create the new record.

        BEWARE: This method runs in a thread.
        '''
        cache_key = (agent_id, instance_id, )
        if cache_key in self._history_id_cache:
            history_id = self._history_id_cache[cache_key]
            return history_id
        else:
            command = text_helper.format_block("""
            SELECT id FROM histories WHERE agent_id = ? AND instance_id = ?
            """)
            cursor = connection.cursor()
            cursor.execute(command, (agent_id, instance_id, ))
            res = cursor.fetchall()
            if res:
                history_id = res[0][0]
                self._history_id_cache[cache_key] = history_id
                return history_id
            else:
                command = 'INSERT INTO histories VALUES (NULL, ?, ?)'
                cursor.execute(command, (agent_id, instance_id, ))
                history_id = cursor.lastrowid
                self._history_id_cache[cache_key] = history_id
                return history_id

    @in_state(State.connected)
    def _flush_next(self):
        if len(self._cache) == 0:
            return defer.succeed(None)
        else:
            d = self._semaphore.run(self._perform_inserts, self._cache)
            d.addCallback(defer.drop_param, self._flush_next)
            return d

class Record(object):
    implements(IRecord)

    def __init__(self, journaler):
        self._journaler = journaler

    def commit(self, **data):
        data['entry_type'] = 'journal'
        self._journaler.insert_entry(**data)

class JournalerConnection(log.Logger, log.LogProxy):
    implements(IJournalerConnection)

    def __init__(self, journaler, externalizer):
        log.LogProxy.__init__(self, journaler)
        log.Logger.__init__(self, self)

        self.serializer = banana.Serializer(externalizer=externalizer)
        self.snapshot_serializer = banana.Serializer()
        self.journaler = IJournaler(journaler)

    ### IJournalerConnection ###

    def new_entry(self, agent_id, instance_id, journal_id, function_id,
                  *args, **kwargs):
        record = self.journaler.prepare_record()
        entry = AgencyJournalEntry(
            self.serializer, record, agent_id, instance_id,
            journal_id, function_id, *args, **kwargs)
        return entry

    def get_filename(self):
        return self.journaler.get_filename()

    def snapshot(self, agent_id, instance_id, snapshot):
        record = self.journaler.prepare_record()
        entry = AgencyJournalEntry(
            self.snapshot_serializer, record, agent_id, instance_id,
            'agency', 'snapshot', snapshot)
        entry.set_result(None)
        entry.commit()

class AgencyJournalSideEffect(object):

    implements(IJournalSideEffect)

    ### IJournalSideEffect ###

    def __init__(self, serializer, record, function_id, *args, **kwargs):
        self._serializer = serializer
        self._record = record
        self._fun_id = function_id
        self._args = serializer.freeze(args or tuple())
        self._kwargs = serializer.freeze(kwargs or dict())
        self._effects = []
        self._result = None

    ### IJournalSideEffect Methods ###

    def add_effect(self, effect_id, *args, **kwargs):
        assert self._record is not None
        data = (effect_id,
                self._serializer.convert(args),
                self._serializer.convert(kwargs))
        self._effects.append(data)

    def set_result(self, result):
        assert self._record is not None
        self._result = self._serializer.convert(result)
        return self

    def commit(self):
        assert self._record is not None
        data = (self._fun_id, self._args, self._kwargs,
                self._effects, self._result)
        self._record.extend(data)
        self._record = None
        return self

class History(formatable.Formatable, pb.Copyable):
    '''
    Mapping for objects in history database.
    '''

    formatable.field('history_id', None)
    formatable.field('agent_id', None)
    formatable.field('instance_id', None)

    @classmethod
    def fetch(cls, db):
        d = db.runQuery(
            "SELECT id, agent_id, instance_id FROM histories")
        d.addCallback(cls._parse_resp)
        return d

    @classmethod
    def _parse_resp(cls, resp):
        columns = map(operator.attrgetter('name'), cls._fields)
        return map(lambda row: cls(**dict(zip(columns, row))), resp)

class AgencyJournalEntry(object):

    implements(IJournalEntry)

    def __init__(self, serializer, record, agent_id, instance_id, journal_id,
                 function_id, *args, **kwargs):
        self._serializer = serializer
        self._record = record

        self._data = {
            'agent_id': agent_id,
            'instance_id': instance_id,
            'journal_id': self._serializer.convert(journal_id),
            'function_id': function_id,
            'fiber_id': None,
            'fiber_depth': None,
            'side_effects': list()}

        self._not_serialized = {
            'args': args or None,
            'kwargs': kwargs or None,
            'result': None}

    ### IJournalEntry Methods ###

    def set_fiber_context(self, fiber_id, fiber_depth):
        assert self._record is not None
        self._data['fiber_id'] = fiber_id
        self._data['fiber_depth'] = fiber_depth
        return self

    def set_result(self, result):
        assert self._record is not None
        self._not_serialized['result'] = result

    def get_result(self):
        return self._not_serialized['result']

    def new_side_effect(self, function_id, *args, **kwargs):
        assert self._record is not None
        record = []
        self._data['side_effects'].append(record)
        return AgencyJournalSideEffect(self._serializer, record,
                                       function_id, *args, **kwargs)

    def commit(self):
        try:
            self._data['args'] = self._serializer.convert(
                    self._not_serialized['args'])
            self._data['kwargs'] = self._serializer.convert(
                    self._not_serialized['kwargs'])
            self._data['result'] = self._serializer.freeze(
                    self._not_serialized['result'])
            self._data['side_effects'] = self._serializer.convert(
                    self._data['side_effects'])
            self._record.commit(**self._data)
            self._record = None
            return self
        except TypeError as e:
            self.set_result(fiber.fail(e))
            self._not_serialized['args'] = None
            self._not_serialized['kwargs'] = None
            self.commit()


import os
import tempfile

import numpy as np
import pandas as pd
from unittest.mock import patch

import pytest

from src.script import Script

@pytest.mark.type_integration
class TestCaseScript:
    @pytest.fixture
    def dataframe1(self):
        return pd.DataFrame({"A": [1.0, np.nan, 3.0], "B": [4.0, np.nan, 6.0]})

    @pytest.fixture
    def dataframe2(self):
        return pd.DataFrame({"C": [7.0, np.nan, 9.0]})

    @pytest.fixture
    def tmp_dir(self):
        with tempfile.TemporaryDirectory() as directory:
            yield directory

    @pytest.fixture
    def script(self, dataframe1, dataframe2, tmp_dir):
        with patch("src.script.CustomProcessor._load_datasets", return_value={"dataset1": dataframe1, "dataset2": dataframe2}):
            yield Script(datasets_paths={"dataset1": "dataset1.parquet"}, output_path=tmp_dir)

    @pytest.mark.execution_slow
    @pytest.mark.priority_high
    @pytest.mark.case_positive
    def test_run_processor(self, script, tmp_dir):
        expected_df1 = pd.DataFrame({"A": [1.0, 0.0, 3.0], "B": [4.0, 0.0, 6.0], "feature": [5.0, 0.0, 9.0]})
        expected_df2 = pd.DataFrame({"C": [7.0, 0.0, 9.0]})

        script.run()

        assert sorted(os.listdir(tmp_dir)) == ["dataset1.parquet", "dataset2.parquet"]
        dataset1 = pd.read_parquet(os.path.join(tmp_dir, "dataset1.parquet"))
        dataset2 = pd.read_parquet(os.path.join(tmp_dir, "dataset2.parquet"))
        assert dataset1.equals(expected_df1)
        assert dataset2.equals(expected_df2)


import re
import requests
from bs4 import BeautifulSoup
from selenium import webdriver

#注：此文件用于提取动漫狂网站内漫画的真实下载地址，网站的网址为 https://www.cartoonmad.com/
def html_encoding():
    return 'big5'                                                #动漫狂网站的编码格式

def get_image_url_prefixion(comic_url):
    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1'}
    html_return = requests.get(comic_url,headers=headers)
    html_return.encoding = 'big5'
    html = BeautifulSoup(html_return.text,'lxml')
    info_lists = html.find_all(name='fieldset')                    #分析网站发现，需要的信息全放在第二个fieldset的td标签中
    info_lists = info_lists[1].find_all(name='td')
    for num in range(len(info_lists)):
        try:
            part_url = info_lists[num].a.get('href')
        except AttributeError:                                     #不需要的td标签用except来排除
            continue
        else:
            break
    new_url = 'https://www.cartoonmad.com' + part_url
    html_return = requests.get(new_url,headers=headers)
    html_return.encoding = 'big5'
    html = BeautifulSoup(html_return.text,'lxml')
    info_lists = html.find_all(name='img')
    for num in range(len(info_lists)):
        try:
            part_url = info_lists[num].get('src')
            if not 'file=/' in part_url:
                continue
        except AttributeError:
            continue
        else:
            break
    part_url = part_url.replace('amp;','')
    new_url = 'https://www.cartoonmad.com/comic/'+part_url
    opt = webdriver.ChromeOptions()
    opt.set_headless()
    driver = webdriver.Chrome(options=opt)
    driver.get(new_url)
    image_url_prefixion=driver.current_url
    driver.quit()
    image_url_prefixion = re.findall(r'(.+)/[0-9]+/[0-9]+.jpg',image_url_prefixion)[0]+'/'
    return image_url_prefixion

def extract_comic_info(url_return):                                #确定动漫狂网站上该漫画一共更新了多少话，每话有多少页数，并将结果存入一个空字典中,其中使用了BeautifulSoup模块
    info_dict = {}
    choosed_comic_name = ''
    html = BeautifulSoup(url_return.text,'lxml')
    all_info_lists = html.find_all(name='fieldset')                    #分析网站发现，需要的信息全放在第二个fieldset的td标签中，且话的名称在a标签中，页数在font标签中
    info_lists = all_info_lists[1].find_all(name='td')
    comic_name_info = all_info_lists[0].find_all(name='legend')
    choosed_comic_name = comic_name_info[0].string.strip()[:-2]
    for num in range(len(info_lists)):
        try:
            chapter_info = info_lists[num].a.string
            pages = info_lists[num].font.string
        except AttributeError:                                     #不需要的td标签用except来排除
            continue
        else:
            pattern = re.compile(r'\d+')
            pages = pattern.findall(pages)[0]
            chapter_info = str(chapter_info)
            info_dict[chapter_info] = pages                        #将话名与相应页数存入字典
    return info_dict,choosed_comic_name

def form_image_url(info_dict,folder_path,image_url_prefixion):                 #此函数返回的字典存入了每一话名称，每一张图片的页码及真实下载地址，其中值为一个字典，存放着每一张图片的页码及真实下载地址，即字典里套字典
    total_info_dict = {}
    for chapter_info,pages in info_dict.items():                         #构造下载链接
        pattern = re.compile(r'\d+')
        chapter = pattern.findall(chapter_info)[0]         #获取下载地址中的一串数字,等下构建图片的真实下载地址时要用到
        pages = int(pages)+1
        interim_info_dict = {}
        for page in range(1,pages):
            if page < 10:
                image_url = image_url_prefixion+chapter+'/00'+str(page)+'.jpg'
            elif page <100:
                image_url = image_url_prefixion+chapter+'/0'+str(page)+'.jpg'
            else:
                image_url = image_url_prefixion+chapter+'/'+str(page)+'.jpg'       #确定真实下载地址
            interim_info_dict[page] = image_url
        total_info_dict[chapter_info] = interim_info_dict
    return total_info_dict


from GANLib import CGAN
from GANLib import plotter

from keras.datasets import mnist, fashion_mnist, cifar10
from keras.layers import Input, Dense, Reshape, Flatten, Dropout, concatenate
from keras.layers import BatchNormalization, Activation, Embedding, ZeroPadding2D
from keras.layers.advanced_activations import LeakyReLU
from keras.layers.convolutional import UpSampling2D, Conv2D
from keras.models import Model
from keras.optimizers import Adam, RMSprop, Nadam

import matplotlib.pyplot as plt
import numpy as np

class conv_model_28():
    def build_generator(self):
        input_lat = Input(shape=(self.latent_dim,))
        input_lbl = Input(shape=self.label_shape)

        layer = concatenate([input_lat, input_lbl])
        layer = Dense(256)(layer)
        layer = LeakyReLU(alpha=0.2)(layer)
        layer = BatchNormalization(momentum=0.8)(layer)

        layer = Dense(784)(layer)
        layer = LeakyReLU(alpha=0.2)(layer)
        layer = BatchNormalization(momentum=0.8)(layer)

        layer = Reshape((7,7,16))(layer)

        #7 -> 14 -> 28
        layer = UpSampling2D(2)(layer)
        layer = Conv2D(8, (3,3), padding='same')(layer)
        layer = LeakyReLU(alpha=0.2)(layer) #14x14x8
        layer = BatchNormalization(momentum=0.8, axis = -1)(layer)

        layer = UpSampling2D(2)(layer)
        layer = Conv2D(4, (3,3), padding='same')(layer)
        layer = LeakyReLU(alpha=0.2)(layer) #28x28x4
        layer = BatchNormalization(momentum=0.8, axis = -1)(layer)

        img = Conv2D(1, (1,1), padding='same')(layer)
        return Model([input_lat, input_lbl], img)

    def build_discriminator(self):
        input_img = Input(shape=self.input_shape)
        input_lbl = Input(shape=self.label_shape)

        layer = Conv2D(8, (3,3), strides = 2, padding='same')(input_img)
        layer = LeakyReLU(alpha=0.2)(layer)
        layer = Conv2D(16, (3,3), strides = 2, padding='same')(layer)
        layer = LeakyReLU(alpha=0.2)(layer)
        layer = Conv2D(32, (3,3), strides = 2, padding='same')(layer)
        layer = LeakyReLU(alpha=0.2)(layer)
        layer = Flatten()(layer)

        layer = concatenate([layer, input_lbl])
        layer = Dense(256)(layer)
        layer = LeakyReLU(alpha=0.2)(layer)
        layer = Dense(128)(layer)
        layer = LeakyReLU(alpha=0.2)(layer)

        validity = Dense(1, activation=self.disc_activation)(layer)
        return Model([input_img, input_lbl], validity)

class dense_model():
    def build_generator(self):
        input_lat = Input(shape=(self.latent_dim,))

        layer = input_lat

        layer = Dense(128)(layer)
        layer = LeakyReLU(alpha=0.2)(layer)
        layer = BatchNormalization(momentum=0.8)(layer)

        layer = Dense(256)(layer)
        layer = LeakyReLU(alpha=0.2)(layer)
        layer = BatchNormalization(momentum=0.8)(layer)

        layer = Dense(784)(layer)
        layer = LeakyReLU(alpha=0.2)(layer)
        layer = BatchNormalization(momentum=0.8)(layer)

        img = Reshape((28,28,1))(layer)
        return Model(input_lat, img)

    def build_discriminator(self):
        input_img = Input(shape=self.input_shape)

        layer = Flatten()(input_img)
        layer = Dense(784)(layer)
        layer = LeakyReLU(alpha=0.2)(layer)

        layer = Dense(256)(layer)
        layer = LeakyReLU(alpha=0.2)(layer)

        layer = Dense(128)(layer)
        layer = LeakyReLU(alpha=0.2)(layer)

        validity = Dense(1, activation=self.disc_activation)(layer)

        return Model(input_img, validity)

class conv_model_32():
    def build_generator(self):
        input_lat = Input(shape=(self.latent_dim,))

        layer = input_lat
        layer = Dense(512)(layer)
        layer = LeakyReLU(alpha=0.2)(layer)
        layer = BatchNormalization(momentum=0.8)(layer)

        layer = Dense(1024)(layer)
        layer = LeakyReLU(alpha=0.2)(layer)
        layer = BatchNormalization(momentum=0.8)(layer)

        layer = Reshape((8,8,16))(layer)

        #8 -> 16 -> 32
        layer = UpSampling2D(2)(layer)
        layer = Conv2D(16, (3,3), padding='same')(layer)
        layer = LeakyReLU(alpha=0.2)(layer) #16x16x16
        layer = BatchNormalization(momentum=0.8, axis = -1)(layer)

        layer = UpSampling2D(2)(layer)
        layer = Conv2D(8, (3,3), padding='same')(layer)
        layer = LeakyReLU(alpha=0.2)(layer) #32x32x8
        layer = BatchNormalization(momentum=0.8, axis = -1)(layer)

        img = Conv2D(3, (1,1), padding='same')(layer)
        return Model(input_lat, img)

    def build_discriminator(self):
        input_img = Input(shape=self.input_shape)

        layer = Conv2D(8, (3,3), strides = 2, padding='same')(input_img)
        layer = LeakyReLU(alpha=0.2)(layer)
        layer = Conv2D(16, (3,3), strides = 2, padding='same')(layer)
        layer = LeakyReLU(alpha=0.2)(layer)
        layer = Conv2D(32, (3,3), strides = 2, padding='same')(layer)
        layer = LeakyReLU(alpha=0.2)(layer)
        layer = Flatten()(layer)

        layer = Dense(256)(layer)
        layer = LeakyReLU(alpha=0.2)(layer)
        layer = Dense(128)(layer)
        layer = LeakyReLU(alpha=0.2)(layer)

        validity = Dense(1, activation=self.disc_activation)(layer)
        return Model(input_img, validity)

tests = { 'dataset':  (mnist,         mnist,         fashion_mnist, fashion_mnist, cifar10,       cifar10),
          'img_path': ('mnist',       'mnist',       'fashion',     'fashion',     'cifar10',     'cifar10'),
          'mode':     ('vanilla',     'stable',      'vanilla',     'stable',      'vanilla',     'stable'),
          'model':    (conv_model_28, conv_model_28, conv_model_28, conv_model_28, conv_model_32, conv_model_32)
        }

noise_dim = 100

def sample_images(gen, file):
    r, c = 5, 5

    noise = np.random.uniform(-1, 1, (r * c, noise_dim))
    labels = np.zeros((r*c,10))
    for i in range(r):
        labels[i::r, i] = 1.

    gen_imgs = gen.predict([noise, labels])

    # Rescale images 0 - 1
    gen_imgs = 0.5 * gen_imgs + 0.5
    gen_imgs = np.clip(gen_imgs,0,1)

    fig, axs = plt.subplots(r, c)
    cnt = 0
    for i in range(r):
        for j in range(c):
            if gen_imgs.shape[-1] == 1:
                axs[i,j].imshow(gen_imgs[cnt,:,:,0], cmap='gray')
            else:
                axs[i,j].imshow(gen_imgs[cnt,:,:])
            axs[i,j].axis('off')
            cnt += 1
    fig.savefig(file) #% epoch
    plt.close()

for i in range(len(tests['dataset'])):
    model = tests['model'][i]

    # Load the dataset
    (X_train, labels), (_, _) = tests['dataset'][i].load_data()

    Y_train = np.zeros((X_train.shape[0],10))
    Y_train[np.arange(X_train.shape[0]), labels] = 1.

    # Configure input
    X_train = (X_train.astype(np.float32) - 127.5) / 127.5

    if len(X_train.shape)<4:
        X_train = np.expand_dims(X_train, axis=3)

    #Run GAN for 20000 iterations
    gan = CGAN(X_train.shape[1:], (10,), noise_dim, mode = tests['mode'][i])
    gan.build_generator = lambda self=gan: model.build_generator(self)
    gan.build_discriminator = lambda self=gan: model.build_discriminator(self)
    gan.build_models()

    def callback():
        path = 'images/CGAN/'+tests['img_path'][i]+'/conv_'+tests['mode'][i]
        sample_images(gan.generator, path+'.png')
        plotter.save_hist_image(gan.history, path+'_hist.png')

    gan.train(X_train, Y_train, epochs=20000, batch_size=64, checkpoint_callback = callback, validation_split = 0.1)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Nov 18 10:05:09 2017

@author: nainggolan
"""

import requests

def getHTMLText(url):
    try:
        r = requests.get(url, timeout=30)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
    except:
        return "产生异常"

if __name__=="__main__":
    url = "http://www.baidu.com"
    print(getHTMLText(url))


# Copyright 2021 MosaicML. All Rights Reserved.

import os
from unittest.mock import MagicMock, Mock

import pytest
import torch
import torch.distributed as dist
import torch.utils.data
from _pytest.monkeypatch import MonkeyPatch

from composer import Logger, State
from composer.core.types import DataLoader, Model, Precision, Tensors
from composer.datasets import DataloaderHparams, DataloaderSpec, DatasetHparams, SyntheticDatasetHparams
from composer.models import BaseMosaicModel, MnistClassifierHparams, ModelHparams, MosaicClassifier
from composer.optim import AdamHparams, ExponentialLRHparams
from composer.trainer import TrainerHparams
from composer.trainer.ddp import DDPDataLoader, DDPHparams, FileStoreHparams
from composer.trainer.devices import CPUDeviceHparams

@pytest.fixture()
def dummy_model_hparams() -> ModelHparams:
    return MnistClassifierHparams(num_classes=10)

@pytest.fixture()
def dummy_model(dummy_model_hparams: ModelHparams) -> BaseMosaicModel:
    return dummy_model_hparams.initialize_object()

@pytest.fixture()
def dummy_dataset_hparams() -> SyntheticDatasetHparams:
    return SyntheticDatasetHparams(num_classes=10,
                                   shape=[1, 28, 28],
                                   device="cpu",
                                   sample_pool_size=256,
                                   one_hot=False,
                                   drop_last=True,
                                   shuffle=False)

@pytest.fixture()
def dummy_dataloader_spec(dummy_dataset_hparams: SyntheticDatasetHparams) -> DataloaderSpec:
    return dummy_dataset_hparams.initialize_object()

@pytest.fixture()
def dummy_state_without_rank(dummy_model: BaseMosaicModel) -> State:
    state = State(
        model=dummy_model,
        epoch=5,
        step=50,
        precision=Precision.FP32,
        grad_accum=1,
        train_batch_size=10,
        eval_batch_size=10,
        max_epochs=10,
    )
    return state

@pytest.fixture
def dummy_dataloader_hparams() -> DataloaderHparams:
    return DataloaderHparams(
        num_workers=0,
        prefetch_factor=2,
        persistent_workers=False,
        pin_memory=False,
        timeout=0,
    )

def get_dataloader(dataloader_spec: DataloaderSpec, dataloader_hparams: DataloaderHparams) -> DataLoader:
    batch_size = 10

    sampler = torch.utils.data.DistributedSampler[int](
        dataloader_spec.dataset,
        drop_last=dataloader_spec.drop_last,
        shuffle=dataloader_spec.shuffle,
        rank=0,
        num_replicas=1,
    )

    dataloader = torch.utils.data.DataLoader(
        dataloader_spec.dataset,
        batch_size=batch_size,
        shuffle=False,  # set in the sampler
        num_workers=dataloader_hparams.num_workers,
        pin_memory=dataloader_hparams.pin_memory,
        drop_last=dataloader_spec.drop_last,
        sampler=sampler,
        collate_fn=dataloader_spec.collate_fn,
        worker_init_fn=dataloader_spec.worker_init_fn,
        multiprocessing_context=dataloader_spec.multiprocessing_context,
        generator=dataloader_spec.generator,
        timeout=dataloader_hparams.timeout,
        prefetch_factor=dataloader_hparams.prefetch_factor,
        persistent_workers=dataloader_hparams.persistent_workers,
    )
    return DDPDataLoader(dataloader)

@pytest.fixture
def dummy_train_dataloader(dummy_dataloader_spec: DataloaderSpec,
                           dummy_dataloader_hparams: DataloaderHparams) -> DataLoader:
    return get_dataloader(dummy_dataloader_spec, dummy_dataloader_hparams)

@pytest.fixture
def dummy_val_dataloader(dummy_dataloader_spec: DataloaderSpec,
                         dummy_dataloader_hparams: DataloaderHparams) -> DataLoader:
    return get_dataloader(dummy_dataloader_spec, dummy_dataloader_hparams)

@pytest.fixture()
def dummy_state(dummy_state_without_rank: State, monkeypatch: MonkeyPatch) -> State:
    monkeypatch.setattr(dist, "get_rank", lambda: 0)
    return dummy_state_without_rank

@pytest.fixture()
def dummy_state_dl(dummy_state: State, dummy_train_dataloader: DataLoader) -> State:
    dummy_state.train_dataloader = dummy_train_dataloader
    return dummy_state

@pytest.fixture()
def dummy_logger(dummy_state: State):
    return Logger(dummy_state)

@pytest.fixture
def logger_mock():
    return MagicMock()

"""
Dummy algorithms
"""

@pytest.fixture()
def algorithms(always_match_algorithms):
    return always_match_algorithms

@pytest.fixture()
def always_match_algorithms():
    attrs = {'match.return_value': True}
    return [Mock(**attrs) for _ in range(5)]

@pytest.fixture()
def never_match_algorithms():
    attrs = {'match.return_value': False}
    return [Mock(**attrs) for _ in range(5)]

@pytest.fixture
def mosaic_trainer_hparams(
    dummy_model_hparams: ModelHparams,
    dummy_dataset_hparams: DatasetHparams,
    ddp_tmpdir: str,
) -> TrainerHparams:
    return TrainerHparams(
        algorithms=[],
        optimizer=AdamHparams(),
        schedulers=[ExponentialLRHparams(gamma=0.1)],
        max_epochs=2,
        precision=Precision.FP32,
        total_batch_size=64,
        eval_batch_size=64,
        ddp=DDPHparams(
            store=FileStoreHparams(os.path.join(ddp_tmpdir, "store")),
            node_rank=0,
            num_nodes=1,
            fork_rank_0=False,
        ),
        dataloader=DataloaderHparams(
            num_workers=0,
            prefetch_factor=2,
            persistent_workers=False,
            pin_memory=False,
            timeout=0,
        ),
        device=CPUDeviceHparams(n_cpus=1),
        loggers=[],
        model=dummy_model_hparams,
        val_dataset=dummy_dataset_hparams,
        train_dataset=dummy_dataset_hparams,
        grad_accum=1,
    )

"""
Simple models
"""

class SimpleConvModel(torch.nn.Module):

    def __init__(self):
        super().__init__()

        conv_args = dict(kernel_size=(3, 3), padding=1)
        self.conv1 = torch.nn.Conv2d(in_channels=32, out_channels=8, stride=2, bias=False, **conv_args)  # stride > 1
        self.conv2 = torch.nn.Conv2d(in_channels=8, out_channels=32, stride=2, bias=False,
                                     **conv_args)  # stride > 1 but in_channels < 16
        self.conv3 = torch.nn.Conv2d(in_channels=32, out_channels=64, stride=1, bias=False, **conv_args)  # stride = 1

        self.pool1 = torch.nn.MaxPool2d(kernel_size=(2, 2), stride=2, padding=1)

    def forward(self, x: Tensors) -> Tensors:  # type: ignore
        # Very basic forward operation with no activation functions
        # used just to test that model surgery doesn't create forward prop bugs.
        out = self.conv1(x)
        out = self.conv2(out)
        out = self.conv3(out)
        out = self.pool1(out)
        return out

@pytest.fixture()
def simple_conv_model():
    return MosaicClassifier(SimpleConvModel())

@pytest.fixture()
def simple_conv_model_input():
    return torch.rand((64, 32, 64, 64))

@pytest.fixture()
def state_with_model(simple_conv_model: Model):
    state = State(
        epoch=50,
        step=50,
        train_batch_size=100,
        eval_batch_size=100,
        grad_accum=1,
        max_epochs=100,
        model=simple_conv_model,
        precision=Precision.FP32,
    )
    return state


import os
import sys
import osgeo
from osgeo import gdal
from osgeo import ogr
from osgeo import osr
import numpy as np
from random import seed
from random import random
from common_utils import vector_operations as vop

import time
seed(time.time() * 1000)

def preview2geotiff (input_preview, output_geotiff, long_min, lat_min, long_max, lat_max, utm_zone = None):
    #calc EPSG code based on UTM zone
    if utm_zone is None:
        utm_zone = int((0.5*(long_min + long_max) + 180)/6) + 1

    epsg_code = 100*(326 if (lat_min + lat_max >= 0) else 327) + utm_zone

    #calc UTM XY BBOX
    latlongSpatialRef = osr.SpatialReference()
    latlongSpatialRef.ImportFromEPSG(4326)

    utmSpatialRef = osr.SpatialReference()
    utmSpatialRef.ImportFromEPSG(epsg_code)

    coordTrans = osr.CoordinateTransformation(latlongSpatialRef, utmSpatialRef)

    point_ll = ogr.Geometry(ogr.wkbPoint)
    point_ll.AddPoint(lat_min,long_min)
    point_ll.Transform(coordTrans)

    point_ur = ogr.Geometry(ogr.wkbPoint)
    point_ur.AddPoint(lat_max,long_max)
    point_ur.Transform(coordTrans)

    #N1 calc pixel size
    gdal_ds = gdal.Open(input_preview)
    w,h = gdal_ds.RasterXSize,gdal_ds.RasterYSize

    pixel_size = (0.5 *  ( ( (point_ur.GetX()-point_ll.GetX() )/w) +
                          ( (point_ur.GetY()-point_ll.GetY() )/h) ) )

    #img = gdal_ds.ReadAsArray()
    array2geotiff(output_geotiff,
                    (point_ll.GetX(),point_ur.GetY()),
                    pixel_size,
                    utmSpatialRef.ExportToWkt(),
                    gdal_ds.ReadAsArray())

def get_clipped_inmem_raster (raster_file,
                            cutline = None,
                            dst_nodata = None,
                            cutline_where = None,
                            crop_to_cutline = True,
                            pixel_width=None,
                            pixel_height=None,
                            output_bounds=None,
                            output_type = None,
                            dst_srs = None,
                            pixel_res = None,
                            resample_alg = None
                              ) :
    in_mem_tiff = os.path.join('/vsimem',str(random()) + '.tif')
    warp(raster_file=raster_file,output_tiff=in_mem_tiff,cutline=cutline,dst_nodata=dst_nodata,
         cutline_where=cutline_where,crop_to_cutline=crop_to_cutline,pixel_width=pixel_width,pixel_height=pixel_height,
         output_type=output_type,output_bounds=output_bounds,dst_srs=dst_srs,pixel_res=pixel_res,resample_alg=resample_alg)

    return in_mem_tiff

def open_clipped_raster_as_image (raster_file,
                                cutline = None,
                                dst_nodata = None,
                                cutline_where = None,
                                crop_to_cutline = True,
                                pixel_width = None,
                                pixel_height = None,
                                output_bounds = None,
                                output_type = None,
                                dst_srs = None,
                                pixel_res = None,
                                resample_alg = None
                                  ) :
    in_mem_tiff = None
    if any(arg is not None for arg in [cutline,dst_nodata,pixel_width,pixel_height,dst_srs,pixel_res,resample_alg]):
        in_mem_tiff = get_clipped_inmem_raster (raster_file,
                                                cutline,
                                                dst_nodata,
                                                cutline_where,
                                                crop_to_cutline,
                                                pixel_width,
                                                pixel_height,
                                                output_bounds,
                                                output_type,
                                                dst_srs,
                                                pixel_res,
                                                resample_alg)

    raster_file = in_mem_tiff if in_mem_tiff is not None else raster_file

    gdal_ds = gdal.Open(raster_file)
    gdal_band = gdal_ds.GetRasterBand(1)
    img = gdal_band.ReadAsArray()

    gdal_ds_obj = None
    if in_mem_tiff is not None:
        gdal.Unlink(in_mem_tiff)

    return img

def get_raster_bbox (raster_file, t_srs = None) :
    gdal_ds = gdal.Open(raster_file)
    if not gdal_ds :
        return None

    #assume north up image
    geotr = gdal_ds.GetGeoTransform()
    ul_lr = ogr.Geometry(ogr.wkbLineString)
    ul_lr.AddPoint(geotr[0],geotr[3]) #add upper left corner of image
    ul_lr.AddPoint(geotr[0] + geotr[1]*gdal_ds.RasterXSize,
                    geotr[3] + geotr[5]*gdal_ds.RasterYSize ) #calc. lower right corner of image

    if t_srs:
        if int(osgeo.__version__[0]) >= 3:
            # GDAL 3 changes axis order: https://github.com/OSGeo/gdal/issues/154
            t_srs.SetAxisMappingStrategy(osr.OAMS_TRADITIONAL_GIS_ORDER)
        raster_srs=osr.SpatialReference(wkt=gdal_ds.GetProjection())
        coordTrans = osr.CoordinateTransformation(raster_srs,t_srs)
        #ulp.Transform(coordTrans)
        #lrp.Transform(coordTrans)
        ul_lr.Transform(coordTrans)
    gdal_ds = None

    return ul_lr.GetEnvelope()

def generate_virtual_random_tif_path ():
    return '/vsimem/memory_name' + str(random()) + '.tif'

def warp   (raster_file,
            output_tiff,
            cutline = None,
            src_nodata = None,
            dst_nodata = None,
            cutline_where = None,
            crop_to_cutline = True,
            pixel_width=None,
            pixel_height=None,
            output_bounds = None,
            output_type= None,
            dst_srs = None,
            pixel_res = None,
            resample_alg = None
            ):

    return gdal.Warp(output_tiff,
              [raster_file],
              format='GTiff',
              cutlineDSName=cutline,
              cutlineWhere=cutline_where,
              dstNodata=dst_nodata,
              width=pixel_width,
              height=pixel_height,
              cropToCutline= crop_to_cutline,
              outputBounds = output_bounds,
              srcNodata=src_nodata,
              dstSRS=dst_srs,
              resampleAlg=resample_alg,
              outputType = output_type,
              xRes=pixel_res,
              yRes=pixel_res
              )

def get_statistics(single_band_raster):
    gdal_ds = gdal.Open(single_band_raster)
    if gdal_ds is None: return None

    #min,max,mean,stdDev
    stat = gdal_ds.GetRasterBand(1).GetStatistics(False,True)
    gdal_ds = None

    return stat

def array2geotiff(output_geotiff, rasterOrigin, pixel_size, srs, array, nodata_val = None, compress=None):

    array_ref = None
    if (array.ndim == 2) :
        array_ref = list()
        array_ref.append(array)
    else: array_ref = array

    bands_num = 1 if (array.ndim==2) else array.shape[0]

    rows,cols = array_ref[0].shape[0],array_ref[0].shape[1]
    originX,originY = rasterOrigin[0],rasterOrigin[1]

    driver = gdal.GetDriverByName('GTiff')
    output_type = {np.uint8:gdal.GDT_Byte,
                    np.uint16:gdal.GDT_UInt16,
                    np.int16:gdal.GDT_Int16,
                    np.int32:gdal.GDT_Int32,
                    np.uint32:gdal.GDT_UInt32,
                    np.float32:gdal.GDT_Float32,
                    np.float64:gdal.GDT_Float32}[type(array_ref[0][0][0])]
    options = [f'COMPRESS={compress}'] if compress is not None else None
    outRaster = (driver.Create(output_geotiff, cols, rows, bands_num, output_type) if options is None
                else driver.Create(output_geotiff, cols, rows, bands_num, output_type, options = options) )
    outRaster.SetGeoTransform((originX, pixel_size, 0, originY, 0, -pixel_size))

    #outRaster.SetSpatialRef(srs) - fails for unknown reason
    prj_wkt = srs.ExportToWkt()
    outRaster.SetProjection(prj_wkt)

    for b in range(bands_num):
        outband = outRaster.GetRasterBand(b+1)
        outband.WriteArray(array_ref[b])
        if nodata_val is not None:
            outband.SetNoDataValue(nodata_val)
        outband.FlushCache()
    array_ref = None
    outRaster = None

def extract_georeference (raster_file, cutline = None):
    if cutline is not None:
        in_mem_tif = os.path.join('/vsimem',str(random()) + '.tif')
        gdal.Warp(in_mem_tif,
                [raster_file],
                format = 'GTiff',
                cutlineDSName = cutline,
                cropToCutline = True,
                srcNodata=0,
                dstNodata=0
                )

        raster_file = in_mem_tif

    gdal_ds = gdal.Open(raster_file)

    #srs = gdal_ds.GetSpatialRef().Clone() - fails for unknown reason
    srs = osr.SpatialReference(gdal_ds.GetProjection())

    geotransform = gdal_ds.GetGeoTransform()
    gdal_ds = None
    if cutline is not None:
        gdal.Unlink(raster_file)

    return  (srs,geotransform)

def calc_ndvi_as_image_from_mem (array_red,
                                 array_nir,
                                 ndv_in = 0,
                                 ndv_out = -10000,
                                 uint8_adjust = False):
    ndv_out = 0 if uint8_adjust else ndv_out

    array_ndvi = np.full(array_red.shape,ndv_out,np.ubyte if uint8_adjust else np.float32)

    red_vec = np.full(array_red.shape[1], fill_value=ndv_in, dtype=np.float32)
    nir_vec = np.full(array_red.shape[1], fill_value=ndv_in, dtype=np.float32)
    tmp1_vec = np.full(array_red.shape[1], fill_value=ndv_in, dtype=np.float32)
    tmp2_vec = np.full(array_red.shape[1], fill_value=ndv_in, dtype=np.float32)

    for i in range( 0, len(array_nir) ):
        np.copyto(dst=red_vec, src=array_red[i])
        np.copyto(dst=nir_vec, src=array_nir[i])
        np.subtract(nir_vec, red_vec, out=tmp1_vec)
        np.add(nir_vec, red_vec, out=tmp2_vec)
        tmp1_vec = np.divide(tmp1_vec, tmp2_vec,
                             out=np.full(tmp1_vec.shape,fill_value=ndv_out, dtype=np.float32),
                             where=np.logical_or(red_vec!=ndv_in,nir_vec!=ndv_in))
        if (uint8_adjust) :
            array_ndvi[i] = np.add(100*tmp1_vec, 101.5, dtype=np.ubyte, casting='unsafe',
                                out=np.zeros_like(tmp1_vec),
                                where=(tmp1_vec!=0))
        else:
            array_ndvi[i] = tmp1_vec

    return array_ndvi

def calc_ndvi_as_image (red_band_file, nir_band_file,
                        ndv_in=None, ndv_out=-10000, uint8_adjust = False):
  #create output:
    # - the same srs, pixel size as input bands
    # - pixel size = byte
    # - ndvi formulae = 101 + 100*ndvi
    # open dataset
    ds_nir = gdal.Open(nir_band_file)
    ds_red = gdal.Open(red_band_file)

    if ds_nir is None:
        print('ERROR: can\'t open file: ' + nir_band_file)
        exit(1)

    if ds_red is None:
        print('ERROR: can\'t open file: ' + red_band_file)
        exit(1)

    #array_ndvi = np.full((ds_nir.RasterYSize,ds_nir.RasterXSize),0,np.ubyte)
    band_nir = ds_nir.GetRasterBand(1)
    band_red = ds_red.GetRasterBand(1)
    array_nir = band_nir.ReadAsArray()
    array_red = band_red.ReadAsArray()

    if ndv_in is None:
        ndv_in = 0 if band_nir.GetNoDataValue() is None else band_nir.GetNoDataValue()

    array_ndvi = calc_ndvi_as_image_from_mem(array_red,array_nir,ndv_in,ndv_out, uint8_adjust)

    # close dataset
    ds_red = None
    ds_nir = None
    array_nir = None
    array_red = None

    return array_ndvi

def create_ndvi_uint8_file (red_band_file, nir_band_file, output_file) :
    #create output:
    # - the same srs, pixel size as input bands
    # - pixel size = byte
    # - ndvi formulae = 101 + 100*ndvi
    # open dataset
    ds_nir = gdal.Open(nir_band_file)

    if ds_nir is None:
        print('ERROR: can\'t open file: ' + nir_band_file)
        exit(1)

    prj_wkt=ds_nir.GetProjection()
    geotr = ds_nir.GetGeoTransform()

    ndvi_img = calc_ndvi_as_image(red_band_file, nir_band_file,True)

    array2geotiff(output_file,[geotr[0],geotr[3]],geotr[1],prj_wkt,ndvi_img)

    # close dataset
    ds_nir = None
    ndvi_img = None


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import requests
import re
from requests.exceptions import RequestException
import json
def get_html(url):
    try:
        res=requests.get(url)
        if res.status_code==200:
            return res.text
        return None
    except RequetException:
        return None
    return res
def get_wanted(html):
    rex=re.compile('
.*?board-index.*?>(\d+).*?data-src="(.*?)".*?name">(.*?).*?star">(.*?)
.*?releasetime">(.*?)
.*?integer">(.*?).*?fraction">(.*?).*?
',re.S)
    wanted=re.findall(rex,html)
    for item in wanted:
        yield {
            'index':item[0],
            'image':item[1],
            'name':item[2],
            'actor':item[3].strip()[3:],
            'time':item[4].strip()[5:],
            'score':item[5]+item[6]
        }

def write_to_file(content):
    with open('D://Spider/maoyantop100/result.txt','a',encoding='utf-8') as f:
        f.write(json.dumps(content,ensure_ascii=False)+'\n')
        f.close()

def main(offset):
    url='https://maoyan.com/board/4?offset='+str(offset)
    html=get_html(url)
    for item in get_wanted(html):
        print(item)
        write_to_file(item)

if __name__=='__main__':
    for i in range(10):
        main(i*10)


import re
import datetime
import pandas as pd
import h5py
import numpy as np

class get_realtime_data:
    def get_ticker_df(self,h5file):
        df_list = []
        with h5py.File(h5file, 'r') as f:
            allsymbols = f.keys()
            # print(allsymbols)
            for symbol in allsymbols:
                symbol_list = symbol.split('-')
                if len(symbol_list) != 4 or symbol_list[0] != 'BTC':
                    pass
                else:
                    df = pd.DataFrame(np.array(f[symbol]))
                    df['symbol'] = symbol
                    df_list.append(df)
        df = df_list[0].append(df_list[1:])
        return df

    def change_maturity(self,mtr):
        month_list = ['JAN', 'FEB', 'MAR', 'APR', 'MAY', 'JUN', 'JUL', 'AUG', 'SEP', 'OCT', 'NOV', 'DEC']
        mtr = re.split('([A-Z]+)', mtr)
        month = month_list.index(mtr[1]) + 1
        maturity = datetime.date(int(mtr[2]) + 2000, month, int(mtr[0]))
        return maturity

    def get_realtime_orderbook(self, date):
        date_str = date + datetime.timedelta(days= 1)
        date_str = str(date_str).replace("-", "")
        h5file = 'Y:\deribit\DERIBIT_' + date_str +'.h5ticker'
        option_df = self.get_ticker_df(h5file)
        option_df['timestamp'] = option_df['timestamp'].apply(lambda i: datetime.datetime.fromtimestamp(i/1000)- datetime.timedelta(hours= 8))
        option_df['Maturity'] = option_df['symbol'].apply(lambda x: x.split('-')[1])
        option_df['Maturity'] = option_df['Maturity'].apply(self.change_maturity)
        option_df['Strike'] = option_df['symbol'].apply(lambda x: int(x.split('-')[2]))
        option_df['CallPut'] = option_df['symbol'].apply(lambda x: x.split('-')[3])

        option_df_new = option_df.loc[:, ['symbol', 'timestamp', 'Maturity','Strike', 'CallPut', 'underlying_index', 'underlying_price',
                                          'best_bid_price', 'bid_iv', 'best_ask_price', 'ask_iv', 'mark_price', 'mark_iv', 'delta']]

        return option_df_new

    def get_tradetime_vol(self, dailytradetime, delta_T):
        delayTradeTime = dailytradetime + datetime.timedelta(minutes=delta_T)
        previousTradeTime = dailytradetime - datetime.timedelta(minutes=delta_T)
        option_df = self.get_realtime_orderbook(dailytradetime.date())
        option_df_by_symbol = option_df.groupby('symbol')
        symbol_count = 0
        all_data = pd.DataFrame()
        for symbol, symbol_group in option_df_by_symbol:
            data_1_time = symbol_group['timestamp']
            option_count = 0
            for t_count in range(0, len(data_1_time)):
                t = data_1_time[t_count]
                if previousTradeTime <= t <= delayTradeTime:
                    option_count = t_count
                    break
            start_t = data_1_time[option_count]
            if option_count == 0 and (start_t < previousTradeTime or start_t > delayTradeTime):
                continue
            if symbol_count == 0 :
                all_data = pd.DataFrame(columns = symbol_group.columns)
                all_data.loc[symbol_count, :] = symbol_group.loc[option_count, :]
            else:
                all_data.loc[symbol_count, :] = symbol_group.loc[option_count, :]
            symbol_count += 1
        return all_data

#ss = get_realtime_data()
#option_df = ss.get_realtime_orderbook()
#option_df = option_df.sort_values(['symbol','timestamp'])


import config
from geom._geom import *
from core.clip_helper import ClipHelper
from utils.progress import Progress
from prep.preprocessor import Preprocessor
import os

class RoadImporter:
    def __init__ (self, model, grid):
        self.__model = model
        self.__grid = grid
        self.__clipHelper = ClipHelper(grid)
        self.__preprocessor = Preprocessor()

    def process (self, filename):
        curSize = 0
        progress = Progress(filename, 0, os.stat(filename)[6])
        for ln in open(filename):
            if ln.startswith('RF'):
                self.processRF(ln)
            curSize += len(ln)
            progress.update(curSize)

    def getNodeIndex(self, parcel, coordinate):
        for nodeId, node in parcel.roadNodeDict.items():
            if abs(node.coordinate.lat - coordinate.lat) < 0.00000001 and abs(node.coordinate.lon - coordinate.lon) < 0.00000001:
               return nodeId
        node = XRoadNode()
        node.coordinate = coordinate
        nodeId = config.ROAD_NODE_SEQ_BASE
        parcel.roadNodeDict[nodeId] = node
        config.ROAD_NODE_SEQ_BASE += 1
        return nodeId

    def processRF (self, rf):
        grp = rf.split(';')
        link_id = grp[1]
        points = grp[2].split(',')
        priority = int(grp[3])
        direction = int(grp[4][0])
        roadType = int(grp[5])
        subtype = int(grp[6])
        speed = int(grp[7])
        name = self.__preprocessor.nominalizeName(grp[9])
        adminArea = grp[11]
        pointList = [XCoordinate(int(points[i]), int(points[i+1])) for i in range(0, len(points), 2)]
        for meshCode, polyline in self.__clipHelper.clipPolyline(pointList):
            parcel = self.__model.get(meshCode)
            if not parcel:
                parcel = XParcel()
                parcel.parcelId = meshCode
            link = XRoadLink()
            link.fNode = self.getNodeIndex(parcel, polyline[0])
            link.tNode = self.getNodeIndex(parcel, polyline[-1])
            link.trafficDir  = direction
            link.width      = 0 # width
            link.speed      = speed
            link.priority   = priority-1
            link.roadType   = roadType
            link.subType    = subtype
            link.length     = 0 # length
            link.roadName       = name
            link.adminArea  = adminArea
            link.polyline = polyline
            link.fAngle, link.tAngle = 0,0#self.__calcAngle(polyline)
            parcel.roadLinkDict[config.ROAD_LINK_SEQ_BASE] = link

            #for link in parcel.roadLinkDict.values():
            #    assert link.fNode in parcel.roadNodeDict
            #    assert link.tNode in parcel.roadNodeDict

            self.__model.put(meshCode, parcel)
            config.ROAD_LINK_SEQ_BASE += 1

    def __calcAngle (self, polyline):
        p1 = polyline[0]
        p2 = polyline[-1]
        a1 = angle(p1, XCoordinate(p1.lon+100, p1.lat), (p2.lon, p2.lat))
        a2 = angle(p2, XCoordinate(p2.lon+100, p2.lat), (p1[1], p1[0]))
        return (a1, a2)


"""
# 过滤器本质是函数
自定义过滤器步骤：
1.自定义一个python的函数去实现对应业务逻辑
2.通过app对象将函数添加到系统过滤器中
3.使用自定义过滤器

"""

# 1.自定义一个python的函数去实现对应业务逻辑
from flask import session, current_app, jsonify, g

from info.response_code import RET

def do_ranklist_class(index):

    if index == 0:
        return "first"
    elif index == 1:
        return "second"
    elif index == 2:
        return "third"
    else:
        return ""

import functools

# 需求：查询当前登录用户对象的代码在多个视图函数中都需要使用，我们可以使用装饰器将其封装
# view_func： 被装饰的函数名称
# 问题： 装饰器会改变被装饰的视图函数名称
# 方案：@functools.wraps(视图函数名称)

def get_user_info(view_func):

    @functools.wraps(view_func)
    def wrapper(*args, **kwargs):

        # 1.装饰视图函数新增的需求
        # 1.获取session中的用户id
        user_id = session.get("user_id")

        # 延迟导入解决db循环导入的问题
        from info.models import User

        # 2.根据user_id查询当前用户对象
        user = None  # type: User
        if user_id:
            try:
                user = User.query.get(user_id)
            except Exception as e:
                current_app.logger.error(e)
                return jsonify(errno=RET.DBERR, errmsg="查询用户对象异常")

        # 3.将用户对象保存起来提供给视图函数使用
        # 全局的临时变量保存用户对象，只要请求未结束，g变量中的值就不会改变
        g.user = user

        # 2.被装饰的视图函数原有功能实现
        result = view_func(*args, **kwargs)
        return result

    return wrapper


#!/bin/python
# programa que levanta los datos de energia y magnetizacion y los plotea

import numpy as np
from matplotlib import pyplot as plt

energiaanti = np.genfromtxt('energiaantiferro.txt', delimiter = '\t')
magnetanti = np.genfromtxt('magnetizacionantiferro.txt', delimiter = '\t')
energia = np.genfromtxt('energiaferro.txt', delimiter = '\t')
magnet = np.genfromtxt('magnetizacionferro.txt', delimiter = '\t')

T = np.linspace(0,5, len(energia))

plt.plot(T, energia, 'r^', label = 'J=1', ms = 5)
plt.plot(T, energiaanti, 'b.', label = 'J=-1', ms = 5)
plt.title('Caso antiferromagnético (J=-1)')
plt.ylabel('Energia')
plt.grid(True)
plt.legend(loc = 'best')
plt.xlabel('T')

plt.figure()
#plt.plot(T, magnet, 'r^', label = 'J=1', ms = 5)
plt.plot(T[0:-1], magnetanti[0:-1], 'b.', label = 'J=-1', ms = 5)
plt.title('Caso antiferromagnético (J=-1)')
plt.ylabel('Magnetización')
plt.grid(True)
#plt.legend(loc = 'best')
plt.xlabel('T')

plt.show()


from typing import AnyStr
import pandas as pd
import nltk
import pycrfsuite
import sklearn
from collections import Counter
from sklearn.metrics import classification_report, confusion_matrix
from sklearn.preprocessing import LabelBinarizer
from itertools import chain

DATA_DIRECTORY: str = 'data'
TEST_FILE_NAME: str = 'test.txt'
VALID_FILE_NAME: str = 'valid.txt'

"""
    string management
"""
SPACE: str = ' '
EMPTY: str = ''
NEW_LINE: str = '\n'
OPEN_PARENTHESIS: str = '('
END_PARENTHESIS: str = ')'
DOT: str = '.'
COMMA: str = ','
COLONS: str = ':'
SLASH: str = '/'
QUOTE: str = "'"

"""
    document mnagament
"""
DOCUMENT_START: str = '-DOCSTART-'
LINE_START: str = '- : O O'
LINE_END: str = '. . O O'

"""
    data files columns
"""
INDEX_WORD: int = 0
INDEX_POST_TAG: int = 2
INDEX_ENTITY: int = 3

class DocumentsHelper:

    def remove_last_char(self, value: str) -> str:
        return value.rstrip(self.get_last_char(value))

    def get_last_char(self, value: str) -> str:
        if len(value) > 0:
            return value[-1]
        return ''

    def is_parenthesis(self, value: str) -> bool:
        return value == OPEN_PARENTHESIS or value == END_PARENTHESIS

    def is_special_char_without_left_space(self, value: str) -> bool:
        return value == COMMA or value == DOT or value == SLASH or value == QUOTE or value == END_PARENTHESIS

    def is_special_char_without_right_space(self, value: str) -> bool:
        return value == OPEN_PARENTHESIS or value == SLASH

    def clean_string(self, value: str) -> str:
        return value.replace(NEW_LINE, EMPTY).replace('\'', "'")

    def is_string_start_line(self, line: str) -> bool:
        return line.__contains__(LINE_START)

    def is_string_end_line(self, line: str) -> bool:
        return line.__contains__(LINE_END)

    def is_string_not_start_or_end_line(self, line: str) -> bool:
        return not (self.is_string_start_line(line) or self.is_string_end_line(line))

    def is_string_not_document_start(self, line: str) -> bool:
        return not line.__contains__(DOCUMENT_START)

    def is_string_not_empty_or_none(self, value: str) -> bool:
        return not (value is None or value == '')

    def is_valid_token(self, line: str) -> bool:
        return self.is_string_not_empty_or_none(line) and self.is_string_not_document_start(
            line) and self.is_string_not_start_or_end_line(line)

    def is_documents_property_empty(self, documents: list) -> bool:
        return self.get_documents_size(documents) == 0

    def get_documents_size(self, documents: list) -> int:
        return len(documents)

def load(file_name: str):
    doc_lines = []
    h = DocumentsHelper()
    file = open(file_name, 'r')
    lines: [AnyStr] = file.readlines()

    for line in lines:
        split_line: [str] = line.split(SPACE)
        word: str = split_line[INDEX_WORD]
        try:
            if h.is_string_start_line(line):
                doc_line = []
            if word != NEW_LINE and not h.is_string_end_line(line) and not h.is_string_start_line(line):
                entity: str = split_line[INDEX_ENTITY]
                post_tag: str = split_line[INDEX_POST_TAG]
                if not entity.__contains__('I-'):
                    line_component = (word, post_tag, entity)
                    doc_line.append(line_component)
            if h.is_string_end_line(line):
                doc_lines.append(doc_line)
        except Exception as e:
            pass
    return doc_lines

def word2features(sent, i):
    word = sent[i][0]
    postag = sent[i][1]
    features = [
        'bias',
        'word.lower=' + word.lower(),
        'word[-3:]=' + word[-3:],
        'word[-2:]=' + word[-2:],
        'word.isupper=%s' % word.isupper(),
        'word.istitle=%s' % word.istitle(),
        'word.isdigit=%s' % word.isdigit(),
        'postag=' + postag,
        'postag[:2]=' + postag[:2],
    ]
    if i > 0:
        word1 = sent[i - 1][0]
        postag1 = sent[i - 1][1]
        features.extend([
            '-1:word.lower=' + word1.lower(),
            '-1:word.istitle=%s' % word1.istitle(),
            '-1:word.isupper=%s' % word1.isupper(),
            '-1:postag=' + postag1,
            '-1:postag[:2]=' + postag1[:2],
        ])
    else:
        features.append('BOS')

    if i < len(sent) - 1:
        word1 = sent[i + 1][0]
        postag1 = sent[i + 1][1]
        features.extend([
            '+1:word.lower=' + word1.lower(),
            '+1:word.istitle=%s' % word1.istitle(),
            '+1:word.isupper=%s' % word1.isupper(),
            '+1:postag=' + postag1,
            '+1:postag[:2]=' + postag1[:2],
        ])
    else:
        features.append('EOS')

    return features

def sent2features(sent):
    return [word2features(sent, i) for i in range(len(sent))]

def sent2labels(sent):
    return [label for token, postag, label in sent]

def sent2tokens(sent):
    return [token for token, postag, label in sent]

def get_X_Train():
    X_Tr = []
    for s in train_sents:
        X_Tr[s] = sent2features(s)
    return X_Tr
    # X_train = [sent2features(s) for s in train_sents]

def bio_classification_report(y_true, y_pred):
    """
    Classification report for a list of BIO-encoded sequences.
    It computes token-level metrics and discards "O" labels.

    Note that it requires scikit-learn 0.15+ (or a version from github master)
    to calculate averages properly!
    """
    lb = LabelBinarizer()
    y_true_combined = lb.fit_transform(list(chain.from_iterable(y_true)))
    y_pred_combined = lb.transform(list(chain.from_iterable(y_pred)))

    tagset = set(lb.classes_) - {'O'}
    tagset = sorted(tagset, key=lambda tag: tag.split('-', 1)[::-1])
    class_indices = {cls: idx for idx, cls in enumerate(lb.classes_)}

    return classification_report(
        y_true_combined,
        y_pred_combined,
        labels=[class_indices[cls] for cls in tagset],
        target_names=tagset,
    )

def print_transitions(trans_features):
    for (label_from, label_to), weight in trans_features:
        print("%-6s -> %-7s %0.6f" % (label_from, label_to, weight))

def print_state_features(state_features):
    for (attr, label), weight in state_features:
        print("%0.6f %-6s %s" % (weight, label, attr))

if __name__ == "__main__":
    valid_data = load(VALID_FILE_NAME)
    TRAIN_FILE_NAME = 'train.txt'
    train_sents = load(TRAIN_FILE_NAME) + valid_data
    test_sents = load(TEST_FILE_NAME)

    nltk.download('conll2002')
    print(sklearn.__version__)
    nltk.corpus.conll2002.fileids()

    # train_sents = list(nltk.corpus.conll2002.iob_sents('esp.train'))
    # test_sents = list(nltk.corpus.conll2002.iob_sents('esp.testb'))

    print(train_sents[0])
    print(test_sents[0])

    print(sent2features(train_sents[0])[0])

    X_train = [sent2features(s) for s in train_sents]
    y_train = [sent2labels(s) for s in train_sents]

    X_test = [sent2features(s) for s in test_sents]
    y_test = [sent2labels(s) for s in test_sents]

    trainer = pycrfsuite.Trainer(verbose=False)

    for xseq, yseq in zip(X_train, y_train):
        trainer.append(xseq, yseq)

    trainer.set_params({
        'c1': 1.0,  # coefficient for L1 penalty
        'c2': 1e-3,  # coefficient for L2 penalty
        'max_iterations': 50,  # stop earlier

        # include transitions that are possible, but not observed
        'feature.possible_transitions': True
    })

    trainer.params()

    trainer.train('conll2002-en.crfsuite')

    print(trainer.logparser.last_iteration)

    print(len(trainer.logparser.iterations), trainer.logparser.iterations[-1])

    tagger = pycrfsuite.Tagger()
    tagger.open('conll2002-en.crfsuite')

    example_sent = test_sents[0]
    print(' '.join(sent2tokens(example_sent)), end='\n\n')

    print("Predicted:", ' '.join(tagger.tag(sent2features(example_sent))))
    print("Correct:  ", ' '.join(sent2labels(example_sent)))

    y_pred = [tagger.tag(xseq) for xseq in X_test]

    print(bio_classification_report(y_test, y_pred))

    info = tagger.info()

    print("Top likely transitions:")
    print_transitions(Counter(info.transitions).most_common(15))

    print("\nTop unlikely transitions:")
    print_transitions(Counter(info.transitions).most_common()[-15:])

    ' '.split('-', 1)

    print("Top positive:")
    print_state_features(Counter(info.state_features).most_common(20))

    print("\nTop negative:")
    print_state_features(Counter(info.state_features).most_common()[-20:])