import requests
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
import logging
import json

from singular_api_client.helpers import CohortMetric, SkanEventsResponse, SkanEvent
from .params import Format, Dimensions, DiscrepancyMetrics, TimeBreakdown, CountryCodeFormat, Metrics
from .exceptions import ArgumentValidationException, APIException, UnexpectedAPIException
from .helpers import ReportStatusResponse, CustomDimension, CohortMetricsResponse, \
    DataAvailabilityResponse
from .version import __version__

logger = logging.getLogger("singular_client")

class SingularClient(object):
    """
    Client for Singular Reporting API
    See:
        - https://github.com/singular-labs/singular_api_client
        - https://developers.singular.net/v2.0/reference
    """
    BASE_API_URL = "https://api.singular.net/api/"
    DEFAULT_HTTP_TIMEOUT = 60 * 5

    def __init__(self, api_key, http_timeout=DEFAULT_HTTP_TIMEOUT, user_agent='Singular API Client v%s' % __version__):
        self.api_key = api_key
        self.user_agent = user_agent
        session = requests.Session()
        retry = Retry(
            connect=5,
            backoff_factor=0.5,
            status_forcelist=(500, 502, 503, 504),
            method_whitelist=('GET', 'POST')
        )
        adapter = HTTPAdapter(max_retries=retry)
        session.mount('http://', adapter)
        session.mount('https://', adapter)
        self.default_http_timeout = http_timeout
        self.session = session

    def create_async_report(self, start_date, end_date,
                            format=Format.JSON,
                            dimensions=(Dimensions.APP, Dimensions.OS, Dimensions.SOURCE),
                            metrics=(Metrics.ADN_COST, Metrics.ADN_IMPRESSIONS),
                            discrepancy_metrics=(DiscrepancyMetrics.ADN_CLICKS, DiscrepancyMetrics.ADN_INSTALLS),
                            cohort_metrics=None,
                            cohort_periods=None,
                            source=None,
                            app=None,
                            display_alignment=True,
                            time_breakdown=TimeBreakdown.ALL,
                            country_code_format=CountryCodeFormat.ISO3,
                            filters=None,
                            **kwargs
                            ):
        """
        Use this endpoint to run custom queries in the Singular platform for aggregated statistics without keeping
         a live connection throughout the request

        :param start_date: "YYYY-mm-dd" format date
        :param end_date: "YYYY-mm-dd" format date
        :param format: Format for returned results, for example Format.CSV
        :param dimensions: A list of dimensions, for example [Dimensions.APP, Dimensions.Source]
        :param metrics: A list of metrics, for example [Metrics.ADN_IMPRESSIONS, Metrics.ADN_COST]
        :param discrepancy_metrics: List of metrics that may help detect discrepancies between Ad Networks
         and Attribution providers, for example [DiscrepancyMetrics.ADN_CLICKS, DiscrepancyMetrics.ADN_INSTALLS]
        :param cohort_metrics: list of cohorted metrics by name or ID; A full list can be retrieved through
         the Cohorted Metrics endpoint
        :param cohort_periods: list of cohorted periods; A full list can be retrieved through the Cohorted Metrics
          endpoint
        :param source: optional list of source names to filter by
        :param app: optional list of app names to filter by
        :param display_alignment: When set to True, results will include an alignment row to account for any difference
         between campaign and creative statistics
        :param time_breakdown: Break results by the requested time period, for example TimeBreakdown.DAY
        :param country_code_format: Country code formatting option, for example CountryCodeFormat.ISO3
        :param filters: a JSON encoded list of filters. Can be used to apply more complex filters than simply filtering
          by app or source. The relation between different elements of the list is an AND relation.
          A full list of the dimensions you can filter by and potential values can be retrieved from the
          `get_reporting_filters` endpoint.
        :return: report_id
        """

        query_dict = self._build_reporting_query(start_date, end_date, format, dimensions, metrics,
                                                 discrepancy_metrics, cohort_metrics, cohort_periods, app,
                                                 source, display_alignment, time_breakdown, country_code_format,
                                                 filters, **kwargs)

        response = self._api_post("v2.0/create_async_report", data=query_dict)
        parsed_response = response.json()
        return parsed_response["value"]["report_id"]

    def create_async_skadnetwork_raw_report(self, start_date, end_date,
                                            format=Format.JSON,
                                            dimensions=(Dimensions.APP, Dimensions.SOURCE,
                                                        Dimensions.SKAN_CAMPAIGN_ID, Dimensions.SKAN_CONVERSION_VALUE),
                                            metrics=(Metrics.SKAN_INSTALLS,),
                                            source=None,
                                            app=None,
                                            time_breakdown=TimeBreakdown.ALL,
                                            country_code_format=CountryCodeFormat.ISO3,
                                            filters=None,
                                            skadnetwork_date_type=None,
                                            **kwargs
                                            ):
        """
        Use this endpoint to run custom queries in the Singular platform for aggregated skadnetwork raw data without
         keeping a live connection throughout the request

        :param start_date: "YYYY-mm-dd" format date
        :param end_date: "YYYY-mm-dd" format date
        :param format: Format for returned results, for example Format.CSV
        :param dimensions: A list of dimensions, for example [Dimensions.APP, Dimensions.Source, Dimensions.SKAN_CAMPAIGN_ID]
        :param metrics: A list of metrics, for example [Metrics.SKAN_INSTALLS]
        :param source: optional list of source names to filter by
        :param app: optional list of app names to filter by
        :param time_breakdown: Break results by the requested time period, for example TimeBreakdown.DAY
        :param country_code_format: Country code formatting option, for example CountryCodeFormat.ISO3
        :param filters: a JSON encoded list of filters. Can be used to apply more complex filters than simply filtering
          by app or source. The relation between different elements of the list is an AND relation.
          A full list of the dimensions you can filter by and potential values can be retrieved from the
          `get_reporting_filters` endpoint.
        :param skadnetwork_date_type: the type of date you want the report to be based on:
            - "skan_postback_date" (default) - the date the SKAN postback was sent by the device.
            - "estimated_install_date" - the install date as calculated by
        :return: report_id
        """

        query_dict = self._build_skan_reporting_query(start_date, end_date, format, dimensions, metrics,
                                                      app, source, time_breakdown, country_code_format,
                                                      filters, skadnetwork_date_type, None, **kwargs)

        response = self._api_post("v2.0/create_async_skadnetwork_raw_report", data=query_dict)
        parsed_response = response.json()
        return parsed_response["value"]["report_id"]

    def create_async_skadnetwork_report(self, start_date, end_date,
                                        format=Format.JSON,
                                        dimensions=(Dimensions.APP, Dimensions.SOURCE,
                                                    Dimensions.SKAN_CAMPAIGN_ID, Dimensions.SKAN_CONVERSION_VALUE),
                                        metrics=(Metrics.SKAN_INSTALLS,),
                                        discrepancy_metrics=[],
                                        source=None,
                                        app=None,
                                        time_breakdown=TimeBreakdown.ALL,
                                        country_code_format=CountryCodeFormat.ISO3,
                                        filters=None,
                                        skadnetwork_date_type=None,
                                        skan_events=None,
                                        **kwargs
                                        ):
        """
        Use this endpoint to run custom queries in the Singular platform for aggregated skadnetwork data without
         keeping a live connection throughout the request

        :param start_date: "YYYY-mm-dd" format date
        :param end_date: "YYYY-mm-dd" format date
        :param format: Format for returned results, for example Format.CSV
        :param dimensions: A list of dimensions, for example [Dimensions.APP, Dimensions.Source, Dimensions.SKAN_CAMPAIGN_ID]
        :param metrics: A list of metrics, for example [Metrics.SKAN_INSTALLS]
        :param discrepancy_metrics: List of metrics that may help detect discrepancies between Ad Networks
         and Attribution providers, for example [DiscrepancyMetrics.TRACKER_INSTALLS]
        :param source: optional list of source names to filter by
        :param app: optional list of app names to filter by
        :param time_breakdown: Break results by the requested time period, for example TimeBreakdown.DAY
        :param country_code_format: Country code formatting option, for example CountryCodeFormat.ISO3
        :param filters: a JSON encoded list of filters. Can be used to apply more complex filters than simply filtering
          by app or source. The relation between different elements of the list is an AND relation.
          A full list of the dimensions you can filter by and potential values can be retrieved from the
          `get_reporting_filters` endpoint.
        :param skadnetwork_date_type: the type of date you want the report to be based on:
            - "skan_postback_date" (default) - the date the SKAN postback was sent by the device.
            - "estimated_install_date" - the install date as calculated by
        :param skan_events: list of skan events by name or ID; A full list can be retrieved through
            the Skan Events endpoint
        :return: report_id
        """

        query_dict = self._build_skan_reporting_query(start_date, end_date, format, dimensions, metrics,
                                                      app, source, time_breakdown, country_code_format,
                                                      filters, skadnetwork_date_type, skan_events,
                                                      discrepancy_metrics=discrepancy_metrics,
                                                      **kwargs)

        response = self._api_post("v2.0/create_async_skadnetwork_report", data=query_dict)
        parsed_response = response.json()
        return parsed_response["value"]["report_id"]

    def get_report_status(self, report_id):
        """
        This endpoint returns the status of a given report.
          If a report has failed, an error message is returned. If the report completes, a download url is returned
          together with the status.

        :param report_id: id generated by the create_async_report method
        :return: return the status of the report
        :rtype: ReportStatusResponse
        """
        params = {"report_id": report_id}
        response = self._api_get("v2.0/get_report_status", params=params)
        parsed_response = response.json()
        self._verify_legacy_error(parsed_response)
        return ReportStatusResponse(parsed_response["value"])

    def get_custom_dimensions(self):
        """
        Use this endpoint to return all the custom dimensions configured for your account by name and ID.
          Dimension IDs can then be used in Reporting API queries to group the data using Custom Dimensions

        :return: list of `CustomDimension` instances
        :rtype: list[CustomDimension]
        """
        response = self._api_get("custom_dimensions")
        parsed_response = response.json()
        self._verify_legacy_error(parsed_response)
        return CustomDimension.parse_list(parsed_response["value"]["custom_dimensions"])

    def get_cohort_metrics(self):
        """
        Use this endpoint to return all cohorted metrics and cohort periods configured for your account

        :return: a new `CohortMetricsResponse` instance
        :rtype: CohortMetricsResponse
        """
        response = self._api_get("cohort_metrics")
        parsed_response = response.json()
        self._verify_legacy_error(parsed_response)
        return CohortMetricsResponse(parsed_response["value"])

    def get_skan_events(self):
        """
        Use this endpoint to return all skan events for your account

        :return: a new `SkanEventsResponse` instance
        :rtype: SkanEventsResponse
        """
        response = self._api_get("v2.0/skan_events")
        parsed_response = response.json()
        self._verify_legacy_error(parsed_response)
        return SkanEventsResponse(parsed_response["value"])

    def data_availability_status(self, data_date, format=Format.JSON, display_non_active_sources=False):
        """
        Use this endpoint to determine whether for a given day, data is available for each of your data data sources.
         This data can then be used to determine whether to pull data.

        :param data_date: You can only select a single day. The API will check whether there is data for this day.
            Date format: "YYYY-mm-dd"
        :param format: Format for returned results, for example Format.CSV
        :type format: str
        :param display_non_active_sources: Active source is defined as a source that has data in the last 30 days
        :type display_non_active_sources: bool
        :return: DataAvailabilityResponse if format==Format.JSON, or unicode if format==Format.CSV
        :rtype: DataAvailabilityResponse | unicode
        """

        self._verify_param("format", format, Format)
        query_dict = dict(data_date=data_date, format=format,
                          display_non_active_sources=self._bool(display_non_active_sources))

        response = self._api_get("v2.0/data_availability_status", params=query_dict)
        if format == Format.JSON:
            parsed_response = response.json()
            self._verify_legacy_error(parsed_response)
            return DataAvailabilityResponse(parsed_response["value"])
        elif format == Format.CSV:
            return response.text
        else:
            raise ArgumentValidationException("unsupported format")

    def get_reporting_filters(self):
        """
        This endpoint returns all available filters and their respective available options.
        Please note that filters can differ between users, per the configured.

        example response:
        {
            "dimensions": [
              {
                "name": "os",
                "display_name": "OS",
                "values": [
                    {"name": 4, "display_name": "Android"},
                  {"name": 1, "display_name": "iOS"}
                ],
              },
              {
                "name": "source",
                "display_name": "Source",
                "values": [
                  {"name": "adwords", "display_name": "AdWords"}
                ]
              }
            ]
          }

        :return: dictionary of available filters and their respected values
        :rtype: dict[str, list[dict]]
        """
        response = self._api_get("v2.0/reporting/filters")
        parsed_response = response.json()
        self._verify_legacy_error(parsed_response)
        return parsed_response["value"]

    @staticmethod
    def _bool(value):
        if value:
            return "true"
        else:
            return "false"

    @classmethod
    def _build_reporting_query(cls, start_date, end_date, format, dimensions, metrics, discrepancy_metrics,
                               cohort_metrics, cohort_periods, app, source, display_alignment, time_breakdown,
                               country_code_format, filters, **kwargs):
        """
        build reporting query format that can be used by either the `create_async_report` or `reporting` endpoints
        """
        cls._verify_param("format", format, Format)
        cls._verify_param("time_breakdown", time_breakdown, TimeBreakdown)
        cls._verify_param("country_code_format", country_code_format, CountryCodeFormat)

        if filters is None:
            filters = []

        if (cohort_metrics or cohort_periods) and (not cohort_metrics or not cohort_periods):
            raise ArgumentValidationException("`cohort_metrics` must be used with `cohort_periods`")

        dimensions_request = ",".join(dimensions)
        metrics_request = ",".join(metrics)
        discrepancy_metrics_request = ",".join(discrepancy_metrics)
        query_dict = dict(
            start_date=start_date,
            end_date=end_date,
            dimensions=dimensions_request,
            metrics=metrics_request,
            discrepancy_metrics=discrepancy_metrics_request,
            display_alignment=display_alignment,
            format=format,
            time_breakdown=time_breakdown,
            country_code_format=country_code_format,
        )
        if source is not None:
            if not isinstance(source, list):
                sources = [source]
            else:
                sources = source
            filters.append({"dimension": "source", "operator": "in", "values": sources})
        if app is not None:
            if not isinstance(app, list):
                apps = [app]
            else:
                apps = app
            filters.append({"dimension": "app", "operator": "in", "values": apps})
        if cohort_metrics:
            if isinstance(cohort_metrics, list):
                cohort_metrics = [(i.name if isinstance(i, CohortMetric) else i) for i in cohort_metrics]
                cohort_metrics = ",".join(cohort_metrics)
            query_dict.update({'cohort_metrics': cohort_metrics})
        if cohort_periods:
            if isinstance(cohort_periods, list):
                cohort_periods = ",".join(cohort_periods)
            query_dict.update({'cohort_periods': cohort_periods})
        if filters:
            query_dict["filters"] = json.dumps(filters)

        query_dict.update(kwargs)
        return query_dict

    @classmethod
    def _build_skan_reporting_query(cls, start_date, end_date, format, dimensions, metrics, app, source, time_breakdown,
                                    country_code_format, filters, skadnetwork_date_type, skan_events,
                                    discrepancy_metrics=[],
                                    **kwargs):
        query_dict = cls._build_reporting_query(start_date, end_date, format, dimensions, metrics, discrepancy_metrics,
                                                None, None, app, source, None, time_breakdown,
                                                country_code_format, filters, **kwargs)

        if skadnetwork_date_type:
            query_dict.update({'skadnetwork_date_type': skadnetwork_date_type})
        if skan_events:
            if isinstance(skan_events, list):
                skan_events = [(event.name if isinstance(event, SkanEvent) else event) for event in skan_events]
                skan_events = ",".join(skan_events)
            query_dict.update({'skan_events': skan_events})

        return query_dict

    @staticmethod
    def _verify_param(param_name, value, base_class):
        expected_values = base_class.__ALL_OPTIONS__
        if value not in expected_values:
            raise ArgumentValidationException("unexpected %s value %s, expected one of %s" %
                                              (param_name, repr(value), repr(expected_values)))

    @staticmethod
    def _verify_legacy_error(parsed_response):
        if parsed_response["status"] != 0:
            raise APIException("API request failed: %s" % parsed_response["value"])

    def _api_get(self, endpoint, params=None):
        return self._api_request("GET", endpoint, params=params)

    def _api_post(self, endpoint, data=None, json=None):
        return self._api_request("POST", endpoint, data=data, json=json)

    def _api_request(self, method, endpoint, **kwargs):
        url = self.BASE_API_URL + endpoint
        headers = {"Authorization": self.api_key,
                   'User-Agent': self.user_agent}

        response = self.session.request(method, url,
                                        headers=headers,
                                        timeout=self.default_http_timeout,
                                        **kwargs)

        logger.info("%(method)s %(url)s, kwargs = %(kwargs)s --> code = %(code)s" %
                    dict(method=method, url=url, kwargs=repr(kwargs), code=response.status_code))

        if not response.ok:
            if response.status_code is None or response.status_code >= 500 < 600:
                raise UnexpectedAPIException("%s failed with code = %s, payload = %s" % (
                    endpoint, response.status_code, response.text))
            else:
                raise APIException("%s failed with code = %s, payload = %s" % (
                    endpoint, response.status_code, response.text))

        return response


import cv2
import cv2 as cv

imagename="C:\\Users\\admin\Desktop\\2.png"
image = cv2.imread(imagename, 0)
blood = cv2.normalize(image.astype('double'), None, 0.0, 1.0, cv2.NORM_MINMAX)  # Convert to normalized floating point
# 用normalize函数导致图像中的像素为浮点型，导致后面用HoughLinesP函数时出错？
outIm = FrangiFilter2D(blood)
edges = outIm * (10000)
# edges=outIm*(10000).astype(np.int8)  # 这样也不行

cv2.imshow('Frangi Filter Result', edges)

minLineLength = 200
maxLineGap = 50
lines = cv2.HoughLinesP(edges, 1, np.pi / 180, 80, minLineLength, maxLineGap)


import random

import pytest
from populus.contracts.exceptions import NoKnownAddress
from pytest import raises

from nkms_eth.escrow import Escrow
from nkms_eth.miner import Miner
from nkms_eth.token import NuCypherKMSToken

M = 10 ** 6

def test_create_escrow(testerchain):
    with raises(NoKnownAddress):
        NuCypherKMSToken.get(blockchain=testerchain)

    token = NuCypherKMSToken(blockchain=testerchain)
    token.arm()
    token.deploy()

    same_token = NuCypherKMSToken.get(blockchain=testerchain)
    with raises(NuCypherKMSToken.ContractDeploymentError):
        same_token.arm()
        same_token.deploy()

    assert len(token.contract.address) == 42
    assert token.contract.address == same_token.contract.address

    with raises(NoKnownAddress):
        Escrow.get(blockchain=testerchain, token=token)

    escrow = Escrow(blockchain=testerchain, token=token)
    escrow.arm()
    escrow.deploy()

    same_escrow = Escrow.get(blockchain=testerchain, token=token)
    with raises(Escrow.ContractDeploymentError):
        same_escrow.arm()
        same_escrow.deploy()

    assert len(escrow.contract.address) == 42
    assert escrow.contract.address == same_escrow.contract.address

def test_get_swarm(testerchain, token, escrow):
    token._airdrop(amount=10000)

    # Create 9 Miners
    for u in testerchain._chain.web3.eth.accounts[1:]:
        miner = Miner(blockchain=testerchain, token=token, escrow=escrow, address=u)
        amount = (10+random.randrange(9000)) * M
        miner.lock(amount=amount, locktime=1)

    testerchain.wait_time(escrow.hours_per_period)

    swarm = escrow.swarm()
    swarm_addresses = list(swarm)
    assert len(swarm_addresses) == 9

    # Grab a miner address from the swarm
    miner_addr = swarm_addresses[0]
    assert isinstance(miner_addr, str)

    # Verify the address is hex
    try:
        int(miner_addr, 16)
    except ValueError:
        pytest.fail()


import os
import json

import requests

GOOGLE_API = 'https://www.googleapis.com/urlshortener/v1/url'
API_KEY = os.environ['api_key']
google_url = '{}?key={}'.format(GOOGLE_API, API_KEY)

def handler(event, context):

    print(json.dumps(event))

    payload = {'longUrl': event['url']}
    r = requests.post(google_url, data=json.dumps(payload), headers={'content-type': 'application/json'})
    result = r.json()
    print(json.dumps(result))

    return result


#!/usr/bin/python
#data.py

'''
Classes and functions for implementing data collection and output
'''

#geonmics imports
from geonomics.utils.io import (_write_csv, _write_shapefile, _write_geojson,
                                _write_file)

#other imports
import numpy as np
from random import sample as rand_sample
from numpy import random as r
import os, sys
import datetime
import re
from shapely.geometry import Point
from shapely.ops import cascaded_union
import pandas as pd
import geopandas as gpd
from itertools import chain

######################################
# -----------------------------------#
# CLASSES ---------------------------#
# -----------------------------------#
######################################

#a DataCollector class, to paramtereize and manage data
#collection, and write data to disk
class _DataCollector:
    def __init__(self, model_name, params):

    #some lookup dicts for writing data
        self.file_extension_dict =   {'vcf': 'vcf',
                            'fasta': 'fasta',
                            'csv': 'csv',
                            'shapefile': 'shp',
                            'geojson': 'json',
                            'geotiff': 'tif'
                            }

        self.write_geodata_fn_dict = {'csv': _write_csv,
                            'shapefile': _write_shapefile,
                            'geojson': _write_geojson,
                            }

        #set model name and T
        self.model_name = model_name
        self.T = params.model.T

        #grab the params['data'] contents into objects
        sampling_params = params.model.data.sampling
        format_params = params.model.data.format

        #get the sampling scheme
        self.scheme = sampling_params.scheme

        #and run some asserts
        assert self.scheme in ['all', 'random',
            'point', 'transect'], ("The sampling scheme provided in the "
            "parameters must be one of the following values: 'all', 'random', "
            "'point', or 'transect'.")

        if sampling_params.scheme != 'all':
            assert 'n' in sampling_params.keys(), ("If the "
            "sampling scheme is not 'all' then the 'n' parameter must be "
            "defined, indicating the number of individuals to be sampled "
            "each time data is collected.")
            assert type(sampling_params.n) is int, ("The "
            "'n' data-sampling parameter must be an integer.")
        #TODO: Add more assert statements here to check that only the right
        #combinations of parameters can be provided

        #get the number of individuals to sample, if applicable
        self.n = None
        if sampling_params.scheme != 'all':
            self.n = sampling_params.n

        #calculate the transect points, if 'transect' is the chosen sampling
        #scheme, or just get the points if 'point' is the scheme
        self.pts = None
        if sampling_params.scheme == 'point':
            self.pts = sampling_params.points
        if sampling_params.scheme == 'transect':
            endpts = sampling_params.transect_endpoints
            n_transect_pts = sampling_params.n_transect_points
            pts = _get_transect_points(endpoints = endpts, n = n_transect_pts)
            self.pts = pts

        #create the point buffers, if either 'transect' or 'point' is the
        #chosen sampling scheme
        self.pts_area = None
        if sampling_params.scheme in ['transect', 'point']:
            self.pts_area = _make_point_buffers(self.pts,
                                        sampling_params.radius)

        #get the 'include_landscape' param (defaults False)
        self.include_landscape = False
        if ('include_landscape' in sampling_params.keys() and
            type(sampling_params.include_landscape) is bool):
            self.include_landscape = sampling_params.include_landscape

        #get the 'include_fixed_sites' param (defaults False)
        self.include_fixed_sites = False
        if ('include_fixed_sites' in sampling_params.keys() and
            type(sampling_params.include_landscape) is bool):
            self.include_fixed_sites = sampling_params.include_fixed_sites

        #set the when attribute to the 'when' parameter
        self.when = sampling_params.when
        #check type- and value-validity of self.when, and update its value
        #as necessary
        assert type(self.when in (list, float, int, type(None)))
        #if it's a list, make sure no values are greater than final timestep
        if type(self.when) is list:
            assert np.array([n < self.T for n in self.when]).all(), (' Values '
                'provided for sampling times must be less than total '
                'model run-time.')
        #if it's a float, int, or None
        elif type(self.when) in (float, int, type(None)):
            #check value is less than or equal to last timestep (or None)
            assert self.when is None or self.when < self.T, ('Values '
            'provided for sampling times must be less than total '
            'model run-time.')
            #make it a list containing nothing (last timestep will be added),
            #if 0 or None
            if self.when in (0, None):
                self.when = []
            #make it a stepwise timestep list, if integer other than 0
            else:
                self.when = [*range(0, self.T, int(self.when))]
        #add the final timestep, if not already in self.when
        if (len(self.when) == 0 or self.when[-1] != self.T-1):
            self.when.append(self.T-1)
        #now turn the when attribute into an iterator
        self.when = iter(self.when)
        #and set next_t
        self._set_next_t()

        #grab the genetic data formats as a separate attribute
        self.gen_formats = format_params.gen_format
        #change the gen_formats attribute to a list if it came in as a string
        if type(self.gen_formats) == str:
            self.gen_formats = [self.gen_formats]
        #also grab the geographic data formats as a separate attribute
        self.geo_formats = [format_params.geo_vect_format]
        #and grab the raster format, if a raster is required
        #NOTE: added to a separate attribute because this is written per
        #timestep, not per species within timestep
        self.rast_format = None
        if (sampling_params.include_landscape
            and 'geo_rast_format' in format_params.keys()):
            self.rast_format = format_params.geo_rast_format

        # grab the format for CSV files containing the non-neutral loci
        # NOTE: None indicates that these files should not be produced
        self.nonneut_loc_format = format_params.nonneut_loc_format
        assert self.nonneut_loc_format in ['csv', None], ("the "
            "'nonneut_loc_format' parameter must be either 'csv' or None.")

    #method to set self.next_t
    def _set_next_t(self):
        try:
            self.next_t = next(self.when)
        #if there are no further timestep values left then this is the last
        #timestep, so set self.next_t to None
        except StopIteration:
            assert self.next_t == self.T -1, ("Model._set_next_t() threw a "
            "StopIteration error, but the current value of Model.next_t is "
            "not the final timestep (instead, it is %i).\n\n") % self.next_t
            self.next_t = None

    #method to create filenames for genetic and geographic datafiles
    def _make_filenames(self, iteration, spp_name):
        filenames = []
        for att_name in ['gen_formats', 'geo_formats']:
            filenames.append(['mod-%s_it-%i_t-%i_spp-%s.%s' % (self.model_name,
              iteration, self.next_t, spp_name, self.file_extension_dict[fmt])
                        for fmt in getattr(self, att_name)])
        return(filenames)

    #a method to be called each timestep, which will collect needed
    #data and then write the data (if write_intermittent == True) if it's
    #the right timestep
    #TODO: CONSIDER NOMENCLATURE CHANGE HERE AND IN CLASS NAME!
    def _write_data(self, community, land, iteration):

        #if this timestep is scheduled for sampling
        if community.t == self.next_t:

            #for each species
            for spp in community.values():

                #TODO: Probably get rid of this conditional; should be
                #unnecessary, and is not a direct check of the in-sync
                #assumption at any rate
                #double-check that each species' timestep is in sync with
                #comm.t and is scheduled for sampling
                if spp.t == self.next_t:

                    #get the data directory name for this timestep
                    dirname = os.path.join(os.getcwd(),
                          'GNX_mod-%s' % self.model_name,
                                    'it-%i' % iteration)

                    #get the subdirectory for this species
                    subdirname = os.path.join(dirname, 'spp-%s' % spp.name)

                    #and create (and its parent data directory, if needed)
                    os.makedirs(subdirname, exist_ok = True)

                    #get filenames
                    gen_files, geo_files = self._make_filenames(
                                    iteration = iteration, spp_name = spp.name)

                    #sample individuals according to the scheme defined
                    sample = self._get_sample(spp)

                    #write files, if sample length > 0
                    #(NOTE: otherwise, an empty file with "ZERO_SAMPLE" in the
                    #name will be written, below)
                    if len(sample) > 0:

                        #save genetic data, if the spp has a
                        #genomic architeecture
                        if spp.gen_arch is not None:
                            #for each genetic data format to be written
                            for n, data_format in enumerate(self.gen_formats):

                                #format the data accordingly
                                data = self._format_gen_data(
                                    data_format = data_format, sample = sample,
                                                                    spp = spp)

                                #then write it to disk
                                gen_filepath = os.path.join(subdirname,
                                                            gen_files[n])
                                self._write_gendata(filepath = gen_filepath,
                                                            gen_data = data)

                        #also write the geodata for this spp
                        for n, data_format in enumerate(self.geo_formats):
                            #write the geodata to disk
                            geo_filepath = os.path.join(subdirname,
                                                            geo_files[n])
                            self._write_geodata(filepath = geo_filepath,
                                               data_format = data_format,
                                               sample = sample)

                    #if sample was empty, write a placeholder file with name
                    #"<base_filename>_ZERO_SAMPLE"
                    else:
                        filenames = [gen_files, geo_files]
                        filenames = [*chain.from_iterable(filenames)]
                        filename = os.path.splitext(filenames[0])[0]
                        filename = filename + '_ZERO_SAMPLE'
                        filepath = os.path.join(subdirname, filename)
                        self._write_gendata(filepath, '')

                # write the non-neutral loci CSV file, if required,
                # and if this is the final timestep
                if self.nonneut_loc_format is not None:
                    self._write_nonneut_loc_file(spp, subdirname, iteration)

            #write the raster, if necessary
            if self.rast_format is not None:
                #for each Layer
                for lyr in land.values():
                    #get the raster filename
                    filename = 'mod-%s_it-%i_t-%i_lyr-%s.%s' % (
                        self.model_name, iteration, self.next_t, lyr.name,
                                    self.file_extension_dict[self.rast_format])
                    filepath = os.path.join(dirname, filename)
                    #and write it to disk
                    lyr._write_raster(filepath, self.rast_format)

            #update self.next_t to the next timestep to be sampled
            self._set_next_t()

    def _get_random_sample(self, individuals):
        if len(individuals) > self.n:
            sample = r.choice(individuals, size = self.n, replace = False)
        else:
            sample = individuals
        return(sample)

    def _get_point_sample(self, spp):
        #TODO: check if this should be sped up any more
        sample = [i for i,v in spp.items() if self.pts_area.contains(
                                                        Point(v.x, v.y))]
        if len(sample) > self.n:
            sample = self._get_random_sample(individuals = sample)
        return(sample)

    def _get_sample(self, spp):
        #get a set of indices for the individuals in the sample
        sample = set()
        #take the whole species, if scheme is 'all'
        if self.scheme == 'all':
            sample.update([*spp])
        #or take a random sample, if scheme is 'random'
        elif self.scheme == 'random':
            inds = self._get_random_sample([*spp])
            sample.update(inds)
        #or take individuals within a given radius of self.buffs (which could
        #have come from a set of input points or from a calculated set of
        #transect points), if scheme is 'point' or 'transect'
        elif self.scheme in ['point', 'transect']:
            inds = self._get_point_sample(spp)
            sample.update(inds)
        #sort sample, then convert to a dict of individuals
        sample = sorted([*sample])
        sample = {i:spp[i] for i in sample}
        return(sample)

    def _format_gen_data(self, data_format, sample, spp):
        '''<data_format> can be:
                            'fasta'
                            'vcf'
        '''
        genotypes = spp._get_genotypes(individs=[*sample], as_dict=True)
        if data_format == 'fasta':
            formatted_data = _format_fasta(sample, genotypes)
        elif data_format == 'vcf':
            formatted_data = _format_vcf(sample, genotypes, spp.gen_arch,
                    include_fixed_sites = self.include_fixed_sites)
        return(formatted_data)

    def _write_gendata(self, filepath, gen_data):
        _write_file(filepath, gen_data)

    def _write_geodata(self, filepath, data_format, sample):
        write_fn = self.write_geodata_fn_dict[data_format]
        write_fn(filepath = filepath, individuals = sample)

    def _write_nonneut_loc_file(self, spp, subdir, iteration,
                                format='csv'):
        """
        NOTE: this is agnostic of the fact that the number of non-neutral
              loci can grow throughout a simulation because of mutation.
              the user would need to compare across nonneutral locus files,
              and/or compare with the mutation log,
              in order to determine when loci became non-neutral.
        """
        # get all the non-neutral loci for each trait
        locs_dict = {trt.name:
                     [*trt.loci] for trt in spp.gen_arch.traits.values()}
        # make the number of rows even across columns
        max_nrow = np.max([len(v) for v in locs_dict.values()])
        for k,v in locs_dict.items():
            if len(v) < max_nrow:
                locs_dict[k] = np.array([*v] + [np.nan]*(max_nrow-len(v)))
        # recast as a DataFrame
        locs_df = pd.DataFrame.from_dict(locs_dict)
        # make the filename and filepath
        filename = 'mod-%s_it-%i_t-%i_spp-%s_NONNEUTS.%s' % (self.model_name,
                                                             iteration,
                                                             self.next_t,
                                                             spp.name,
                                                             format)
        filepath = os.path.join(subdir, filename)
        # write the file, with trait names as column names
        # and locus numbers down the non-fixed-length columns
        locs_df.to_csv(filepath, index=False)

######################################
# -----------------------------------#
# FUNCTIONS -------------------------#
# -----------------------------------#
######################################

#a function to get a set of n evenly spaced points between endpoints
def _get_transect_points(endpoints, n):
    x_pts = np.linspace(endpoints[0][0] , endpoints[1][0], n)
    y_pts = np.linspace(endpoints[0][1] , endpoints[1][1], n)
    return(list(zip(x_pts, y_pts)))

#a function to make shapely geometry buffers around a set of points
def _make_point_buffers(points, radius):
    pts = [Point(*p) for p in points]
    buffs = [p.buffer(radius) for p in pts]
    cu = cascaded_union(buffs)
    return(cu)

# a function that returns an ad hoc sample of size n
# from the current population of the given spp,
# in the format index-keyed dict format needed by the data-writing functions
# (to be used by the convenience methods of the Model object)
def _get_adhoc_sample(spp, n):
    #get a set of indices for the individuals in the sample
    sample = set()
    #take the whole species, if scheme is 'all'
    if n is None:
        sample.update([*spp])
    #or take a random sample, if scheme is 'random'
    else:
        if len(spp) > n:
            inds = r.choice([*spp], size = n, replace = False)
        else:
            inds = [*spp]
        sample.update(inds)
    #sort sample, then convert to a dict of individuals
    sample = sorted([*sample])
    sample = {i:spp[i] for i in sample}
    return(sample)

def _format_fasta(sample, genotypes):
    '''
    FASTA FORMAT:

    >idx:haploid_num|x_location|y_location|phenotype0;phenotype1;...;
                    phenotypeN|env_var0;env_var1;...;env_varN|age|sex
    001110101010101010010101011101010110.....01011110

    '''
    # ensure that sample and genotypes have identical lenghts and orders
    assert len(sample) == len(genotypes), ("'sample', and 'genotypes' have"
                                           "different lenghts!")
    assert np.all([*sample] == [*genotypes]), ("'sample' and 'genotypes' do not"
                                               " have identical orders!")

    row1 = '>%s:HAP;%s;%s;%s;%s;%s;%s\n'
    file_text = ''

    for individ, genotype in zip(sample.values(), genotypes.values()):
        for hap in range(2):
            individ_row1 = re.sub('HAP', str(hap), row1)
            replace = tuple(map(lambda att: re.sub(',', '|', re.sub('[\[\] ]',
                '', str(getattr(individ, att)))), ['idx', 'x', 'y', 'age',
                                            'sex', 'z', 'e']))
            individ_row1 = individ_row1 % replace
            individ_row2 = ''.join([str(
                            base) for base in genotype[:,hap]]) + '\n'

            file_text = file_text + individ_row1 + individ_row2

    return(file_text)

def _format_vcf(sample, genotypes, gen_arch, include_fixed_sites=False):

    # ensure that sample and genotypes have identical lenghts and orders
    assert len(sample) == len(genotypes), ("'sample', and 'genotypes' have"
                                           "different lenghts!")
    assert np.all([*sample] == [*genotypes]), ("'sample' and 'genotypes' do not"
                                               " have identical orders!")

    #create a template header
        #NOTE: has 1 string slot for a date

        #TODO: DECIDE ON NECESSARY INFO AND FORMAT CONTENTS,
        #THEN ADD METADATA ROWS HERE

    header = '''##fileformat=VCFv4.2
##fileDate=%s
##source=Geonomics
'''

    #template column-header row
    col_header_row = ('#CHROM\tPOS\tID\tREF\tALT\tQUAL'
                      '\tFILTER\tINFO\tFORMAT\t%s\n')
        #NOTE: this has 1 string slot for a tab-separated
        #list of all individ ids

    #template data row
    #TODO: UPDATE/CHANGE THE INFO AND FORMAT PORTIONS OF THIS TEMPLATE,
    #AFTER I DECIDE ON THEIR CONTENTS (above)
    data_row = ('%i\t%i\t.\tA\tT\t1000\tPASS\t%s\tGT\t%s\n')
        #NOTE: this has 2 integer slots, then 1 string slot for:
            #- chrom number (NOTE: unpythonically, starts from 1)
            #- locus number (NOTE: reported cumulative from locus 0,
               #not from start of each chrom)
            #- a tab-separated list of individs' genotypes at this locus

    #create a col_header_row for this data
    inds = [*sample.keys()]
    ind_cols = '\t'.join([str(i) for i in inds])
    cols = col_header_row % (ind_cols)

    #get a list of the chromosome numbers
    #chroms = np.cumsum(gen_arch.l_c)

    #and get all individuals' genomic data in a 'samplome' object (a 3-d array)
    samplome = np.array([genotypes[i] for i in inds])

    #get all segregating sites
    max_val = 2 * len(sample)
    segs = np.where(samplome.sum(axis = 2).sum(axis = 0) > 0)[0]
    segs2 = np.where(samplome.sum(axis = 2).sum(axis = 0) < max_val)[0]
    segs = sorted(list(set(segs).intersection(set(segs2))))

    #if not_include_fixed_sites, include only the segregating sites in the VCF
    if not include_fixed_sites:
        loci = segs
    #or else get all loci
    else:
        loci = range(gen_arch.L)

    #and get the sites' chrom nums
    #chroms = [list((locus - chroms) < 0).index(True) for locus in loci]

    #build all the VCF data rows
    rows = ''
    for n, locus in enumerate(loci):
        gts = samplome[:,locus,:]
        gts = '\t'.join(['%i|%i' % (gts[i,0],
                gts[i,1]) for i in range(np.shape(gts)[0])])
        #get an indicator for whether the site is segregating or fixed
        seg = locus in segs
        seg_dict = {True: 'SEG', False: 'FIX'}

        rows = rows + data_row % (0, locus, seg_dict[seg], gts)

    #get the date
    now = datetime.datetime.now()
    month = str(now.month).zfill(2)
    day = str(now.day).zfill(2)
    date = '%d%s%s' % (now.year, month, day)

    #paste all the VCF content together
    out_vcf = ''.join([header % date, cols, rows])

    #return it
    return(out_vcf)

def _tskit_table_to_pandas(table):
    """
    Takes a raw tskit.TableCollection table dict,
    as returned from tskit.TableCollection.<TABLENAME>.asdict(),
    and converts it into a pandas.DataFrame, for write-out
    """
    # get the max column length
    max_col_len = max([len(col) for col in table.values()])
    # fill in columns with np.nan
    filled_table = {}
    for k, col in table.items():
        if len(col) > 0:
            filled_table[k] = np.concatenate((col,
                                    np.nan * np.ones(max_col_len - len(col))))
        else:
            filled_table[k] = np.ones(max_col_len) * np.nan
    # convert to pandas.DataFrame
    pd_table = pd.DataFrame.from_dict(filled_table)
    return pd_table


# -*- coding: utf-8 -*-
"""
Created on Fri Mar 22 02:09:53 2019

@author: CodersMine
"""

import wikipedia

#wikipedia.search("apple")

#wikipedia.summary("apple",sentences=4)

name="Results"
text=""
c=""
import pandas as pd
df=pd.read_excel("list.xlsx").values

for i in range(len(df)):
	if(df[i][1]==1):
		try:
			tt=wikipedia.page(df[i][0])
			c=tt.content
			text+="\n\n"+df[i][0]+"\n\n"
		except:
			c=""
	elif(df[i][1]==2):
		try:
			file.write(text)
			file.close()
			text=""
			c=""
		except:
			print("file already closed")
		name=df[i][0]
		file=open(name+".txt",'w')
	else:
		try:
			c=wikipedia.summary(df[i][0])
			text+="\n\n"+df[i][0]+"\n\n"
		except:
			print("Not found for "+df[i][0])
			c=""
	text+=c

file.write(text)
file.close()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Author      : Bhishan Poudel; Physics PhD Student, Ohio University
# Date        : May 09, 2017
# Last update :
#

# Imports
import numpy as np
import matplotlib.pyplot as plt

def plot_with_twiny():
    fig = plt.figure()
    ax1 = fig.add_subplot(111)
    ax2 = ax1.twiny()

    X = np.linspace(0,1,1000)
    Y = np.cos(X*20)

    ax1.plot(X,Y)

    x2labels = np.arange(0,1,step=.10)

    ax1.set_xlim(ax1.get_xlim())
    ax1.set_xticks(x2labels)
    ax1.set_xticklabels(x2labels)
    ax1.set_xlabel(r"Original x-axis")

    ax2.set_xlim(ax1.get_xlim())
    ax2.set_xticks(x2labels)
    ax2.set_xticklabels(x2labels)
    ax2.set_xlabel(r"Modified x-axis")

    x1labels = ['zero', 'one', 'two','three','four','five', 'six',
               'seven', 'eight',  'nine']
    ax1.set_xticklabels(x1labels,rotation='vertical')
    ax1.xaxis.grid(True)
    ax1.yaxis.grid(True)
    plt.subplots_adjust(bottom=0.2)
    plt.show()

if __name__ == '__main__':
    plot_with_twiny()


# __author__ = 'wangyazhou'
# -*-coding:utf-8-*-

from .myunit import MyTest
from .base import Page
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.by import By
# from retrying import retry
import unittest
import logging
import re
import time
import sys
import os
'''
===========说明============
功能:测试用例执行
入口:MySQL数据库
==========================
'''

class Testcase(Page):
    """构造执行用例的类
    """
    def get_expected(self, expected):
        """
        :param expected:  数据库中某一字段的值 ，str类型
        :return:  期望值的列表
        获取表格中'【 】'里面的的值,返回列表
        """
        value = re.compile('【(.*?)】')
        expected_value = value.findall(str(expected))

        return expected_value

    def log(self, sheetname, case_id, actual_value, results_value):
        """
        :param sheetname:   数据库表名
        :param case_id:     行号
        :param actual_value:    实际值
        :param results_value:     pass or failed
        :return: 布尔值 True
        实际结果和执行情况写回数据库,记录成功失败
        写入 actual 和results字段
        """
        actual = self.set_mysql(sheetname, case_id, 'actual', actual_value)
        results = self.set_mysql(sheetname, case_id, 'results', results_value)
        if actual and results:
            logging.info('测试结果写入完成!<' + case_id + ':' + results_value+'>')
        else:
            logging.info('测试结果写入失败')
        return True

    def write_log(self, sheetname, testcase):
        """
        :param sheetname:
        :param testcase:
        :return:
        写入数据库前，加些本地log输出到log.txt
        """
        logging.error('元素定位失败')
        self.log(sheetname, testcase['case_id'], u'页面元素无法定位', 'fail')
        self.insert_img(testcase['case_name'] + '.jpg')
        self.driver.quit()
        raise Warning('元素定位失败')

    def refresh(self):
        """
        刷新当前页面
        :return:
        """
        logging.info('准备刷新')
        self.driver.refresh()
        time.sleep(5)
        logging.info('刷新完毕')
        return True

    # @retry(stop_max_attempt_number = 2)
    def execute_case(self, testcase, sheetname):
        """
        :param testcase: 数据库中一行，dict
        :param sheetname:   数据库表名
        :return:
        执行测试用例
        流程：
        执行action字段 --> 结果写入日志 --> 判断页面返回结果 --> 写入日志与数据库
        目前支持action： click、sendkey、scroll、swipe、back（回到前界面）、sleep、switch(选择标签）、get
        """
        # -------------------------------------执行测试用例---------------------------------------
        logging.info('==========开始执行测试用例' + testcase['case_id'] + '===========')
        logging.info('执行: '+testcase['case_name'])
        if testcase['action'] in ['click', 'sendkey', 'scroll', 'swipe', 'back', 'sleep', 'switch', 'get']:
            # 获取页面元素
            if testcase['url']:
                # 如果url栏填入的话，通过xml方式获取元素
                element = self.get_xml(testcase['url'], testcase['name']) #判断页面中url是否存在
                if element:
                    # 判断页面元素是否成功获取
                    if testcase['action'] == 'click':
                        element.click()
                    if testcase['action'] == 'sendkey':
                        # element.clear()
                        element.send_keys(testcase['value'].decode('utf8'))
                    if testcase['action'] == 'submit':
                        element.submit()
                    if testcase['action'] == 'swipe':
                        slider_action = ActionChains(self.driver)
                        slider_action.click_and_hold(element)
                        if testcase['value'] == '15':
                            slider_action.move_by_offset(80, 0).perform()
                        if testcase['value'] == '30':
                            slider_action.move_by_offset(240, 0).perform()
                else:
                    # print element
                    self.log(sheetname, testcase['case_id'], u'页面元素无法定位','fail')
                    self.insert_img(testcase['case_name'] + '.jpg')
                    self.driver.quit()
                    # self.execute_state = 0
                    raise Warning('元素定位失败')
            else:
                logging.info('不解析xml文件,直接开始定位元素')
                logging.info('开始寻找元素： '+testcase['name']+' ; 类型是： '+testcase['type'])

                if testcase['action'] == 'get':
                    try:
                        self.driver.get(testcase['value'])
                    except BaseException as e:
                        print(e)
                        self.refresh()
                        try:
                            self.driver.get(testcase['value'])
                        except Exception as e:
                            print(e)
                            self.write_log(sheetname, testcase)

                if testcase['action'] == 'switch':
                    try:
                        self.driver.switch_to.window(self.driver.window_handles[int(testcase['value'])])
                    except BaseException as e:
                        print(e)
                        self.refresh()
                        try:
                            self.driver.switch_to.window(self.driver.window_handles[int(testcase['value'])])
                        except Exception as e:
                            print(e)
                            self.write_log(sheetname, testcase)

                if testcase['action'] == 'click':
                    try:
                        self.get_element_straight(testcase['type'], testcase['name']).click()
                    except BaseException as e:
                        print(e)
                        self.refresh()
                        try:
                            self.get_element_straight(testcase['type'], testcase['name']).click()
                        except Exception as e:
                            print(e)
                            self.write_log(sheetname, testcase)

                if testcase['action'] == 'sendkey':
                    try:
                        self.get_element_straight(testcase['type'], testcase['name']).clear()
                        self.get_element_straight(testcase['type'], testcase['name']).send_keys(testcase['value'])
                    except BaseException as e:
                        print(e)
                        self.refresh()
                        try:
                            self.get_element_straight(testcase['type'], testcase['name']).clear()
                            self.get_element_straight(testcase['type'], testcase['name']).send_keys(testcase['value'])
                        except Exception as e:
                            print(e)
                            self.write_log(sheetname, testcase)

                if testcase['action'] == 'scroll':
                    swipe_value = self.get_expected(testcase['value'])
                    self.my_scroll(swipe_value[0], swipe_value[1])

                if testcase['action'] == 'sleep':
                    time.sleep(int(testcase['value']))

                if testcase['action'] == 'swipe':
                    swipe_value = self.get_expected(testcase['value'])
                    try:
                        self.my_swipe(swipe_value[0], swipe_value[1])
                    except BaseException as e:
                        print(e)
                        logging.error('滑动失败')

                if testcase['action'] == 'back':
                    try:
                        self.driver.press_keycode('4')
                    except BaseException as e:
                        print(e)
                        logging.error('返回失败')

            if testcase['name'] == '完成':  # 忘了这个方法啥意思了
                time.sleep(8)
        else:
            print('测试用例没有执行动作或未识别执行动作')
            # raise error('测试用例没有执行动作或未识别执行动作')
        logging.info('结束' + testcase['action'] + '动作')
        # time.sleep(1.5)

        # --------------------------------------判断页面预期元素------------------------------------------
        if testcase['expected'] != '':
            # expected字段不为空，才会进入判断阶段： 1.验证当前url是否正确 2.in操作符检测  3.看看新API提供有没有其他方法
            time.sleep(1.5)
            expected_list = self.get_expected(testcase['expected'])
            for expected in expected_list:
                # 查找预期页面元素
                # print expected
                if 'http' in expected:
                    current_url = self.driver.current_url
                    if expected in current_url:
                        logging.info('实际url与预期url一致,实际url<' + current_url + '>')
                        # expect = True
                    else:
                        logging.error('实际url与预期url不一致,实际url<' + current_url + '>')
                        self.insert_img(testcase['case_name'] + '.jpg')
                        expect = False

                        self.log(sheetname, testcase['case_id'], u'预期页面url不正确', 'fail')

                        self.driver.quit()
                        logging.info('==========测试用执行完成' + testcase['case_id'] + '===========\n\r')

                        assert expect, '实际url与预期url不一致,实际url<' + current_url + '>'
                else:
                    if expected in self.driver.page_source:
                        logging.info('实际页面元素与预期页面元素一致,实际text<' + expected + '>')
                    else:
                        logging.error('实际页面中不存在此页面元素:<' + expected + '>')
                        self.insert_img(testcase['case_name'] + '.jpg')
                        expect = False
                        self.log(sheetname, testcase['case_id'], u'预期页面元素显示不正确', 'fail')

                        self.driver.quit()
                        logging.info('==========测试用执行完成' + testcase['case_id'] + '===========\n\r')

                        assert expect, '实际页面中不存在此页面元素:<' + expected + '>'

        self.log(sheetname, testcase['case_id'], u'预期页面显示正常', 'pass')
        # self.insert_img(testcase['case_name'] + '.jpg')
        logging.info('==========测试用执行完成' + testcase['case_id'] + '===========\n\r')

if __name__ == '__main__':
    p = Testcase()
    t = p.log('a_login', 'test1_Login_06', 'no', 'no')


#! /usr/bin/env python
# -*- coding: utf-8 -*-

"""Load dataset for the CTC model (ERATO corpus).
   In addition, frame stacking and skipping are used.
   You can use only the single GPU version.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from os.path import join, isfile
import pickle
import numpy as np

from utils.dataset.ctc import DatasetBase

class Dataset(DatasetBase):

    def __init__(self, data_type, label_type, ss_type, batch_size,
                 max_epoch=None, splice=1,
                 num_stack=1, num_skip=1,
                 shuffle=False, sort_utt=False, sort_stop_epoch=None,
                 progressbar=False):
        """A class for loading dataset.
        Args:
            data_type (string): train or dev or test
            label_type (string): kana
            ss_type (string): remove or insert_left or insert_both or insert_right
            batch_size (int): the size of mini-batch
            max_epoch (int, optional): the max epoch. None means infinite loop.
            splice (int, optional): frames to splice. Default is 1 frame.
            num_stack (int, optional): the number of frames to stack
            num_skip (int, optional): the number of frames to skip
            shuffle (bool, optional): if True, shuffle utterances. This is
                disabled when sort_utt is True.
            sort_utt (bool, optional): if True, sort all utterances by the
                number of frames and utteraces in each mini-batch are shuffled.
                Otherwise, shuffle utteraces.
            sort_stop_epoch (int, optional): After sort_stop_epoch, training
                will revert back to a random order
            progressbar (bool, optional): if True, visualize progressbar
        """
        super(Dataset, self).__init__()

        self.is_test = True if data_type == 'test' else False

        self.data_type = data_type
        self.label_type = label_type
        self.ss_type = ss_type
        self.batch_size = batch_size
        self.max_epoch = max_epoch
        self.splice = splice
        self.num_stack = num_stack
        self.num_skip = num_skip
        self.shuffle = shuffle
        self.sort_utt = sort_utt
        self.sort_stop_epoch = sort_stop_epoch
        self.progressbar = progressbar
        self.num_gpu = 1

        # paths where datasets exist
        dataset_root = ['/data/inaguma/erato',
                        '/n/sd8/inaguma/corpus/erato/dataset']

        input_path = join(dataset_root[0], 'inputs', data_type)
        # NOTE: ex.) save_path:
        # erato_dataset_path/inputs/data_type/speaker/***.npy
        label_path = join(dataset_root[0], 'labels',
                          ss_type, data_type, label_type)
        # NOTE: ex.) save_path:
        # erato_dataset_path/labels/ss_type/data_type/kana/speaker/***.npy

        # Load the frame number dictionary
        if isfile(join(input_path, 'frame_num.pickle')):
            with open(join(input_path, 'frame_num.pickle'), 'rb') as f:
                self.frame_num_dict = pickle.load(f)
        else:
            dataset_root.pop(0)
            input_path = join(dataset_root[0], 'inputs', data_type)
            label_path = join(dataset_root[0], 'labels',
                              ss_type, data_type, label_type)
            with open(join(input_path, 'frame_num.pickle'), 'rb') as f:
                self.frame_num_dict = pickle.load(f)

        # Sort paths to input & label
        axis = 1 if sort_utt else 0
        frame_num_tuple_sorted = sorted(self.frame_num_dict.items(),
                                        key=lambda x: x[axis])
        input_paths, label_paths = [], []
        for input_name, frame_num in frame_num_tuple_sorted:
            speaker = '_'.join(input_name.split('_')[:-1])
            input_paths.append(join(input_path, speaker, input_name + '.npy'))
            label_paths.append(join(label_path, speaker, input_name + '.npy'))
        self.input_paths = np.array(input_paths)
        self.label_paths = np.array(label_paths)
        # NOTE: Not load dataset yet

        self.rest = set(range(0, len(self.input_paths), 1))


# -*- coding: utf-8 -*-

import os
import re
import numpy as np
import pandas as pd
import jieba
import jieba.analyse
import codecs
import tensorflow as tf
import collections

batch_size = 64
learn_rate = 0.01
model_dir = r"F:\model\唐诗"
data_dir = r"F:\data\唐诗"
file_path = os.path.join(data_dir, "唐诗.txt")
epoch = 50

start_token = "G"
end_token = "E"

def process_pomes(file_path, source_label_splitter=":"):
    """
    功能：数据预处理
    参数
        file_path：文件路径
        source_label_splitter：标题与正文之间的分割符号
    返回值：pomes_vector, word2vec_map, words
        pomes_vector：pomes_vector 是list，每一个元素是一段正文的所有词组成的大型词向量
        word2vec_map：是一个字典，每一个词的词向量，key是词，value是词向量
        words：全部词汇
    """
    poems = []
    source_file = codecs.open(file_path, "r", encoding="utf-8")
    # target_file = codecs.open(target_file_path, "w", encoding="utf-8")
    line = source_file.readline()  # 读取一行
    while line:
        # 去掉空格、换行符
        line = line.replace("\r", "").replace("\n", "").strip()
        # 如果 line 为空，就跳过这次循环，进入下一次循环
        if line == "" or line is None:
            line = source_file.readline()  # 读取下一行
            continue
        # 拆分标题、和正文
        line_list = line.split(source_label_splitter)
        if len(line_list) != 2:
            line = source_file.readline()  # 读取下一行
            continue
        title = line_list[0]
        content = line_list[-1]
        # 如果正文的词数量太少或者太多，就跳出这次循环
        if len(content) < 5 or len(content) > 80:
            line = source_file.readline()  # 读取下一行
            continue
        # 构造句子的内容，以 start_token 为开始，以 content 为正文，以 end_token 为结尾
        content = start_token + content + end_token
        poems.append(content)
        line = source_file.readline()  # 读取一行
    poems = sorted(poems, key=lambda le : len(line))  # 排序
    # 计算所有词
    all_words = []
    for poem in poems:
        all_words += [word for word in  poem]
    # 统计词频
    counter = collections.Counter(all_words)
    # 过滤掉低频词、生僻词
    counter_paris = sorted(counter.items(), key=lambda x:x[-1])
    words, _ = zip(*counter_paris)
    words = words[: len(words)]

    # 获取每一个词的词向量，word2vec_map 是一个字典，key是词，value是词向量
    word2vec_map = dict(zip(words, range(len(words))))
    # pomes_vector 是list，每一个元素是一段正文的所有词组成的大型词向量
    to_num = lambda word: word2vec_map.get(word, len(words))
    pomes_vector = [list(map(to_num, poem)) for poem in poems]
    return pomes_vector, word2vec_map, words

def get_batch_data(batch_size, pomes_vector, word2vec_map):
    n_chunk = len(pomes_vector) / batch_size
    n_chunk = int(n_chunk)
    x_batchs = []
    y_batchs = []
    for i in range(n_chunk):
        start_index = i * batch_size
        end_index = start_index + batch_size
        batchs = pomes_vector[start_index : end_index]
        # 当前batch数据中，所有句子中最大长度是多少
        lenght = max(map(len, batchs))
        x_data = np.full((batch_size, lenght), 0, np.float32)  # 填充维度，使用0填充
        for row in range(batch_size):
            x_data[row, :len(batchs[row])] = batchs[row]
        y_data = np.copy(x_data)
        y_data[:, :-1] = x_data[:, 1:]
        x_batchs.append(x_data)
        y_batchs.append(y_data)
    return x_batchs, y_batchs

def run_training(file_path):
    pomes_vector, word2vec_map, words = process_pomes(file_path, source_label_splitter=":")
    x_batchs, y_batchs = get_batch_data(batch_size, pomes_vector, word2vec_map)
    intput_data = tf.placeholder(tf.int32, [batch_size, None])
    output_data = tf.placeholder(tf.int32, [batch_size, None])

def main(is_training, file_path):
    if is_training:
        print("开始训练")
        run_training(file_path)
    else:
        begin_word = input("输入开始词：")


# -*- coding: utf-8 -*-
from pyramid.security import Allow
from schematics.exceptions import ValidationError
from schematics.transforms import blacklist, whitelist
from schematics.types import StringType, BaseType, EmailType, BooleanType
from schematics.types.compound import ModelType
from schematics.types.serializable import serializable

from openprocurement.api.auth import ACCR_5
from openprocurement.api.constants import DK_CODES
from openprocurement.api.models import (
    Document,
    ListType,
    Classification as BaseClassification,
    PeriodEndRequired as BasePeriodEndRequired,
    Identifier as BaseIdentifier,
    Address as BaseAddress,
    ContactPoint as BaseContactPoint,
    schematics_embedded_role,
    schematics_default_role,
)
from openprocurement.api.models import Model
from openprocurement.framework.core.models import (
    Framework,
    Submission as BaseSubmission,
    Qualification as BaseQualification,
)
from openprocurement.framework.electroniccatalogue.utils import (
    AUTHORIZED_CPB,
    get_framework_unsuccessful_status_check_date,
)

class DKClassification(BaseClassification):
    scheme = StringType(required=True, choices=[u"ДК021"])
    id = StringType(required=True)

    def validate_id(self, data, id):
        if id not in DK_CODES:
            raise ValidationError(BaseType.MESSAGES["choices"].format(unicode(DK_CODES)))

class Identifier(BaseIdentifier):
    legalName = StringType(required=True)

class Address(BaseAddress):
    streetAddress = StringType(required=True)
    locality = StringType(required=True)
    region = StringType(required=True)
    postalCode = StringType(required=True)

class ContactPoint(BaseContactPoint):
    email = EmailType(required=True)
    telephone = StringType(required=True)

class CentralProcuringEntity(Model):
    class Options:
        roles = {
            "embedded": schematics_embedded_role,
            "view": schematics_default_role,
            "edit_draft": schematics_default_role,
            "edit_active": whitelist("contactPoint"),
        }
    name = StringType(required=True)
    name_en = StringType()
    name_ru = StringType()
    identifier = ModelType(Identifier, required=True)
    additionalIdentifiers = ListType(ModelType(Identifier))
    address = ModelType(Address, required=True)
    contactPoint = ModelType(ContactPoint, required=True)
    kind = StringType(choices=["central"], default="central")

    def validate_identifier(self, data, identifier):
        id_ = identifier.id
        cpb_with_statuses = {cpb["identifier"]["id"]: cpb["active"] for cpb in AUTHORIZED_CPB}
        if id_ not in cpb_with_statuses or not cpb_with_statuses[id_]:
            raise ValidationError("Can't create framework for inactive cpb")

class ElectronicCatalogueFramework(Framework):
    class Options:
        namespace = "Framework"
        _status_view_role = blacklist(
            "doc_type",
            "successful",
            "transfer_token",
            "owner_token",
            "revisions",
            "_id",
            "_rev",
            "__parent__",
        )
        _edit_role = _status_view_role + blacklist(
            "frameworkType",
            "prettyID",
            "period",
            "enquiryPeriod",
            "dateModified",
            "date",
            "doc_id"
        )
        _create_role = _edit_role + blacklist("status")

        roles = {
            "create": _create_role,
            "edit_draft": _edit_role + blacklist("owner", "old_date"),
            "edit_active": whitelist(
                "status",
                "procuringEntity",
                "qualificationPeriod",
                "description",
                "description_en",
                "description_ru",
                "documents",
                "frameworkDetails"
            ),
            "draft": _status_view_role,
            "active": _status_view_role,
            "complete": _status_view_role,
            "unsuccessful": _status_view_role,
            "view": _edit_role + whitelist(
                "date",
                "period",
                "enquiryPeriod",
                "prettyID",
                "documents",
                "doc_id",
                "dateModified",
                "status",
                "owner",
                "next_check",
            ),
            "chronograph": whitelist("next_check"),
            "chronograph_view": _status_view_role,
            "Administrator": whitelist("status", "mode"),
            "default": blacklist("doc_id", "__parent__"),  # obj.store() use default role
            "plain": blacklist(  # is used for getting patches
                "_attachments", "revisions", "dateModified", "_id", "_rev", "doc_type",
                "__parent__"
            ),
            "listing": whitelist("dateModified", "doc_id"),
            "embedded": blacklist("_id", "_rev", "doc_type", "__parent__"),
        }

    status = StringType(
        choices=[
            "draft",
            "active",
            "deleted",
            "complete",
            "unsuccessful",
        ],
        default="draft",
    )
    period = ModelType(BasePeriodEndRequired)
    qualificationPeriod = ModelType(BasePeriodEndRequired, required=True)
    enquiryPeriod = ModelType(BasePeriodEndRequired)
    frameworkType = StringType(default="electronicCatalogue")
    procuringEntity = ModelType(CentralProcuringEntity, required=True)
    classification = ModelType(DKClassification, required=True)
    additionalClassifications = ListType(ModelType(BaseClassification))
    documents = ListType(ModelType(Document, required=True), default=list())

    successful = BooleanType(required=True, default=False)

    procuring_entity_kinds = ["central"]
    central_accreditations = (ACCR_5,)
    edit_accreditations = (ACCR_5,)

    @serializable(serialize_when_none=False)
    def next_check(self):
        checks = []
        if self.status == "active":
            if not self.successful:
                unsuccessful_status_check = get_framework_unsuccessful_status_check_date(self)
                if unsuccessful_status_check:
                    checks.append(unsuccessful_status_check)
            checks.append(self.qualificationPeriod.endDate)
        return min(checks).isoformat() if checks else None

    def __acl__(self):
        acl = super(ElectronicCatalogueFramework, self).__acl__()
        acl.append((Allow, "{}_{}".format(self.owner, self.owner_token), "upload_framework_documents"))
        return acl

class Submission(BaseSubmission):

    status = StringType(
        choices=[
            "draft",
            "active",
            "deleted",
            "complete"
        ],
        default="draft",
    )
    submissionType = StringType(default="electronicCatalogue")

class Qualification(BaseQualification):

    status = StringType(
        choices=[
            "pending",
            "active",
            "unsuccessful"
        ],
        default="pending",
    )

    qualificationType = StringType(default="electronicCatalogue", required=True)


import pandas as pd
import requests
import json
import os

class BitcoinTrade:

    def sort_and_format(self, l, reverse=False):
        l.sort(key=lambda x: float(x["price"]), reverse=reverse)
        r = []
        for i in l:
            r.append({'price': float(i['price']),
                    'volume': float(i['volume'])})
        return r

    def format_depth(self, depth):
        bids = self.sort_and_format(depth['bids'], True)
        asks = self.sort_and_format(depth['asks'], False)
        return {'asks': asks, 'bids': bids}

    def order_book(self):
        url = 'https://app.modiax.com/api/v2/order_book?market=btcbrl'
        data = requests.get(url=url)

        content = json.loads(data.content)
        self.format_depth(content)

        df_asks = pd.DataFrame.from_dict(content['asks'])
        df_bids = pd.DataFrame.from_dict(content['bids'])

        #print(self.format_depth(content))
        print(df_asks)
        print(df_bids)

        json_data = self.format_depth(content)

        with open('data.json', 'w') as outfile:
            json.dump(json_data, outfile, sort_keys = True, indent = 4, ensure_ascii = False)

trade = BitcoinTrade()
trade.order_book()


# -*- coding: UTF-8 -*-
# @Time    : 22/04/2019 17:43
# @Author  : QYD
import torch
import numpy as np
import functools
from models.fpn_inception import FPNInception
import torch.nn as nn
import cv2 as cv

model_use = ["FPNInception", "FPNMobileNet"]

def post_process(x: torch.Tensor) -> np.ndarray:
    x, = x
    x = x.detach().cpu().float().numpy()
    x = (np.transpose(x, (1, 2, 0)) + 1) / 2.0 * 255.0
    return x.astype('uint8')

def blurred_field_cal(img, theta):
    h, w, c = img.shape
    center_x = h // 2
    center_y = w // 2
    yy, xx = np.meshgrid(np.arange(w), np.arange(h))  # 得到列坐标和行坐标
    blurred_field = np.sqrt(
        (xx - center_x) ** 2 + (yy - center_y) ** 2) * theta  # 根据行坐标和列坐标计算模糊域.可以把计算得到的模糊域给存下来不需要重复的计算。

    blurred_field = np.reshape(blurred_field, (h, w, 1))
    return blurred_field.astype(np.int8)

def normal_img(img, mean=0.5, std=0.5):
    img = ((img / 255) - mean) / std
    return img

def q_deblurGAN(img, theta, use_gpu=False, weights="./weights/fpn_inception.h5"):
    blurred_field = blurred_field_cal(img, theta)
    img_input = np.concatenate([img, blurred_field], axis=2)
    img_input = normal_img(img_input, mean=0.5, std=0.5)
    img_input = np.transpose(img_input, (2, 0, 1))
    img_input = torch.tensor(img_input, dtype=torch.float32)
    img_input = img_input.unsqueeze(dim=0)
    norm_layer = functools.partial(nn.InstanceNorm2d, affine=False, track_running_stats=True)
    model = FPNInception(norm_layer=norm_layer)
    model.train()
    if use_gpu:
        img_input = img_input.cuda()
        model.cuda()
    model.load_state_dict({k.replace(
        'module.',
        ''): v
                           for k, v
                           in torch.load(weights)["model"].items()})
    out = model(img_input)
    out = post_process(out)
    return out

if __name__ == '__main__':
    img = cv.imread("../samples/origin_blur.jpg")
    img_deblur = q_deblurGAN(img=img, theta=0.05, use_gpu=True)
    cv.imwrite("../samples/origin_blurGAN.jpg", img_deblur)


#################################################################################
# WaterTAP Copyright (c) 2020-2023, The Regents of the University of California,
# through Lawrence Berkeley National Laboratory, Oak Ridge National Laboratory,
# National Renewable Energy Laboratory, and National Energy Technology
# Laboratory (subject to receipt of any required approvals from the U.S. Dept.
# of Energy). All rights reserved.
#
# Please see the files COPYRIGHT.md and LICENSE.md for full copyright and license
# information, respectively. These files are also available online at the URL
# "https://github.com/watertap-org/watertap/"
#################################################################################
"""
Test of db_api module
"""
import pytest
from ..db_api import ElectrolyteDB
from ..data_model import Component, Reaction, Base
from ..commands import _load_bootstrap
from pymongo import MongoClient
from .util import MockDB

@pytest.fixture
def mockdb():
    return MockDB()

# Test for MongoDB server at default URL
g_mongo_server = True
try:
    conn = MongoClient(
        ElectrolyteDB.DEFAULT_URL,
        serverSelectionTimeoutMS=1000,
    )
    conn.server_info()
except Exception as err:
    print(f"Cannot connect to MongoDB: {err}")
    g_mongo_server = False

requires_mongo = pytest.mark.skipif(
    g_mongo_server is False,
    reason=f"Cannot connect to MongoDB server at {ElectrolyteDB.DEFAULT_URL}",
)

@pytest.fixture(scope="module")
def edb():
    return ElectrolyteDB()

@pytest.mark.component
@requires_mongo
def test_edb_init(edb):
    assert edb is not None
    assert edb.connect_status_str == "Connection succeeded"
    assert type(edb.connect_status) is dict

@pytest.mark.component
@requires_mongo
def test_edb_load(edb):
    # Load bootstrap for temporary testing purposes
    _load_bootstrap(edb)
    base = edb.get_base("default_thermo")
    assert type(base) is Base
    edb.list_bases()

@pytest.mark.component
@requires_mongo
def test_edb_get_components(edb):
    res_obj_comps = edb.get_components(element_names=["H", "O"])
    for comp_obj in res_obj_comps:
        assert type(comp_obj) is Component

    # Just test the _process_species function
    assert edb._process_species("H2O") == "H2O"
    assert edb._process_species("H_+") == "H"
    assert edb._process_species("OH_-") == "OH"

    # Drop the bootstrap database for cleaning
    edb.drop_database(edb.DEFAULT_URL, edb.DEFAULT_DB)
    assert edb.is_empty()

@pytest.mark.unit
def test_edb_load_convention():
    # make sure the convention of collection names mapping to data_model objects is true, since
    # we rely on it in the ElectrolyteDB.load() method
    for wrapper_class in (Component, Reaction, Base):
        assert wrapper_class.__name__.lower() in ElectrolyteDB._known_collections

def insert_reactions(collection, data):
    for obj in data:
        collection.insert_one(obj)

# Data for get_reactions tests

data1 = [
    {
        "stoichiometry": {"Liq": {"H2O": -1, "CO2": -1, "H2CO3": 1}},
        "heat_of_reaction": "constant_dh_rxn",
        "equilibrium_constant": "van_t_hoff",
        "equilibrium_form": "log_power_law",
        "concentration_form": "ConcentrationForm.molarity",
        "parameter_data": {
            "dh_rxn_ref": [{"v": 0, "u": "kJ/mol", "i": 0}],
            "k_eq_ref": [{"v": 0.0017, "u": "m**3/mol", "i": 0}],
            "T_eq_ref": [{"v": 300, "u": "K", "i": 0}],
        },
        "type": "equilibrium",
        "name": "CO2_to_H2CO3",
        "components": ["CO2", "H2CO3"],
        "reactant_elements": ["C", "O", "H"],
    },
    {
        "stoichiometry": {"Liq": {"H2O": -1, "H_+": 1, "OH_-": 1}},
        "heat_of_reaction": "constant_dh_rxn",
        "equilibrium_constant": "van_t_hoff_aqueous",
        "equilibrium_form": "log_power_law",
        "concentration_form": "ConcentrationForm.molarity",
        "parameter_data": {
            "dh_rxn_ref": [{"v": 55.83, "u": "kJ/mol", "i": 0}],
            "ds_rxn_ref": [{"v": -80.7, "u": "J/mol/K", "i": 0}],
        },
        "type": "equilibrium",
        "name": "H2O_Kw",
        "components": ["H2O", "Kw"],
        "reactant_elements": ["O", "H"],
    },
]

# add one more record to data1
data2 = data1.copy() + [
    {
        "stoichiometry": {"Liq": {"H2CO3": -1, "H_+": 1, "HCO3_-": 1}},
        "heat_of_reaction": "constant_dh_rxn",
        "equilibrium_constant": "van_t_hoff_aqueous",
        "equilibrium_form": "log_power_law",
        "concentration_form": "ConcentrationForm.molarity",
        "parameter_data": {
            "dh_rxn_ref": [{"v": 7.7, "u": "kJ/mol", "i": 0}],
            "ds_rxn_ref": [{"v": -95.8, "u": "J/mol/K", "i": 0}],
        },
        "type": "equilibrium",
        "name": "H2CO3_Ka1",
        "components": ["H2CO3", "Ka1"],
        "reactant_elements": ["C", "O", "H"],
    }
]

# The way to read this parameterized test is:
#   With these 'components' and this 'data' in the DB,
#   getting reactions with *any* component should return 'any_num' records,
#   and getting reactions with *all* components should return 'all_num' records
#   and getting reactions with *all* components and *new* components should
#   return 'new_num' records
@pytest.mark.unit
@pytest.mark.parametrize(
    "components,data,any_num,all_num,new_num",
    [
        (["H2O", "CO2", "H2CO3"], data1, 2, 1, 2),
        (["H2O", "H +", "OH -", "H2CO3", "HCO3 -"], data2, 3, 2, 3),
        (["H2CO3"], data2, 2, 0, 2),
    ],
)
def test_get_reactions(mockdb, components, data, any_num, all_num, new_num):
    insert_reactions(mockdb._db.reaction, data)
    reactions = mockdb.get_reactions(components, any_components=True)
    assert len(list(reactions)) == any_num
    reactions = mockdb.get_reactions(components, any_components=False)
    assert len(list(reactions)) == all_num
    reactions = mockdb.get_reactions(
        components, any_components=False, include_new_components=True
    )
    assert len(list(reactions)) == new_num


import pandas as pd
import numpy as np
from matplotlib import pyplot as plt
from pylab import savefig

#   PARCEL IDENTIFICATION NUMBER CAN BE USED WITH CITY WEBSITE, SO USE THAT AS INDEX, DROP ORDER

df = pd.read_excel('AmesHousing.xls',sheet_name='Sheet1',index_col=1)
df = df.drop(['Order'],axis=1)

contvarlist = ['Lot Frontage','Lot Area','Mas Vnr Area','BsmtFin SF 1','BsmtFin SF 2','Bsmt Unf SF',
               'Total Bsmt SF', '1st Flr SF','2nd Flr SF','Low Qual Fin SF','Gr Liv Area','Garage Area',
               'Wood Deck SF', 'Open Porch SF', 'Enclosed Porch','3Ssn Porch','Screen Porch','Pool Area',
               'Misc Val']

nominalvarlist = ['MS SubClass','MS Zoning','Street','Alley','Land Contour','Lot Config','Neighborhood','Condition 1',
                  'Condition 2','Bldg Type','House Style','Roof Style','Roof Matl','Exterior 1st',
                  'Exterior 2nd','Mas Vnr Type','Foundation','Heating','Central Air','Garage Type',
                  'Misc Feature','Sale Type','Sale Condition']

ordinalvarlist = ['Lot Shape','Utilities','Land Slope','Exter Qual','Exter Cond',
                  'Bsmt Qual','Bsmt Cond','Bsmt Exposure','BsmtFin Type 1', 'BsmtFin Type 2','Heating QC',
                  'Electrical','Kitchen Qual','Functional','Fireplace Qu','Garage Finish','Garage Qual',
                  'Garage Cond','Paved Drive','Pool QC','Fence']

discretevarlist = ['Year Built','Year Remod/Add','Bsmt Full Bath','Bsmt Half Bath','Full Bath','Half Bath','Bedroom AbvGr',
                   'Kitchen AbvGr','TotRms AbvGrd','Fireplaces','Garage Yr Blt','Garage Cars','Mo Sold',
                   'Yr Sold','Overall Qual','Overall Cond']

yearvarlist = ['Year Built','Year Remod/Add','Garage Yr Blt','Mo Sold','Yr Sold']

#   TASK 1.1
#   VISUALIZE DISTRIBUTION OF EACH CONTINUOUS VARIABLE AND DISTRIBUTION OF TARGET
#   NOTE THAT SOME NUMERICAL VARIABLES ARE ACTUALLY CATEGORICAL

task11list = contvarlist + ['SalePrice']
contdf = df[task11list]

#   BESIDES TARGET, THERE ARE 36 NUMERICAL VARIABLES. CAN VISUALIZE IN 6x6 SUBPLOT

fig, ax = plt.subplots(5,4,figsize=(25,15))

for i in range(4):
    for j in range(5):
        colnum = 5*i + j
        ax[j,i].hist(contdf.iloc[:,colnum])
        ax[j,i].set_xlabel(contdf.columns[colnum])

plt.figtext(0.4,0.92,'Task 1.1: Visualize Distribution of Continuous Variables',
            fontsize=16)

savefig('hw2_task1_1_continuous_variables.png',bbox_inches='tight')

#   DATA NOTES:  SOME NUMERIC VARIABLES ARE CATEGORICAL (E.G. MS SUBCLASS, # BATHROOMS, FIREPLACES)
#   OTHERS BUNCH AT 0: E.G. 2ND FLOOR SQFT OR PORCH SIZE (FOR THOSE WITHOUT 2ND FLOOR OR PORCH).
#   VARIABLES SUCH AS YEAR/MONTH SOLD CAN BE GROUPED, ALSO CONSIDER SEASONAL EFFECTS (E.G. HIGHER SALES PRICES IN SUMMER)
#   NEED TO CONSIDER THAT SOME VARIABLES HIGHLY RELATED: E.G. IF NO GARAGE, THEN ALL THE NOMINAL AND ORDINAL
#   GARAGE VARIABLES HAVE VALUE 'NA' WHILE THE CONTINOUS GARAGE VARIABLES HAVE VALUE '0'

#   DISCRETE VARIABLES COULD BE TREATED AS CATEGORIES OR MODELED AS IF CONTINUOUS

#   TASK 1.2
#   VISUALIZE 2-D SCATTER DEPENDENCIES OF TARGET ON CONTINUOUS VARS

y = df['SalePrice']

fig, ax = plt.subplots(5,4,figsize=(25,15),sharey='row')

for i in range(4):
    for j in range(5):
        colnum = 5*i + j
        if colnum < 19:
            ax[j,i].scatter(contdf.iloc[:,colnum],y)
            ax[j,i].set_xlabel(contdf.columns[colnum])
        if i==0:
            ax[j,i].set_ylabel('Sale Price')

plt.figtext(0.4,0.92,'Task 1.2: Visualize Dependency of Sale Price on Continous Vars',
            fontsize=16)

savefig('hw2_task1_2__scatter_continuous_variables.png',bbox_inches='tight')

#   BEFORE SPLITTING, DEAL WITH NAN VALUES SEPARATELY FOR DIFFERENT DATA TYPES

#   treat NA in nominal and ordinal as MM, categorical as -9, and continous as 0
#   note: for items like garage yr built, would probably want to set value to zero AND
#   create a dummy for no garage.  For year variables, fill with median value

df1 = df.copy()
df1.loc[:,ordinalvarlist + nominalvarlist] = df1.loc[:, ordinalvarlist + nominalvarlist].fillna('MM')
df1.loc[:,yearvarlist] = df1.loc[:,yearvarlist].fillna(df1.loc[:,yearvarlist].mean())
df1.loc[:,contvarlist + discretevarlist] = df1.loc[:,contvarlist + discretevarlist].fillna(0)

#   TASK 1.3: SPLIT INTO TRAIN-TEST SPLIT. DO NOT USE TEST FOR ANYTHING UNTIL FINAL EVALUATION IN 1.6
#   FOR EACH CATEGORICAL VAR, CROSS VALIDATE A LINEAR REGRESSION MODEL USING JUST THIS VARIABLE (ONE HOT ENCODED).
#   VISUALIZE RELATIONSHIP OF CATEGORICAL VARIABLES THAT PROVIDE BEST R2 WITH TARGET

from sklearn.model_selection import train_test_split
X = df1.drop(['SalePrice'],axis=1)
y = df1['SalePrice']

X_train, X_test, y_train, y_test = train_test_split(X,y,random_state=0)

#   LOOP THROUGH EACH CATEGORICAL VARIABLE. ONE-HOT ENCODE, SO THAT ANY NUMERICAL VARIABLES
#   HAVE TO FIRST BE CONVERTED TO CHARACTERS.  RUN REGRESSIONS, SAVE R^2. VISUALIZE HIGHEST R2

from sklearn.preprocessing import OneHotEncoder
from sklearn.linear_model import LinearRegression

def get_top_feature(trainX,trainy,featurelist):
    r2list = []
    for v in featurelist:
        tmp=trainX[v]
        tmp2=pd.get_dummies(tmp)
        lr = LinearRegression().fit(tmp2,trainy)
        r2list.append(lr.score(tmp2,trainy))

    highr2feature = featurelist[np.argmax(r2list)]
    return highr2feature

def plot_top_feature(trainX,trainy,featurelist):
    fig,ax = plt.subplots(1,1,figsize=(6,6))
    highr2feature = get_top_feature(trainX,trainy,featurelist)
    ax.scatter(trainX[highr2feature],trainy)
    ax.set_xlabel(highr2feature)
    ax.set_ylabel('Sale Price')

plot_top_feature(X_train[ordinalvarlist],y_train,ordinalvarlist)
plot_top_feature(X_train[nominalvarlist],y_train,nominalvarlist)
plot_top_feature(X_train[discretevarlist],y_train,discretevarlist)

fig,ax = plt.subplots(1,1,figsize=(15,10))
highr2feature = get_top_feature(X_train,y_train,ordinalvarlist)

ax.scatter(X_train[highr2feature],y_train)
ax.set_xlabel(highr2feature)
ax.set_ylabel('Sale Price')

plt.figtext(0.3,0.92,'Task 1.3: Visualize Sale Price on Best Ordinal Variable Values',
            fontsize=16)

savefig('hw2_task1_3_scatter_best_ordinal_variables.png',bbox_inches='tight')

#1.4 Use ColumnTransformer and pipeline to encode categorical variables. Evaluate Linear
#Regression (OLS), Ridge, Lasso and ElasticNet using cross-validation with the default
#parameters. Does scaling the data (within the pipeline) with StandardScaler help?

from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import MinMaxScaler
from sklearn.pipeline import make_pipeline
from sklearn.model_selection import cross_val_score

from sklearn.linear_model import Ridge
from sklearn.linear_model import Lasso
from sklearn.linear_model import ElasticNet

#   SET UP PRE-PROCESSING WITH AND WITHOUT SCALING

catlist = ordinalvarlist + nominalvarlist
preprocess_noscaling = ColumnTransformer(
        [('onehotencoder',OneHotEncoder(handle_unknown='ignore'),catlist)])

preprocess_scaling = ColumnTransformer(
        [('standardscaler',StandardScaler(),contvarlist),
         ('minmaxscaler',MinMaxScaler(),discretevarlist),
         ('onehotencoder',OneHotEncoder(handle_unknown='ignore'),catlist)])

#   FUNCTION THAT INPUTS REGRESSION MODEL AND PREPROCESSING, OUTPUTS MEAN CV SCORE

def get_mean_cv(toscale,estimatorname):
    preprocess = eval("preprocess_"+toscale)
    estimator = eval(estimatorname + "()")
    model = make_pipeline(preprocess,estimator)
    scores = cross_val_score(model,X_train,y_train,cv=10)
    df = pd.DataFrame([[estimatorname,toscale,np.mean(scores)]],columns=['Estimator','Scaling','CV Score'])
    return df

#estimatorlist = [LinearRegression(),Ridge(),Lasso(),ElasticNet()]
#preprocesslist = [preprocess_noscaling,preprocess_scaling]
estimatorlist = ["LinearRegression","Ridge","Lasso","ElasticNet"]
preprocesslist = ["noscaling","scaling"]
results = pd.DataFrame()

for e in estimatorlist:
    for p in preprocesslist:
        results=results.append(get_mean_cv(p,e))

print(results)
results.to_csv('hw2_task1_4_cv_results_estimator_scaling.csv',index=False)

# 1.5: TUNE RESULTS WITH GRIDSEARCHCV
from sklearn.model_selection import GridSearchCV

def get_mean_cv_grid(toscale,estimatorname,param_to_tune,tunelist):
    preprocess = eval("preprocess_"+toscale)
    estimator = eval(estimatorname + "()")
    pipe = make_pipeline(preprocess,estimator)
    param = str(estimatorname).lower()+"__"+param_to_tune
    param_grid = {param : tunelist}
    grid = GridSearchCV(pipe,param_grid,cv=10)
    grid.fit(X_train,y_train)
    df = pd.DataFrame([[estimatorname,toscale,grid.best_score_,grid.best_params_,list(grid.cv_results_['mean_test_score']),tunelist]],
                      columns=['Estimator','Scaling','CV Score','Best Alpha','Param CV','Param List'])
    return df

preprocesslist = ["scaling"]
estimatorlist = ["Ridge","Lasso","ElasticNet"]
alphalist = [.01,.1,1.0,10,100]

for e in estimatorlist:
    for p in preprocesslist:
        results=results.append(get_mean_cv_grid(p,e,"alpha",alphalist))

results.to_csv('hw2_task1_5_cv_results_with_tuning.csv',index=False)

#   Visualize dependence of validation scores on paramaters for ridge, lasso, elasticnet

gridresults = results[results['Param CV'].notnull()]

l1 = gridresults[gridresults['Estimator']=="Ridge"].iloc[:,3][0]
l2 = gridresults[gridresults['Estimator']=="Lasso"].iloc[:,3][0]
l3 = gridresults[gridresults['Estimator']=="ElasticNet"].iloc[:,3][0]

fig, ax = plt.subplots(1,1,figsize=(10,5))
lns1 = ax.plot(range(5),l1,linewidth=1.0,color='r',marker='D',ms=4,label='Ridge Tuning')
lns2 = ax.plot(range(5),l2,linewidth=1.0,color='b',marker='D',ms=4,label='Lasso Tuning')
lns3 = ax.plot(range(5),l3,linewidth=1.0,color='g',marker='D',ms=4,label='ElasticNet Tuning')

lns = lns1 + lns2 + lns3
labs = [l.get_label() for l in lns]
ax.legend(lns, labs, loc='upper center', bbox_to_anchor=(0.5,-0.1),
          fancybox=True,shadow=True,ncol=2)

ax.set_xticks(range(5))
ax.set_xticklabels(['.01','.1','1','10','100'])
ax.set_title('CV Score by Alpha Value')

savefig('hw2_task1_5_cv_score_by_alpha_tuning.png',bbox_inches='tight')

#   1.6 VISUALIZE COEFFICIENTS OF THE RESULTING MODELS. DO THEY AGREE ON WHICH FEATURES ARE IMPORTANT?

#   retrain using best parameters and save coefficients

coefs = pd.DataFrame()

pipe = make_pipeline(preprocess_scaling,Ridge(alpha=1.0))
pipe.fit(X_train,y_train)
coefs['Ridge']=pipe.named_steps.ridge.coef_

pipe = make_pipeline(preprocess_scaling,Lasso(alpha=100))
pipe.fit(X_train,y_train)
coefs['Lasso']=pipe.named_steps.lasso.coef_

pipe = make_pipeline(preprocess_scaling,ElasticNet(alpha=.01))
pipe.fit(X_train,y_train)
coefs['Elastic Net']=pipe.named_steps.elasticnet.coef_

coeff_corr = pd.DataFrame(coefs.corr())
coeff_corr.to_csv('hw2_task1_6_correlation_coefficients_by_model.csv')


#-*- coding: utf8 -*-
import os, settings
from django.db import models

class Product(models.Model):
    name = models.CharField(verbose_name=u'Наименование', max_length=100)
    description = models.TextField(verbose_name=u'Описание')
    price = models.CharField(verbose_name=u'Цена', max_length=100)
    picture = models.ImageField(verbose_name=u'Изображение с продуктом', upload_to='uploads/products/')
    public = models.BooleanField(verbose_name=u'Опубликовать?', default=True)

    class Meta():
        ordering = ['name']
        verbose_name = u'Продукт'
        verbose_name_plural = u'Продукты'

    def __unicode__(self):
        return self.name

class Prices(models.Model):
    title = models.CharField(verbose_name=u'Название', max_length=150)
    file = models.FileField(verbose_name=u'Файл с прайс-листом', upload_to='uploads/prices/')
    file_size = models.CharField(verbose_name=u'Размер файла', editable=False, max_length=100)
    public = models.BooleanField(verbose_name=u'Опубликовать?', default=True)

    class Meta():
        ordering = ['title']
        verbose_name = u'Прайс'
        verbose_name_plural = u'Прайсы'

    def save(self):
        super(Prices, self).save()
        fullpath = os.path.join(
                settings.MEDIA_ROOT,
                '/uploads/prices/',
                self.file.path)
        self.file_size = os.path.getsize(fullpath)
        super(Prices, self).save()

    def __unicode__(self):
        return self.title