# Copyright (c) 2020, TU Wien, Department of Geodesy and Geoinformation
# All rights reserved.

# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#    * Redistributions of source code must retain the above copyright notice,
#      this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright
#      notice, this list of conditions and the following disclaimer in the
#      documentation and/or other materials provided with the distribution.
#    * Neither the name of TU Wien, Department of Geodesy and Geoinformation
#      nor the names of its contributors may be used to endorse or promote
#      products derived from this software without specific prior written
#      permission.

# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL TU WIEN DEPARTMENT OF GEODESY AND
# GEOINFORMATION BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""
Readers for lvl1b and lvl2 data in bufr format.
"""

import os
from datetime import datetime, timedelta
import warnings
from tempfile import NamedTemporaryFile
from gzip import GzipFile

import numpy as np

from pygeobase.io_base import ImageBase
from pygeobase.io_base import MultiTemporalImageBase
from pygeobase.io_base import IntervalReadingMixin
from pygeobase.object_base import Image

try:
    from pybufr_ecmwf import raw_bufr_file
    from pybufr_ecmwf import ecmwfbufr
    from pybufr_ecmwf import ecmwfbufr_parameters
except ImportError:
    warnings.warn(
        'pybufr-ecmwf can not be imported, H08 and H07 images can '
        'not be read.')

class AscatL1BufrFile(ImageBase):
    """
    Reads ASCAT L1b data in BUFR format.

    Parameters
    ----------
    filename : str
        Filename path.
    mode : str, optional
        Opening mode. Default: r
    msg_name_lookup: dict, optional
        Dictionary mapping bufr msg number to parameter name.
        See :ref:`ascatformattable`.

        Default:

             === =====================================================
             Key Value
             === =====================================================
             4: "Satellite Identifier",
             6: "Direction Of Motion Of Moving Observing Platform",
             16: "Orbit Number",
             17: "Cross-Track Cell Number",
             21: "f_Beam Identifier",
             22: "f_Radar Incidence Angle",
             23: "f_Antenna Beam Azimuth",
             24: "f_Backscatter",
             25: "f_Radiometric Resolution (Noise Value)",
             26: "f_ASCAT KP Estimate Quality",
             27: "f_ASCAT Sigma-0 Usability",
             34: "f_ASCAT Land Fraction",
             35: "m_Beam Identifier",
             36: "m_Radar Incidence Angle",
             37: "m_Antenna Beam Azimuth",
             38: "m_Backscatter",
             39: "m_Radiometric Resolution (Noise Value)",
             40: "m_ASCAT KP Estimate Quality",
             41: "m_ASCAT Sigma-0 Usability",
             48: "m_ASCAT Land Fraction",
             49: "a_Beam Identifier",
             50: "a_Radar Incidence Angle",
             51: "a_Antenna Beam Azimuth",
             52: "a_Backscatter",
             53: "a_Radiometric Resolution (Noise Value)",
             54: "a_ASCAT KP Estimate Quality",
             55: "a_ASCAT Sigma-0 Usability",
             62: "a_ASCAT Land Fraction"
             === =====================================================
    """

    def __init__(self, filename, mode='r', msg_name_lookup=None, **kwargs):
        """
        Initialization of i/o object.

        """
        zipped = False
        if os.path.splitext(filename)[1] == '.gz':
            zipped = True

        # for zipped files use an unzipped temporary copy
        if zipped:
            with NamedTemporaryFile(delete=False) as tmp_fid:
                with GzipFile(filename) as gz_fid:
                    tmp_fid.write(gz_fid.read())
                filename = tmp_fid.name

        super(AscatL1BufrFile, self).__init__(filename, mode=mode,
                                              **kwargs)
        if msg_name_lookup is None:
            msg_name_lookup = {
                4: "Satellite Identifier",
                6: "Direction Of Motion Of Moving Observing Platform",
                16: "Orbit Number",
                17: "Cross-Track Cell Number",
                21: "f_Beam Identifier",
                22: "f_Radar Incidence Angle",
                23: "f_Antenna Beam Azimuth",
                24: "f_Backscatter",
                25: "f_Radiometric Resolution (Noise Value)",
                26: "f_ASCAT KP Estimate Quality",
                27: "f_ASCAT Sigma-0 Usability",
                34: "f_ASCAT Land Fraction",
                35: "m_Beam Identifier",
                36: "m_Radar Incidence Angle",
                37: "m_Antenna Beam Azimuth",
                38: "m_Backscatter",
                39: "m_Radiometric Resolution (Noise Value)",
                40: "m_ASCAT KP Estimate Quality",
                41: "m_ASCAT Sigma-0 Usability",
                48: "m_ASCAT Land Fraction",
                49: "a_Beam Identifier",
                50: "a_Radar Incidence Angle",
                51: "a_Antenna Beam Azimuth",
                52: "a_Backscatter",
                53: "a_Radiometric Resolution (Noise Value)",
                54: "a_ASCAT KP Estimate Quality",
                55: "a_ASCAT Sigma-0 Usability",
                62: "a_ASCAT Land Fraction"}
        self.msg_name_lookup = msg_name_lookup

    def read(self, timestamp=None):
        """
        Read specific image for given datetime timestamp.

        Parameters
        ----------
        timestamp : datetime.datetime
            exact observation timestamp of the image that should be read

        Returns
        -------
        data : dict
            dictionary of numpy arrays that hold the image data for each
            variable of the dataset
        metadata : dict
            dictionary of numpy arrays that hold the metadata
        timestamp : datetime.datetime
            exact timestamp of the image
        lon : numpy.array or None
            array of longitudes, if None self.grid will be assumed
        lat : numpy.array or None
            array of latitudes, if None self.grid will be assumed
        time_var : string or None
            variable name of observation times in the data dict, if None all
            observations have the same timestamp
        """
        # lookup table between names and message number in the BUFR file

        data = {}
        dates = []
        # 13: Latitude (High Accuracy)
        latitude = []
        # 14: Longitude (High Accuracy)
        longitude = []

        with BUFRReader(self.filename) as bufr:
            for message in bufr.messages():
                # read fixed fields
                latitude.append(message[:, 12])
                longitude.append(message[:, 13])
                years = message[:, 6].astype(int)
                months = message[:, 7].astype(int)
                days = message[:, 8].astype(int)
                hours = message[:, 9].astype(int)
                minutes = message[:, 10].astype(int)
                seconds = message[:, 11].astype(int)

                dates.append(
                    julday(months, days, years, hours, minutes, seconds))

                # read optional data fields
                for mid in self.msg_name_lookup:
                    name = self.msg_name_lookup[mid]

                    if name not in data:
                        data[name] = []

                    data[name].append(message[:, mid - 1])

        dates = np.concatenate(dates)
        longitude = np.concatenate(longitude)
        latitude = np.concatenate(latitude)
        n_records = latitude.shape[0]

        for mid in self.msg_name_lookup:
            name = self.msg_name_lookup[mid]
            data[name] = np.concatenate(data[name])

        data['jd'] = dates
        if 'Direction Of Motion Of Moving Observing Platform' in data:
            data['as_des_pass'] = (data[
                "Direction Of Motion Of Moving Observing Platform"
            ] < 270).astype(np.uint8)

        if 'Cross-Track Cell Number' in data:
            if data['Cross-Track Cell Number'].max() == 82:
                data['swath_indicator'] = 1 * (
                    data['Cross-Track Cell Number'] > 41)
            elif data['Cross-Track Cell Number'].max() == 42:
                data['swath_indicator'] = 1 * (
                    data['Cross-Track Cell Number'] > 21)
            else:
                raise ValueError("Unsuspected node number.")
            n_lines = n_records / max(data['Cross-Track Cell Number'])
            data['line_num'] = np.arange(n_lines).repeat(
                max(data['Cross-Track Cell Number']))

        # There are strange elements with a value of 32.32 instead of the
        # typical nan_values
        # Since some elements rly have this value we check the other triplet
        # data of that beam to filter the nan_values out
        beams = ['f', 'm', 'a']
        for beam in beams:
            azi = beam + '_Antenna Beam Azimuth'
            sig = beam + '_Backscatter'
            inc = beam + '_Radar Incidence Angle'
            if azi in data:
                mask_azi = data[azi] == 32.32
                mask_sig = data[sig] == 1.7e+38
                mask_inc = data[inc] == 1.7e+38
                mask = np.all([mask_azi, mask_sig, mask_inc], axis=0)
                data[azi][mask] = 1.7e+38

        # 1 ERS 1
        # 2 ERS 2
        # 3 METOP-1 (Metop-B)
        # 4 METOP-2 (Metop-A)
        # 5 METOP-3 (Metop-C)
        sat_id = {3: 1, 4: 2, 5: 3}

        metadata = {}
        metadata['SPACECRAFT_ID'] = np.int8(
            sat_id[data['Satellite Identifier'][0]])
        metadata['ORBIT_START'] = np.uint32(data['Orbit Number'][0])

        return Image(longitude, latitude, data, metadata,
                     timestamp, timekey='jd')

    def write(self, data):
        raise NotImplementedError()

    def flush(self):
        pass

    def close(self):
        pass

class AscatL2SsmBufrFile(ImageBase):
    """
    Reads ASCAT SSM swath files in BUFR format. There are the
    following products:

    - H101 SSM ASCAT-A NRT O 12.5 Metop-A ASCAT NRT SSM orbit geometry
    12.5 km sampling
    - H102 SSM ASCAT-A NRT O 25.0 Metop-A ASCAT NRT SSM orbit geometry
    25 km sampling
    - H16  SSM ASCAT-B NRT O 12.5 Metop-B ASCAT NRT SSM orbit geometry
    12.5 km sampling
    - H103 SSM ASCAT-B NRT O 25.0 Metop-B ASCAT NRT SSM orbit geometry
    25 km sampling
    - H104 SSM ASCAT-C NRT O 12.5 Metop-C ASCAT NRT SSM orbit geometry
    12.5 km sampling
    - H105 SSM ASCAT-C NRT O 25.0 Metop-C ASCAT NRT SSM orbit geometry
    25 km sampling
    - EUMETSAT ASCAT Soil Moisture at 12.5 km Swath Grid - Metop in BUFR format
    - EUMETSAT ASCAT Soil Moisture at 25.0 km Swath Grid - Metop in BUFR format

    Parameters
    ----------
    filename : str
        Filename path.
    mode : str, optional
        Opening mode. Default: r
    msg_name_lookup: dict, optional
        Dictionary mapping bufr msg number to parameter name.
        See :ref:`ascatformattable`.

        Default:

             === =====================================================
             Key Value
             === =====================================================
             4: "Satellite Identifier",
             6: "Direction Of Motion Of Moving Observing Platform",
             16: "Orbit Number",
             17: "Cross-Track Cell Number",
             21: "f_Beam Identifier",
             22: "f_Radar Incidence Angle",
             23: "f_Antenna Beam Azimuth",
             24: "f_Backscatter",
             25: "f_Radiometric Resolution (Noise Value)",
             26: "f_ASCAT KP Estimate Quality",
             27: "f_ASCAT Sigma-0 Usability",
             34: "f_ASCAT Land Fraction",
             35: "m_Beam Identifier",
             36: "m_Radar Incidence Angle",
             37: "m_Antenna Beam Azimuth",
             38: "m_Backscatter",
             39: "m_Radiometric Resolution (Noise Value)",
             40: "m_ASCAT KP Estimate Quality",
             41: "m_ASCAT Sigma-0 Usability",
             48: "m_ASCAT Land Fraction",
             49: "a_Beam Identifier",
             50: "a_Radar Incidence Angle",
             51: "a_Antenna Beam Azimuth",
             52: "a_Backscatter",
             53: "a_Radiometric Resolution (Noise Value)",
             54: "a_ASCAT KP Estimate Quality",
             55: "a_ASCAT Sigma-0 Usability",
             62: "a_ASCAT Land Fraction",
             65: "Surface Soil Moisture (Ms)",
             66: "Estimated Error In Surface Soil Moisture",
             67: "Backscatter",
             68: "Estimated Error In Sigma0 At 40 Deg Incidence Angle",
             69: "Slope At 40 Deg Incidence Angle",
             70: "Estimated Error In Slope At 40 Deg Incidence Angle",
             71: "Soil Moisture Sensitivity",
             72: "Dry Backscatter",
             73: "Wet Backscatter",
             74: "Mean Surface Soil Moisture",
             75: "Rain Fall Detection",
             76: "Soil Moisture Correction Flag",
             77: "Soil Moisture Processing Flag",
             78: "Soil Moisture Quality",
             79: "Snow Cover",
             80: "Frozen Land Surface Fraction",
             81: "Inundation And Wetland Fraction",
             82: "Topographic Complexity"
             === =====================================================
    """

    def __init__(self, filename, mode='r', msg_name_lookup=None, **kwargs):
        """
        Initialization of i/o object.

        """
        super(AscatL2SsmBufrFile, self).__init__(filename, mode=mode,
                                                 **kwargs)
        if msg_name_lookup is None:
            msg_name_lookup = {
                4: "Satellite Identifier",
                6: "Direction Of Motion Of Moving Observing Platform",
                16: "Orbit Number",
                17: "Cross-Track Cell Number",
                21: "f_Beam Identifier",
                22: "f_Radar Incidence Angle",
                23: "f_Antenna Beam Azimuth",
                24: "f_Backscatter",
                25: "f_Radiometric Resolution (Noise Value)",
                26: "f_ASCAT KP Estimate Quality",
                27: "f_ASCAT Sigma-0 Usability",
                34: "f_ASCAT Land Fraction",
                35: "m_Beam Identifier",
                36: "m_Radar Incidence Angle",
                37: "m_Antenna Beam Azimuth",
                38: "m_Backscatter",
                39: "m_Radiometric Resolution (Noise Value)",
                40: "m_ASCAT KP Estimate Quality",
                41: "m_ASCAT Sigma-0 Usability",
                48: "m_ASCAT Land Fraction",
                49: "a_Beam Identifier",
                50: "a_Radar Incidence Angle",
                51: "a_Antenna Beam Azimuth",
                52: "a_Backscatter",
                53: "a_Radiometric Resolution (Noise Value)",
                54: "a_ASCAT KP Estimate Quality",
                55: "a_ASCAT Sigma-0 Usability",
                62: "a_ASCAT Land Fraction",
                65: "Surface Soil Moisture (Ms)",
                66: "Estimated Error In Surface Soil Moisture",
                67: "Backscatter",
                68: "Estimated Error In Sigma0 At 40 Deg Incidence Angle",
                69: "Slope At 40 Deg Incidence Angle",
                70: "Estimated Error In Slope At 40 Deg Incidence Angle",
                71: "Soil Moisture Sensitivity",
                72: "Dry Backscatter",
                73: "Wet Backscatter",
                74: "Mean Surface Soil Moisture",
                75: "Rain Fall Detection",
                76: "Soil Moisture Correction Flag",
                77: "Soil Moisture Processing Flag",
                78: "Soil Moisture Quality",
                79: "Snow Cover",
                80: "Frozen Land Surface Fraction",
                81: "Inundation And Wetland Fraction",
                82: "Topographic Complexity"}
        self.msg_name_lookup = msg_name_lookup

    def read(self, timestamp=None, ssm_masked=False):
        """
        Read specific image for given datetime timestamp.

        Parameters
        ----------
        timestamp : datetime.datetime (optional)
            exact observation timestamp of the image that should be read
        ssm_masked : flag (optional)
            set to True to filter data by ssm values

        Returns
        -------
        data : dict
            dictionary of numpy arrays that hold the image data for each
            variable of the dataset
        metadata : dict
            dictionary of numpy arrays that hold the metadata
        timestamp : datetime.datetime
            exact timestamp of the image
        lon : numpy.array or None
            array of longitudes, if None self.grid will be assumed
        lat : numpy.array or None
            array of latitudes, if None self.grid will be assumed
        time_var : string or None
            variable name of observation times in the data dict, if None all
            observations have the same timestamp
        """
        # lookup table between names and message number in the BUFR file

        data = {}
        dates = []
        # 13: Latitude (High Accuracy)
        latitude = []
        # 14: Longitude (High Accuracy)
        longitude = []

        with BUFRReader(self.filename) as bufr:
            for message in bufr.messages():
                # read fixed fields
                latitude.append(message[:, 12])
                longitude.append(message[:, 13])
                years = message[:, 6].astype(int)
                months = message[:, 7].astype(int)
                days = message[:, 8].astype(int)
                hours = message[:, 9].astype(int)
                minutes = message[:, 10].astype(int)
                seconds = message[:, 11].astype(int)

                dates.append(
                    julday(months, days, years, hours, minutes, seconds))

                # read optional data fields
                for mid in self.msg_name_lookup:
                    name = self.msg_name_lookup[mid]

                    if name not in data:
                        data[name] = []

                    data[name].append(message[:, mid - 1])

        dates = np.concatenate(dates)
        longitude = np.concatenate(longitude)
        latitude = np.concatenate(latitude)
        n_records = latitude.shape[0]

        for mid in self.msg_name_lookup:
            name = self.msg_name_lookup[mid]
            data[name] = np.concatenate(data[name])
            if mid == 74:
                # ssm mean is encoded differently
                data[name] = data[name] * 100

        data['jd'] = dates

        if 'Direction Of Motion Of Moving Observing Platform' in data:
            data['as_des_pass'] = (data[
                "Direction Of Motion Of Moving Observing Platform"]
                <= 270).astype(np.uint8)

        if 'Cross-Track Cell Number' in data:
            if data['Cross-Track Cell Number'].max() == 82:
                data['swath_indicator'] = 1 * (
                    data['Cross-Track Cell Number'] > 41)
            elif data['Cross-Track Cell Number'].max() == 42:
                data['swath_indicator'] = 1 * (
                    data['Cross-Track Cell Number'] > 21)
            else:
                raise ValueError("Unsuspected node number.")
            n_lines = n_records / max(data['Cross-Track Cell Number'])
            data['line_num'] = np.arange(n_lines).repeat(
                max(data['Cross-Track Cell Number']))

        # There are strange elements with a value of 32.32 instead of the
        # typical nan_values
        # Since some elements rly have this value we check the other triplet
        # data of that beam to filter the nan_values out
        beams = ['f', 'm', 'a']
        for beam in beams:
            azi = beam + '_Antenna Beam Azimuth'
            sig = beam + '_Backscatter'
            inc = beam + '_Radar Incidence Angle'
            if azi in data:
                mask_azi = data[azi] == 32.32
                mask_sig = data[sig] == 1.7e+38
                mask_inc = data[inc] == 1.7e+38
                mask = np.all([mask_azi, mask_sig, mask_inc], axis=0)
                data[azi][mask] = 1.7e+38

        # if the ssm_masked is True we mask out data with missing ssm value
        if 65 in self.msg_name_lookup and ssm_masked is True:
            # mask all the arrays based on fill_value of soil moisture
            valid_data = np.where(data[self.msg_name_lookup[65]] != 1.7e+38)
            latitude = latitude[valid_data]
            longitude = longitude[valid_data]
            for name in data:
                data[name] = data[name][valid_data]

        # 1 ERS 1
        # 2 ERS 2
        # 3 METOP-1 (Metop-B)
        # 4 METOP-2 (Metop-A)
        # 5 METOP-3 (Metop-C)
        sat_id = {3: 1, 4: 2, 5: 3}

        metadata = {}

        try:
            metadata['SPACECRAFT_ID'] = np.int8(
                sat_id[data['Satellite Identifier'][0]])
        except KeyError:
            metadata['SPACECRAFT_ID'] = 0

        try:
            metadata['ORBIT_START'] = np.uint32(data['Orbit Number'][0])
        except KeyError:
            metadata['ORBIT_START'] = 0

        return Image(longitude, latitude, data, metadata,
                     timestamp, timekey='jd')

    def resample_data(self, image, index, distance, weights, **kwargs):
        """
        Takes an image and resample (interpolate) the image data to
        arbitrary defined locations given by index and distance.

        Parameters
        ----------
        image : object
            pygeobase.object_base.Image object
        index : np.array
            Index into image data defining a look-up table for data elements
            used in the interpolation process for each defined target
            location.
        distance : np.array
            Array representing the distances of the image data to the
            arbitrary defined locations.
        weights : np.array
            Array representing the weights of the image data that should be
            used during resampling.
            The weights of points not to use are set to np.nan
            This array is of shape (x, max_neighbors)

        Returns
        -------
        image : object
            pygeobase.object_base.Image object
        """
        total_weights = np.nansum(weights, axis=1)

        resOrbit = {}
        # resample backscatter
        for name in image.dtype.names:
            if name in ['Soil Moisture Correction Flag',
                        'Soil Moisture Processing Flag']:
                # The flags are resampled by taking the minimum flag This works
                # since any totally valid observation has the flag 0 and
                # overrides the flagged observations. This is true in cases
                # where the data was set to NaN by the flag as well as when the
                # data was set to 0 or 100. The last image element is the one
                # standing for NaN so we fill it with all flags filled to not
                # interfere with the minimum.
                image[name][-1] = 255
                bits = np.unpackbits(image[name].reshape(
                    (-1, 1)).astype(np.uint8), axis=1)
                resampled_bits = np.min(bits[index, :], axis=1)
                resOrbit[name] = np.packbits(resampled_bits)
            else:
                resOrbit[name] = np.nansum(
                    image[name][index] * weights, axis=1) / total_weights

        return resOrbit

    def write(self, data):
        raise NotImplementedError()

    def flush(self):
        pass

    def close(self):
        pass

class AscatL2SsmBufr(MultiTemporalImageBase):
    """
    Class for reading HSAF ASCAT SSM images in bufr format.
    The images have the same structure as the ASCAT 3 minute pdu files
    and these 2 readers could be merged in the future
    The images have to be uncompressed in the following folder structure
    path - month_path_str (default 'h07_%Y%m_buf')

    For example if path is set to /home/user/hsaf07 and month_path_str is left
    to the default 'h07_%Y%m_buf' then the images for March 2012 have to be in
    the folder /home/user/hsaf07/h07_201203_buf/

    Parameters
    ----------
    path: string
        path where the data is stored
    month_path_str: string, optional
        if the files are stored in folders by month as is the standard on the
        HSAF FTP Server then please specify the string that should be used in
        datetime.datetime.strftime Default: 'h07_%Y%m_buf'
    day_search_str: string, optional
        to provide an iterator over all images of a day the
        method _get_possible_timestamps looks for all available images on a day
        on the harddisk. This string is used in datetime.datetime.strftime and
        in glob.glob to search for all files on a day.
        Default : 'h07_%Y%m%d_*.buf'
    file_search_str: string, optional
        this string is used in datetime.datetime.strftime and glob.glob to find
        a 3 minute bufr file by the exact date.
        Default: 'h07_{datetime}*.buf'
    datetime_format: string, optional
        datetime format by which {datetime} will be replaced in file_search_str
        Default: %Y%m%d_%H%M%S
    msg_name_lookup: dict, optional
        Dictionary mapping bufr msg number to parameter name.
        See :ref:`ascatformattable`.

        Default:

             === =====================================================
             Key Value
             === =====================================================
             4:  "Satellite Identifier",
             6:  "Direction Of Motion Of Moving Observing Platform",
             16: "Orbit Number",
             17: "Cross-Track Cell Number",
             21: "f_Beam Identifier",
             22: "f_Radar Incidence Angle",
             23: "f_Antenna Beam Azimuth",
             24: "f_Backscatter",
             25: "f_Radiometric Resolution (Noise Value)",
             26: "f_ASCAT KP Estimate Quality",
             27: "f_ASCAT Sigma-0 Usability",
             34: "f_ASCAT Land Fraction",
             35: "m_Beam Identifier",
             36: "m_Radar Incidence Angle",
             37: "m_Antenna Beam Azimuth",
             38: "m_Backscatter",
             39: "m_Radiometric Resolution (Noise Value)",
             40: "m_ASCAT KP Estimate Quality",
             41: "m_ASCAT Sigma-0 Usability",
             48: "m_ASCAT Land Fraction",
             49: "a_Beam Identifier",
             50: "a_Radar Incidence Angle",
             51: "a_Antenna Beam Azimuth",
             52: "a_Backscatter",
             53: "a_Radiometric Resolution (Noise Value)",
             54: "a_ASCAT KP Estimate Quality",
             55: "a_ASCAT Sigma-0 Usability",
             62: "a_ASCAT Land Fraction",
             65: "Surface Soil Moisture (Ms)",
             66: "Estimated Error In Surface Soil Moisture",
             67: "Backscatter",
             68: "Estimated Error In Sigma0 At 40 Deg Incidence Angle",
             69: "Slope At 40 Deg Incidence Angle",
             70: "Estimated Error In Slope At 40 Deg Incidence Angle",
             71: "Soil Moisture Sensitivity",
             72: "Dry Backscatter",
             73: "Wet Backscatter",
             74: "Mean Surface Soil Moisture",
             75: "Rain Fall Detection",
             76: "Soil Moisture Correction Flag",
             77: "Soil Moisture Processing Flag",
             78: "Soil Moisture Quality",
             79: "Snow Cover",
             80: "Frozen Land Surface Fraction",
             81: "Inundation And Wetland Fraction",
             82: "Topographic Complexity"
             === =====================================================
    """

    def __init__(self, path, month_path_str='h07_%Y%m_buf',
                 day_search_str='h07_%Y%m%d_*.buf',
                 file_search_str='h07_{datetime}*.buf',
                 datetime_format='%Y%m%d_%H%M%S',
                 filename_datetime_format=(4, 19, '%Y%m%d_%H%M%S'),
                 msg_name_lookup=None):
        self.path = path
        self.month_path_str = month_path_str
        self.day_search_str = day_search_str
        self.file_search_str = file_search_str
        self.filename_datetime_format = filename_datetime_format
        super(AscatL2SsmBufr, self).__init__(path, AscatL2SsmBufrFile,
                                             subpath_templ=[month_path_str],
                                             fname_templ=file_search_str,
                                             datetime_format=datetime_format,
                                             exact_templ=False,
                                             ioclass_kws={
                                                 'msg_name_lookup':
                                                     msg_name_lookup})

    def _get_orbit_start_date(self, filename):
        orbit_start_str = \
            os.path.basename(filename)[self.filename_datetime_format[0]:
                                       self.filename_datetime_format[1]]
        return datetime.strptime(orbit_start_str,
                                 self.filename_datetime_format[2])

    def tstamps_for_daterange(self, startdate, enddate):
        """
        Get the timestamps as datetime array that are possible for the
        given day.

        For this product it is not fixed but has to be looked up from
        the hard disk since bufr files are not regular spaced and only
        europe is in this product. For a global product a 3 minute
        spacing could be used as a fist approximation

        Parameters
        ----------
        startdate : datetime.date or datetime.datetime
            start date
        enddate : datetime.date or datetime.datetime
            end date

        Returns
        -------
        dates : list
            list of datetimes
        """
        file_list = []
        delta_all = enddate - startdate
        timestamps = []

        for i in range(delta_all.days + 1):
            timestamp = startdate + timedelta(days=i)

            files = self._search_files(
                timestamp, custom_templ=self.day_search_str)

            file_list.extend(sorted(files))

        for filename in file_list:
            timestamps.append(self._get_orbit_start_date(filename))

        timestamps = [dt for dt in timestamps if (
            startdate <= dt <= enddate)]
        return timestamps

class AscatL2SsmBufrChunked(IntervalReadingMixin, AscatL2SsmBufr):
    """
    Reads BUFR files but does not return them on a file by file basis but in
    bigger chunks. For example it allows to read multiple 3 minute PDU's in
    half orbit chunks of 50 minutes. This speeds up operations like e.g.
    resampling of the data.

    Parameters
    ----------
    chunk_minutes: int, optional
        How many minutes should a chunk of data cover.
    """

    def __init__(self, path, month_path_str='h07_%Y%m_buf',
                 day_search_str='h07_%Y%m%d_*.buf',
                 file_search_str='h07_{datetime}*.buf',
                 datetime_format='%Y%m%d_%H%M%S',
                 filename_datetime_format=(4, 19, '%Y%m%d_%H%M%S'),
                 msg_name_lookup=None, chunk_minutes=50):
        super(AscatL2SsmBufrChunked, self).__init__(
            path,
            month_path_str=month_path_str,
            day_search_str=day_search_str,
            file_search_str=file_search_str,
            datetime_format=datetime_format,
            filename_datetime_format=filename_datetime_format,
            msg_name_lookup=msg_name_lookup,
            chunk_minutes=chunk_minutes)

class BUFRReader(object):
    """
    BUFR reader based on the pybufr-ecmwf package but faster

    Parameters
    ----------
    filename : string
        filename of the bufr file
    kelem_guess : int, optional
        if the elements per variable in as message are known
        please specify here.
        Otherwise the elements will be found out via trial and error
        This works most of the time but is not 100 percent failsafe
        Default: 500
    max_tries : int, optional
        the Reader will try max_tries times to unpack a bufr message.
        Some messages can not be read even if the array sizes are ok.
        Most of the time these files are corrupt.

    """

    def __init__(self, filename, kelem_guess=500, max_tries=10):
        self.bufr = raw_bufr_file.RawBUFRFile()
        self.bufr.open(filename, 'rb')
        self.nr_messages = self.bufr.get_num_bufr_msgs()
        self.max_tries = max_tries

        if 'BUFR_TABLES' not in os.environ:
            path = os.path.split(ecmwfbufr.__file__)[0]
            os.environ["BUFR_TABLES"] = os.path.join(
                path, 'ecmwf_bufrtables' + os.sep)
            # os.environ['PRINT_TABLE_NAMES'] = "false"

        self.size_ksup = ecmwfbufr_parameters.JSUP
        self.size_ksec0 = ecmwfbufr_parameters.JSEC0
        self.size_ksec1 = ecmwfbufr_parameters.JSEC1
        self.size_ksec2 = ecmwfbufr_parameters.JSEC2
        self.size_key = ecmwfbufr_parameters.JKEY
        self.size_ksec3 = ecmwfbufr_parameters.JSEC3
        self.size_ksec4 = ecmwfbufr_parameters.JSEC4

        self.kelem_guess = kelem_guess

    def messages(self):
        """
        Raises
        ------
        IOError:
            if a message cannot be unpacked after max_tries tries

        Returns
        -------
        data : yield results of messages
        """
        count = 0
        for i in np.arange(self.nr_messages) + 1:
            tries = 0

            ksup = np.zeros(self.size_ksup, dtype=np.int)
            ksec0 = np.zeros(self.size_ksec0, dtype=np.int)
            ksec1 = np.zeros(self.size_ksec1, dtype=np.int)
            ksec2 = np.zeros(self.size_ksec2, dtype=np.int)
            ksec3 = np.zeros(self.size_ksec3, dtype=np.int)
            ksec4 = np.zeros(self.size_ksec4, dtype=np.int)

            kerr = 0
            data = self.bufr.get_raw_bufr_msg(i)

            ecmwfbufr.bus012(data[0],  # input
                             ksup,  # output
                             ksec0,  # output
                             ksec1,  # output
                             ksec2,  # output
                             kerr)  # output

            kelem = self.kelem_guess
            ksup_first = ksup[5]
            kvals = ksup_first * kelem
            max_kelem = 500000
            self.init_values = np.zeros(kvals, dtype=np.float64)
            self.cvals = np.zeros((kvals, 80), dtype=np.character)
            # try to expand bufr message with the first guess for
            # kelem
            increment_arraysize = True
            while increment_arraysize:
                cnames = np.zeros((kelem, 64), dtype='|S1')
                cunits = np.zeros((kelem, 24), dtype='|S1')

                with warnings.catch_warnings():
                    warnings.filterwarnings(
                        "ignore", category=DeprecationWarning)
                    ecmwfbufr.bufrex(data[0],  # input
                                     ksup,  # output
                                     ksec0,  # output
                                     ksec1,  # output
                                     ksec2,  # output
                                     ksec3,  # output
                                     ksec4,  # output
                                     cnames,  # output
                                     cunits,  # output
                                     self.init_values,  # output
                                     self.cvals,  # output
                                     kerr)  # output
                # no error - stop loop
                if kerr == 0 and ksec4[0] != 0:
                    increment_arraysize = False
                # error increase array size and try to unpack again
                else:
                    tries += 1
                    if tries >= self.max_tries:
                        raise IOError('This file seems corrupt')
                    kelem = kelem * 5
                    kvals = ksup_first * kelem

                    if kelem > max_kelem:
                        kelem = kvals / 2
                        max_kelem = kvals

                    self.init_values = np.zeros(kvals, dtype=np.float64)
                    self.cvals = np.zeros((kvals, 80), dtype=np.character)

            decoded_values = ksup[4]
            # set kelem_guess to decoded values of last message
            # only increases reading speed if all messages are the same
            # not sure if this is the best option
            self.kelem_guess = decoded_values
            decoded_msg = ksup[5]
            # calculate first dimension of 2D array
            factor = int(kvals / kelem)

            # reshape and trim the array to the actual size of the data
            values = self.init_values.reshape((factor, kelem))
            values = values[:decoded_msg, :decoded_values]
            count += values.shape[0]

            yield values

    def __enter__(self):
        return self

    def __exit__(self, exc, val, trace):
        self.bufr.close()

def julday(month, day, year, hour=0, minute=0, second=0):
    """
    Julian date from month, day, and year (can be scalars or arrays)
    (function from pytesmo)
    Parameters
    ----------
    month : numpy.ndarray or int32
        Month.
    day : numpy.ndarray or int32
        Day.
    year : numpy.ndarray or int32
        Year.
    hour : numpy.ndarray or int32, optional
        Hour.
    minute : numpy.ndarray or int32, optional
        Minute.
    second : numpy.ndarray or int32, optional
        Second.
    Returns
    -------
    jul : numpy.ndarray or double
        Julian day.
    """
    month = np.array(month)
    day = np.array(day)
    inJanFeb = month <= 2
    jy = year - inJanFeb
    jm = month + 1 + inJanFeb * 12

    jul = np.int32(np.floor(365.25 * jy) +
                   np.floor(30.6001 * jm) + (day + 1720995.0))
    ja = np.int32(0.01 * jy)
    jul += 2 - ja + np.int32(0.25 * ja)

    jul = jul + hour / 24.0 - 0.5 + minute / 1440.0 + second / 86400.0

    return jul


"""
DPO Deskew
Uses a single 0V-1V rising edge source split into
two scope channels to deskew those channels
Edited: 9/17
Windows 7 64-bit
Python 3.6.1 64-bit (Anaconda 4.4.0)
PyVISA 1.8
Get Anaconda: http://continuum.io/downloads
Anaconda includes NumPy
Get PyVISA: pip install pyvisa
"""

import visa

print('PyVISA Version:', visa.__version__)

# Set up VISA instrument object
rm = visa.ResourceManager()
dpo = rm.open_resource('TCPIP::192.168.1.12::INSTR')
dpo.timeout = 5000
dpo.encoding = 'latin_1'
dpo.write_termination = None
dpo.read_termination = '\n'
print('Connected to ', dpo.query('*idn?'))
dpo.write('header off')
dpo.write('*rst')
dpo.write('*cls')

channels = [1, 3]
vertScale = 250e-3
horizScale = 1e-9

for c in channels:
    dpo.write('select:ch{} on'.format(c))
    dpo.write('ch{}:scale {}'.format(c, vertScale))

dpo.write('horiz:mode:scale {}'.format(horizScale))
dpo.write('trigger:a setlevel')
dpo.write('acquire:mode average')

dpo.write('acquire:stopafter sequence')
dpo.query('*opc?')

dpo.write('measurement:immed:type delay')
dpo.write('measurement:immed:source1 ch1')
dpo.write('measurement:immed:source2 ch3')
dpo.write('measurement:immed:delay:edge1 rise')
dpo.write('measurement:immed:delay:edge2 rise')
dpo.write('measurement:immed:delay:direction forwards')

print(dpo.query('measurement:immed?'))
deskew = dpo.query('measurement:immed:value?')
print(deskew)

# dpo.write('acquire:stopafter runstop')
dpo.write('acquire:state on')

dpo.write('ch3:deskew ', deskew)

# Check for errors
dpo.query('*esr?')
error = dpo.query('system:error:all?')
print('Status: {}'.format(error))

dpo.close()


from django.conf.urls import url

from apps.hygiene import views

app_name = 'hygiene'

urlpatterns = [
    url('^check/$', views.check, name='check'),
    url('^check/(?P<pk>[0-9]+)/$', views.check, name='check_day'),

    url('^plan/$', views.plan, name='plan'),
    url('^plan/(?P<year>[0-9]+)/(?P<month>[0-9]+)/$', views.plan, name='plan_month'),

    url('^calendar/(?P<pk>[0-9]+).ics', views.ical, name='ical'),
]


from django.conf import settings
from django.contrib import admin
from django.contrib.admin.util import unquote
from django.contrib.admin.views.main import ChangeList
from django.core.urlresolvers import reverse
from django.http import HttpResponseRedirect
from django.shortcuts import get_object_or_404
from functools import update_wrapper
from django.utils.html import strip_spaces_between_tags as short
from django.utils.translation import ugettext_lazy as _

class OrderedModelAdmin(admin.ModelAdmin):
    def get_urls(self):
        from django.conf.urls import patterns, url

        def wrap(view):
            def wrapper(*args, **kwargs):
                return self.admin_site.admin_view(view)(*args, **kwargs)
            return update_wrapper(wrapper, view)
        info = self.model._meta.app_label, self.model._meta.module_name
        return patterns('',
            url(r'^(.+)/move-(up)/$',
                wrap(self.move_view),
                name='%s_%s_move_up' % info),
            url(r'^(.+)/move-(down)/$',
                wrap(self.move_view),
                name='%s_%s_move_down' % info),
        ) + super(OrderedModelAdmin, self).get_urls()

    def _get_changelist(self, request):
        list_display = self.get_list_display(request)
        list_display_links = self.get_list_display_links(request, list_display)

        cl = ChangeList(request, self.model, list_display,
            list_display_links, self.list_filter, self.date_hierarchy,
            self.search_fields, self.list_select_related,
            self.list_per_page, self.list_max_show_all, self.list_editable,
            self)

        return cl

    request_query_string = ''

    def changelist_view(self, request, extra_context=None):
        cl = self._get_changelist(request)
        self.request_query_string = cl.get_query_string()
        return super(OrderedModelAdmin, self).changelist_view(request, extra_context)

    def move_view(self, request, object_id, direction):
        cl = self._get_changelist(request)
        qs = cl.get_query_set(request)

        obj = get_object_or_404(self.model, pk=unquote(object_id))
        obj.move(direction, qs)

        return HttpResponseRedirect('../../%s' % self.request_query_string)

    def move_up_down_links(self, obj):
        link_html = short("""

            """)

        return link_html % {
            'app_label': self.model._meta.app_label,
            'module_name': self.model._meta.module_name,
            'object_id': obj.id,
            'STATIC_URL': settings.STATIC_URL,
            'query_string': self.request_query_string
        }
    move_up_down_links.allow_tags = True
    move_up_down_links.short_description = _(u'Move')


# Code to format our imput data, outputting a clean restaurant list, dictionary and corpus with foodwords also removed.
# Authors Robert Walport and Rashmi Raman

import csv, sys, re
from gensim import corpora, models, similarities

with open(sys.argv[1], "rb") as csvFile:
    rawlist = []
    cleanlist = []
    finallist = []
    reader = csv.reader(csvFile, delimiter=',')
    for row in reader:
        rawlist.append(row)
    rawlist.pop(0)
    for entry in rawlist:
        cleanlist.append([entry[0], ''])
    for entry in cleanlist:
        if entry not in finallist:
            finallist.append(entry)
    for entry in finallist:
        for rest in rawlist:
            if entry[0] == rest[0]:
                try:
                    entry[1] = entry[1] + " " + rest[3]
                except IndexError:
                    entry[1] = entry[1]
    restaurants = open('restaurants.txt', 'w+')
    ratings = open('reviews.txt', 'w+')
    for entry in finallist:
        restaurants.write(entry[0].lower() + "\n")
        ratings.write(entry[1].lower() + '\n')
    restaurants.close()
    ratings.close()

stopwords = open('stopwords.txt', 'r').read().split()
foodwords = open('foodwords.txt', 'r').read().split()

class MyCorpus(object):
    def __iter__(self):
        for line in open('reviews.txt'):
            line = line.lower()
            scrubwords = re.findall(r'\w+', line)
            yield dictionary.doc2bow(scrubwords)

# collect statistics about all tokens
dictionary = corpora.Dictionary(line.lower().split() for line in open('reviews.txt'))
# remove stop words and words that appear only once
stop_ids = [dictionary.token2id[stopword] for stopword in stopwords if stopword in dictionary.token2id]
food_ids = [dictionary.token2id[foodword] for foodword in foodwords if foodword in dictionary.token2id]
once_ids = [tokenid for tokenid, docfreq in dictionary.dfs.iteritems() if docfreq == 1]
dictionary.filter_tokens(stop_ids + food_ids + once_ids) # remove stop words and words that appear only once
dictionary.compactify() # remove gaps in id sequence after words that were removed
dictionary.save('restaurantreviews.dict')

corpus_memory_friendly = MyCorpus() # doesn't load the corpus into memory!

corpora.MmCorpus.serialize('reviewscorpus.mm', corpus_memory_friendly)


import os

import matplotlib.pyplot as plt
import matplotlib.colors as colors
from matplotlib.gridspec import GridSpec
import numpy as np
import pandas as pd
import seaborn as sns
import scipy as sp
from statsmodels.graphics.gofplots import qqplot_2samples
import xarray as xr
import json
import numpy as np

class Validate:
    def __init__(self, analysis_path, quantity, model, versions):

        self.analysis_path = analysis_path
        self.quantity = quantity
        self.save_path = os.path.join(analysis_path, '..', '..', 'plots')
        self.model = model
        self.versions = versions

        # TODO add sth similar for 4x4 if needed
        if self.model== 'mam':
            self.colors = {
                'master-rng': '#ee7733',
                '2.20.1': '#01796f'
            }
        sns.set()

    def load_data(self):
        data = {}
        for version in self.versions:
            if self.model == 'mam':
                data[version] = self._load_data_mam(version)
            elif self.model == '4x4':
                data[version] = self._load_data_4x4(version)
            else:
                raise Exception(f'No data loading implemented for {self.model}')
        self.data = data

# USER ACCESSIBLE FUNCTIONS

    def plot_ks_score(self):
        self._plot_ks_score(self._calc_ks_score())

    def plot_p_val(self, stest, tname, qname, qix, gs, fig):
        return self._plot_p_val(self._calc_p_val(stest), tname, qname, qix, gs, fig)

    def plot_qq(self, area, layer):
        fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(5, 5))
        for pop in self.pops:
            qqplot_2samples(self.data['2.20.1'].loc[area, layer, pop].values,
                            self.data['master-rng'].loc[area, layer, pop].values,
                            ax=ax)
            plt.legend()
            plt.savefig(
                os.path.join(self.save_path, '_'.join(['qq',
                    self.quantity, area, layer, pop]) + '.pdf'))

    def plot_hist(self, area, layer):
        fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(5, 5))
        for pop in self.pops:
            for version in self.versions:

                ax.hist(self.data[version].loc[area, layer, pop].values,
                        color=self.colors[version],
                        density=True,
                        histtype='step',
                        bins=20,
                        linewidth=2,
                        label=version)

            plt.legend()
            plt.savefig(
                os.path.join(self.save_path, '_'.join([
                    self.quantity, area, layer, pop]) + '.png'),
                dpi=600)

# INTERNAL FUNCTIONS

    def _fetch_sim_hashes(self, version):
        sim_hashes = os.listdir(path=os.path.join(self.analysis_path, version))
        for annoying_macos_dir in ['.DS_Store', '._.DS_Store']:
            while annoying_macos_dir in sim_hashes:
                sim_hashes.remove(annoying_macos_dir)
        return sim_hashes

    def _load_data_mam(self, version):

        sim_hashes = self._fetch_sim_hashes(version)

        sample_data = pd.read_pickle(os.path.join(
            self.analysis_path, version,
            sim_hashes[0],
            '348cd785d210258c0da5cceaee62b897',
            self.quantity + '.pkl')).to_xarray()

        # Dummy aray to initalize x-array for connection probabilities
        dummy_data_array = np.zeros((np.shape(sample_data)[0],
                                     np.shape(sample_data)[1],
                                     np.shape(sample_data)[2],
                                     len(sim_hashes)))

        self.areas = sample_data.coords['area'].values
        self.layers = sample_data.coords['layer'].values
        self.pops = sample_data.coords['pop'].values

        data = xr.DataArray(
            dummy_data_array,
            coords={'area': self.areas,
                    'layer': self.layers,
                    'pop': self.pops,
                    'sim_hash': sim_hashes},
            dims=['area',
                  'layer',
                  'pop',
                  'sim_hash'])

        for sim_hash in sim_hashes:
            data.loc[:, :, :, sim_hash] = pd.read_pickle(
                os.path.join(
                    self.analysis_path,
                    version,
                    sim_hash,
                    '348cd785d210258c0da5cceaee62b897',
                    self.quantity + '.pkl')).to_xarray()
        return data

    def _load_data_4x4(self, version):
        with open(os.path.join(
            self.analysis_path, f'seed_comparison_nest_{version}.txt')) as f:
            dic = json.load(f)

        with open(os.path.join(self.analysis_path, 'psview_dict.txt')) as f:
            psview = json.load(f)
            master_seeds = psview['seed_mesocircuit']['custom_params'][
                                  'ranges']['sim_dict']['master_seed']

        self.areas = [''] # only one exists
        self.layers = ['L23', 'L4', 'L5', 'L6']
        self.pops = ['E', 'I']
        num_samples = len(dic['FRs']['mean']['L23E']['values'])

        data_array = np.zeros(
            shape=(1, len(self.layers), len(self.pops), num_samples))

        for l, layer in enumerate(self.layers):
            for p, pop in enumerate(self.pops):
                # take mean value per population
                data_array[0, l, p] = \
                    dic[self.quantity]['mean'][layer + pop]['values']

        data = xr.DataArray(
            data_array,
            coords={'area': self.areas,
                    'layer': self.layers,
                    'pop': self.pops,
                    'master_seeds': master_seeds},
            dims=['area',
                  'layer',
                  'pop',
                  'master_seeds'])
        return data

    def _calc_ks_score(self):
        # Dummy array to initalize x-array for connection probabilities
        dummy_data_array = np.zeros((len(self.areas),
                                     len(self.layers),
                                     len(self.pops)))

        ks_score = xr.DataArray(
            dummy_data_array,
            coords={'area': self.areas,
                    'layer': self.layers,
                    'pop': self.pops},
            dims=['area',
                  'layer',
                  'pop'])

        for area in self.areas:
            for layer in self.layers:
                for pop in self.pops:
                    ks_score.loc[area, layer, pop] = sp.stats.kstest(
                        self.data[self.versions[0]].loc[area, layer, pop],
                        self.data[self.versions[1]].loc[area, layer, pop]).statistic
        return ks_score

    def _calc_p_val(self, stest):
        # Dummy aray to initalize x-array for connection probabilities
        dummy_data_array = np.zeros((len(self.areas),
                                     len(self.layers),
                                     len(self.pops)))

        pval = xr.DataArray(
            dummy_data_array,
            coords={'area': self.areas,
                    'layer': self.layers,
                    'pop': self.pops},
            dims=['area',
                  'layer',
                  'pop'])

        for area in self.areas:
            for layer in self.layers:
                for pop in self.pops:
                    x = self.data[self.versions[0]].loc[area, layer, pop]
                    y = self.data[self.versions[1]].loc[area, layer, pop]

                    if np.all(np.isfinite(x)) and np.all(np.isfinite(y)):
                        try:
                            pval.loc[area, layer, pop] = stest(x, y).pvalue
                        except np.linalg.LinAlgError:
                            pval.loc[area, layer, pop] = np.nan
                    else:
                        pval.loc[area, layer, pop] = np.nan

        return pval

    def _plot_ks_score(self, ks_score):
        for pop in self.pops:
            fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(5, 5))

            sns.heatmap(ks_score.loc[:, :, pop])
            ax.set_xticklabels(self.layers)
            ax.set_yticklabels(self.areas[::2], rotation=0)
            plt.savefig(
                os.path.join(
                    self.save_path,
                    '_'.join([self.quantity, 'ks_score', pop]) + '.png'),
                dpi=600)

    def _plot_p_val(self, p_val, test_name, qname, qix, gs, fig):
        for pix, pop in enumerate(self.pops):
            ax = fig.add_subplot(gs[qix, pix])

            im = ax.pcolormesh(p_val.loc[:, :, pop].T,
                        cmap='plasma_r',
                        norm=colors.BoundaryNorm(boundaries=[0.005, 0.01, 0.05, 0.1, 1],
                                                 ncolors=256, extend='min'))
            if pix == 0:
                ax.set_yticks(0.5 + np.arange(0, len(self.layers)))
                ax.set_yticklabels(self.layers)
            else:
                ax.set_yticks([])
            if qix == 2:
                ax.set_xticks(range(0, len(self.areas), 6))
                ax.set_xticklabels(self.areas[::6], rotation=0)
            else:
                ax.set_xticks([])
            ax.set_title(f'{qname} ({pop}-pop): {test_name}-test')

        return im

if __name__ == '__main__':

    model_specs = [('mam', 'Multi-Area Model',
                    ['2.20.1', 'master-rng'],
                    ['rates', 'cv_isi', 'cc']),
                   ('4x4', '4x4mm2 Model',
                    ['feature_cpp11-librandom_14211b8df', 'master_f1c0bcf43'],
                    ['FRs', 'LVs', 'CCs'])]
    for modshort, modlong, versions, quantities in model_specs:
        for statshort, statlong, func in [('KS', 'Kolmogorov-Smirnov', sp.stats.kstest),
                                  ('ES', 'Epps-Singleton', sp.stats.epps_singleton_2samp)]:
            fig = plt.figure(figsize=(7, 8))
            fig.suptitle(f'{modlong} / {statlong} Test')
            gs = GridSpec(4, 2, width_ratios=[1, 1], height_ratios=[4, 4, 4, 1])

            for ix, quantity in enumerate(quantities):
                validate = Validate(
                    analysis_path=f'./data/{modshort}',
                    quantity=quantity,
                    model=modshort,
                    versions=versions)
                validate.load_data()
                ax = validate.plot_p_val(func, statshort, quantity, ix, gs, fig)
            ax_cb = fig.add_subplot(gs[-1, :])
            plt.colorbar(ax, cax=ax_cb, orientation='horizontal', label='p-value', shrink=0.5)

            plt.savefig(os.path.join(validate.save_path, f'rngtest_{modshort}_{statshort}-test.pdf'))
            plt.savefig(os.path.join(validate.save_path, f'rngtest_{modshort}_{statshort}-test.png'))

    plt.show()

    # validate.plot_hist('V1','4','I')
    # validate.plot_qq('V1','4')


import numpy as np
import xlrd
from sys import path
path.append(r'd:\\MyDocument\\Study_makes_me_happy\\ManchineLearning\\libsvm\\python')
from svmutil import *

DATA_FILE="dataset.xlsx"
book = xlrd.open_workbook(DATA_FILE, encoding_override='utf-8')
sheet = book.sheet_by_index(0)
X = np.zeros(shape=(8,17))
Y = np.zeros(shape=(17,))
for i in range(22,39):
    for j in range(15,23):
        X[j - 15][i - 22] = sheet.cell_value(i,j)
for i in range(22,39):
    if sheet.cell_value(i,23)==1:
        Y[i - 22] = 1
    else:
        Y[i - 22] = -1
X_train = X[:,3:17]
Y_train = Y[3:17]
X_test = X[:,0:3]
Y_test = Y[0:3]
prob  = svm_problem(Y_train, X_train.T)
param = svm_parameter('-s 1 -t 0 -b 1')
m = svm_train(prob, param)
p_label, p_acc, p_val = svm_predict(Y_test, X_test.T, m, '-b 1')


# -*- coding: utf-8 -*-

# mui sample app

import sys
import asyncio
import time

from datetime import timezone, timedelta

from threading import Lock
mutex = Lock()

from mui_ui import Display, MuiFont, Text, Image, Widget, Border, AbsApp, Message, DigitalClock
from mui_ui import TextAlignment, MotionEvent, InputEvent, InputEventListener, InputHandler, OnTouchEventListener, AppEventListener, OnUpdateRequestListener
from mui_ui import GestureListener, GestureDetector
from mui_ui import Keyboard, KeyboardListener
from mui_ui import DisplayManager, DisplayEventListener

from app_wifisetting import WiFiSetting

import os
dir = os.path.dirname(os.path.abspath(__file__))

def get_file_path(name):
    return os.path.normpath(os.path.join(dir, name))

class ImageViewerApp(AbsApp, OnUpdateRequestListener, OnTouchEventListener):

    # please add images to this list.
    # when tap display, change to next image of this list.
    #
    # image requirements :
    # - width 200px
    # - height 32px
    # - PNG Image
    # - RGB format
    # - black pixel is turn on LED dot
    IMAGE_LIST = [
        './images/mount_fuji.png',
        './images/kyoto_arashiyama.png',
        './images/moon.png',
    ]

    def __init__(self, appEventListener: AppEventListener, lang='en-US'):
        super().__init__(appEventListener)

        self._lang = lang

        # create Image view
        # this view is fullscreen size
        def createImageView():
            widget = Widget(200, 32)
            self.addView(widget)

            imageView = Image(get_file_path(ImageViewerApp.IMAGE_LIST[self._image_index]))
            imageView.setSize(0, 0, 200, 32)
            imageView.addOnTouchViewListener(self)
            widget.addParts(imageView)
            self.setView(imageView, 'image_view')

        self._image_index = 0
        createImageView()

        self._next = None
        self._prev = None

    def setRelativeApps(self, next: AbsApp, prev: AbsApp):
        self._next = next
        self._prev = prev

    def _changeToNextImage(self):
        self._image_index += 1
        if self._image_index == len(ImageViewerApp.IMAGE_LIST):
            self._image_index = 0

        imageView = self.getView('image_view')
        imageView.setImage(get_file_path(ImageViewerApp.IMAGE_LIST[self._image_index]))
        self.updateRequest(0)

    # ------------------------
    # override AbsApp methods

    def startTask(self):
        # here is called when start this app
        pass

    def stopTask(self):
        pass

    def dispatchFlingEvent(self, e1, e2, x, y):
        if e1.x > e2.x:
            # next
            self.setNextApp(self._next)

        elif e1.x < e2.x:
            # prev
            self.setNextApp(self._prev)

        return True

    def onTurnOffDisplay(self):
        # allow turn off
        # if you do not want to allow auto turn off, please return False
        return True

    # ------------------------
    # OnUpdateRequestListener implementation

    def onUpdateView(self, view):
        self.updateRequest(0)

    # ------------------------
    # OnTouchEventListener implementation

    def onTouch(self, view, e):
        imageView = self.getView('image_view')

        if view == imageView:
            # change to next image
            self._changeToNextImage()

class DummyWeatherApp(AbsApp, OnUpdateRequestListener, OnTouchEventListener):

    def __init__(self, appEventListener: AppEventListener, lang='en-US'):
        super().__init__(appEventListener)

        self._lang = lang

        # create weather forecast UI
        def createWeatherForecastUI():
            widget = Widget(189, 32)
            widget.visible = False
            self.addView(widget)
            self.setView(widget, 'forecast_view')

            dateText = Text()
            dateText.setSize(9, 3, 80, 8)
            widget.addParts(dateText)
            self.setView(dateText, 'dateText')

            wIcon = Image()
            wIcon.setSize(15, 18, 26, 14)
            widget.addParts(wIcon)
            self.setView(wIcon, 'wIcon')

            highTemp = Text()
            highTemp.setSize(85, 3, 35, 8)
            widget.addParts(highTemp)
            self.setView(highTemp, 'hTemp')

            tempIconName = './assets/weather/icon_f.png' if self._lang == 'en-US' else './assets/weather/icon_c.png'
            tempIconPath = os.path.normpath(os.path.join(dir, tempIconName))

            hTempIcon = Image(tempIconPath)
            hTempIcon.setSize(98, 3, 9, 7)
            widget.addParts(hTempIcon)
            self.setView(hTempIcon, 'hTempIcon')

            lowTemp = Text()
            lowTemp.setSize(85, 21, 35, 8)
            widget.addParts(lowTemp)
            self.setView(lowTemp, 'lTemp')

            lTempIcon = Image(tempIconPath)
            lTempIcon.setSize(98, 21, 9, 7)
            widget.addParts(lTempIcon)
            self.setView(lTempIcon, 'lTempIcon')

            pIcon = Image()
            pIcon.setSize(134, 3, 9, 10)
            widget.addParts(pIcon)
            self.setView(pIcon, 'pIcon')

            pText = Text()
            pText.setSize(148, 3, 32, 8)
            widget.addParts(pText)
            self.setView(pText, 'pText')

            percentIconName = './assets/weather/icon_percent.png'
            percentIconPath = os.path.normpath(os.path.join(dir, percentIconName))

            percentIcon = Image(percentIconPath)
            percentIcon.setSize(160, 3, 3, 7)
            widget.addParts(percentIcon)
            self.setView(percentIcon, 'percentIcon')

            widget.visible = False

        # common area
        def createCommonMenu():
            # add home icon
            iconHome = Image(os.path.normpath(os.path.join(dir, './assets/icon_home.png')))
            iconHome.setSize(190, 0, iconHome.width, iconHome.height)
            iconHome.addOnTouchViewListener(self)
            self.addView(iconHome)
            self.setView(iconHome, 'home')

            # add current icon(this icon has no action)
            iconWeather = Image(os.path.normpath(os.path.join(dir, './assets/icon_weather.png')))
            iconWeather.setSize(191, 23, iconWeather.width, iconWeather.height)
            self.addView(iconWeather)

        createWeatherForecastUI()
        createCommonMenu()

        self._next = None
        self._prev = None

    def setRelativeApps(self, next: AbsApp, prev: AbsApp):
        self._next = next
        self._prev = prev

    def _setupDummyData(self):
        """
        set dummy data
        """
        self.getView('dateText').setText('21 July Sun.')
        self.getView('wIcon').setImage(os.path.normpath(os.path.join(dir, './assets/weather/icon_weather_101.png')))
        self.getView('hTemp').setText('28')
        self.getView('lTemp').setText('12')
        self.getView('pIcon').setImage(os.path.normpath(os.path.join(dir, './assets/weather/icon_precip_10.png')))
        self.getView('pText').setText('10')

        self.getView('forecast_view').visible = True
        self.updateRequest(2)

    # ------------------------
    # override AbsApp methods

    def startTask(self):
        self._setupDummyData()

    def stopTask(self):
        self.getView('forecast_view').visible = False

    def dispatchFlingEvent(self, e1, e2, x, y):
        if e1.x > e2.x:
            # next
            self.setNextApp(self._next)

        elif e1.x < e2.x:
            # prev
            self.setNextApp(self._prev)

        return True

    def dispatchLongPressEvent(self, e):
        pass

    def onTurnOffDisplay(self):
        # always allow turn off
        return True

    # ------------------------
    # OnUpdateRequestListener implementation

    def onUpdateView(self, view):
        self.updateRequest(0)

    # ------------------------
    # OnTouchEventListener implementation

    def onTouch(self, view, e):
        iconHome = self.getView('home')

        if view == iconHome:
            # close this application
            self.close()

class DummyThermoApp(AbsApp, OnUpdateRequestListener, OnTouchEventListener):

    def __init__(self, appEventListener: AppEventListener, lang='en-US'):
        super().__init__(appEventListener)

        dummy = Text('dummy thermostat UI')
        dummy.setSize(0, 11, 189, 8)
        self.addView(dummy)

        # common area
        def createCommonMenu():
            # add home icon
            iconHome = Image(os.path.normpath(os.path.join(dir, './assets/icon_home.png')))
            iconHome.setSize(190, 0, iconHome.width, iconHome.height)
            iconHome.addOnTouchViewListener(self)
            self.addView(iconHome)
            self.setView(iconHome, 'home')

            # add current icon(this icon has no action)
            iconWeather = Image(os.path.normpath(os.path.join(dir, './assets/icon_thermo.png')))
            iconWeather.setSize(191, 23, iconWeather.width, iconWeather.height)
            self.addView(iconWeather)

        createCommonMenu()

        self._next = None
        self._prev = None

    def setRelativeApps(self, next: AbsApp, prev: AbsApp):
        self._next = next
        self._prev = prev

    # ------------------------
    # override AbsApp methods

    def startTask(self):
        pass

    def stopTask(self):
        pass

    def dispatchFlingEvent(self, e1, e2, x, y):
        if e1.x > e2.x:
            # next
            self.setNextApp(self._next)

        elif e1.x < e2.x:
            # prev
            self.setNextApp(self._prev)

        return True

    def dispatchLongPressEvent(self, e):
        pass

    def onTurnOffDisplay(self):
        # always allow turn off
        return True

    # ------------------------
    # OnUpdateRequestListener implementation

    def onUpdateView(self, view):
        self.updateRequest(0)

    # ------------------------
    # OnTouchEventListener implementation

    def onTouch(self, view, e):
        iconHome = self.getView('home')

        if view == iconHome:
            # close this application
            self.close()

class HomeApp(AbsApp, OnUpdateRequestListener, OnTouchEventListener):

    def __init__(self, appEventListener: AppEventListener):
        super().__init__(appEventListener)

        # create home UI

        # add clock
        def addClock():
            tz = timezone(timedelta(hours=9)) # this timezone is JST(+9:00). please change to your timezone
            clock = DigitalClock(timezone=tz)
            clock.setPos(173, 0)
            clock.addOnUpdateViewListener(self)
            self.addView(clock)
            self.setView(clock, 'clock')

        # add menu
        def createMenu():
            # create weahter app icon
            iconWeather = Image(get_file_path('./assets/icon_weather.png'))
            iconWeather.addOnTouchViewListener(self)
            self.setView(iconWeather, 'icon_weather')

            # create thermostat app icon
            iconThermo = Image(get_file_path('./assets/icon_thermo.png'))
            iconThermo.addOnTouchViewListener(self)
            self.setView(iconThermo, 'icon_thermo')

            # create image viewer app icon
            iconImage = Image(get_file_path('./assets/icon_pen.png'))
            iconImage.addOnTouchViewListener(self)
            self.setView(iconImage, 'icon_image')

            # create wi-fi setting app icon
            iconWiFi = Image(get_file_path('./assets/icon_wifi_enable.png'))
            iconWiFi.addOnTouchViewListener(self)
            self.setView(iconWiFi, 'icon_wifi')

            # create menu widget
            menu = Widget(
                (iconWeather.width + iconThermo.width + 2 + iconImage.width + 2),
                22)
            menu.setPos(0, 11)
            self.addView(menu)

            # add icons to menu widget
            menu.addParts(iconWeather)

            iconThermo.x = iconWeather.width + 2
            menu.addParts(iconThermo)

            iconImage.x = iconThermo.x + iconThermo.width + 2
            menu.addParts(iconImage)

            iconWiFi.x = iconWeather.width + 2
            iconWiFi.y = 11
            menu.addParts(iconWiFi)

        # create application
        def createApplication():
            weatherApp = DummyWeatherApp(self.appEventListener)
            thermoApp = DummyThermoApp(self.appEventListener)
            imageApp = ImageViewerApp(self.appEventListener)
            wifiApp = WiFiSetting(self.appEventListener, lang='en-US') # if you want to change English, please change to 'en-US'

            weatherApp.setRelativeApps(next=thermoApp, prev=imageApp)
            thermoApp.setRelativeApps(next=imageApp, prev=weatherApp)
            imageApp.setRelativeApps(next=weatherApp, prev=thermoApp)

            self._apps = {}
            self._apps['weather'] = weatherApp
            self._apps['thermo'] = thermoApp
            self._apps['image'] = imageApp
            self._apps['wifi'] = wifiApp

        def addHomeIcon():
            iconHome = Image(os.path.normpath(os.path.join(dir, './assets/icon_home.png')))
            iconHome.setSize(190, 23, iconHome.width, iconHome.height)
            self.addView(iconHome)

        addClock()
        createMenu()
        createApplication()
        addHomeIcon()

    # ------------------------
    # override AbsApp methods

    def startTask(self):
        # start clock update
        self.getView('clock').startTick()

    def stopTask(self):
        # stop clock update
        self.getView('clock').stopTick()

    def dispatchFlingEvent(self, e1, e2, x, y):
        return False

    def dispatchLongPressEvent(self, e):
        pass

    def onTurnOffDisplay(self):
        # always allow turn off
        return True

    # ------------------------
    # OnUpdateRequestListener implementation

    def onUpdateView(self, view):
        self.updateRequest(0)

    # ------------------------
    # OnTouchEventListener implementation

    def onTouch(self, view, e):
        iconWeather = self.getView('icon_weather')
        iconThermo = self.getView('icon_thermo')
        iconImage = self.getView('icon_image')
        iconWiFi = self.getView('icon_wifi')

        if view == iconWeather:
            next = self._apps['weather']

        elif view == iconThermo:
            next = self._apps['thermo']

        elif view == iconImage:
            next = self._apps['image']

        elif view == iconWiFi:
            next = self._apps['wifi']

        if next is not None:
            self.setNextApp(next)

class MuiMain(InputEventListener, GestureListener, AppEventListener, DisplayEventListener):
    """
    Main class
    this class connect to mui's hardware(display and touch panel) and control applications transition.
    """

    def __init__(self):
        # connect to mui touchpanel
        self.input = InputHandler(self)

        # create gesture detector(for swipe)
        self.gesture_detector = GestureDetector(listener=self, longpress_timeout=2)

        # connect to mui display
        self.display = Display()
        # set birghtness level
        self.display.setDuty(100)
        # clear and turn on display
        self.display.clearDisplay()
        self.display.turnOn(0)

        # create display manager(for auto turn off display)
        # after 15 seconds from last user touch to mui, turn off display.
        self.display_manager = DisplayManager(self, time_to_dismiss=15)

        # create base application and set it to current application
        self.app = self._baseApp = HomeApp(appEventListener = self)

    def main_loop(self):
        # start display manager check
        self.display_manager.startDismissTimer()
        # start capture touch event
        self.input.startEventLoop()

    def startTask(self):
        self.app.startTask()

    def updateUI(self, fade=0, turn_on=False, turn_off=False):
        with mutex:
            if (fade > 0) and (turn_on is False):
                # fade out (1 - 4 : 4 is very slow)
                self.display.turnOff(fade)

            if (turn_on is False) and (turn_off is False):
                # set layout data
                self.display.setLayout(self.app.getUI())
                # update Display internal data buffer (do not refesh display until call refreshDisplay())
                self.display.updateLayout()
                # refresh Display
                self.display.refreshDisplay()

            if (fade > 0) and (turn_off is False):
                # fade in
                self.display.turnOn(fade)

    # ------------------------
    # InputEventListener implementation

    def onInputEvent(self, e: MotionEvent):
        if self.display_manager.on is False:
            if e.action == 0:
                self.display_manager.on = True
                self.updateUI(fade=2, turn_on=True)
                self.display_manager.startDismissTimer()
            return

        # pass to gesture detector
        handle = self.gesture_detector.onTouchEvent(e)

        # dispatch touch event to current application
        if handle is False:
            self.app.dispatchTouchEvent(e)

        # reset display turn off timer
        self.display_manager.startDismissTimer()

    # ------------------------
    # AppEventListener implementation

    def requestUpdateDisplay(self, app, fade):
        self.updateUI(fade)

    def setNextApp(self, app):
        self.display_manager.startDismissTimer()

        # stop old application task
        self.app.stopTask()

        # change current application
        self.app = app
        # update UI
        self.updateUI(2)
        # start new application task
        self.app.startTask()

    def onCloseApp(self, app):
        self.display_manager.startDismissTimer()

        # close current application, so return to base application
        self.app = self._baseApp
        self.updateUI(2)
        self.app.startTask()

    def onChangeDuty(self, duty):
        # request brightness change to display class
        self.display.setDuty(duty)

    # ------------------------
    # DisplayEventListener implementation

    def onDismissTime(self) -> bool:
        # check app allow turn off
        return self.app.onTurnOffDisplay()

    def onDismiss(self):
        # turn off display with fade effect
        self.updateUI(fade=3, turn_off=True)

    # ------------------------
    # GestureListener implementation

    def onFling(self, e1: MotionEvent, e2: MotionEvent, x, y):
        # swipe event occured, pass to current application
        return self.app.dispatchFlingEvent(e1, e2, x, y)

    def onLongPress(self, e: MotionEvent):
        # long press event occured, pass to current application
        self.app.dispatchLongPressEvent(e)

if __name__ == "__main__":
    try:
        app = MuiMain()

        # draw UI
        app.updateUI(fade=3)

        # kick appliction task
        app.startTask()

        # start capture touch panel event
        app.main_loop()

    except (KeyboardInterrupt, EOFError):
        pass

    sys.exit(0)


import pymysql

db = pymysql.connect('localhost', 'root', 'root', 'dada')
cursor = db.cursor()
name = "update"
id = 1
sql = """
    DELETE FROM py_user WHERE id = %d;
""" % (id)
try:
    cursor.execute(sql)
    db.commit()
    print("delete successful")
except:
    print("delete error")
    db.rollback()

finally:
    db.close()


#!/usr/bin/env python

import sys
from queue import Queue
from collections import namedtuple

# Struct for edges.
Edge = namedtuple('Edge', ['src', 'dest'])

# https://docs.python.org/2/library/userdict.html
class Graph(object):

    def __init__(self, number=4):
        self.adj = {}
        print ("Inside graph creation...")
        for i in range (1,number+1):
            self.adj[i] = []
        print("Graph: ", self.adj)

    def build_graph(self, M):
        while (M > 0):
            x, y = read_ints()
            edge = Edge(x,y)
            self.adj[edge.src].append(edge.dest)  # node_edges[x].append(y)
            self.adj[edge.dest].append(edge.src)  # node_edges[y].append(x)
            M -= 1
        print ("Built graph: ", self.adj)

    def compute_distances(self, S):
        distances = {S: 0}  # distance to itself is obviously zero!
        queue = myQueue()
        queue.put (S)   #  queue.put(S)
        while (not queue.empty()):
            print ("Queue: ", queue)
            element = queue.get()
            distance = distances[element] + 6    # varies based on levels, 1st level = 0 + 6
            for neighbor in self.adj[element]:   # iterate through the neighbours
                if (neighbor in distances):
                    continue                     # ignore nodes in a cycles?
                distances[neighbor] = distance   # update distance to starting node
                print ("Distances: ", distances)
                queue.put(neighbor)              # so its neighbours get traversed in the while loop
        print ("Distances: ", distances)
        return distances

def read_ints():
    return [int(x) for x in sys.stdin.readline().split(" ")]

class myQueue(Queue):
    def __str__(self):
        return self.queue.__str__()

def print_distances(S, N, distances):
    for i in range(1, N + 1):
        if (i == S):
            continue  # to mute the printing of distance (aka 0) to itself
        if i in distances:
            print(distances[i], end=" "),
        else:
            print(-1, end=" "),
    print()

def test_case():
    (N, M) = read_ints()
    graph = Graph(N)               # with placeholders for N vertices
    graph.build_graph(M)           # process M edges as (src, dest)
    S = int(sys.stdin.readline())  # Get the Starting node for which distances will be calculated
    distances = graph.compute_distances(S)
    print_distances(S, N, distances)

def main():
    T = int(sys.stdin.readline())
    while (T > 0):
        test_case()
        T -= 1

if __name__ == '__main__':
    main()


from requests import Session
from weixin.config import *
from weixin.db import RedisQueue
from weixin.mysql import MySQL
from weixin.request import WeixinRequest
from urllib.parse import urlencode
import requests
from pyquery import PyQuery as pq
from requests import ReadTimeout,ConnectionError

class Spider():
    base_url='http://weixin.sogou.com/weixin'
    keyword='Python'
    headers = {
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
        'Accept-Encoding': 'gzip, deflate',
        'Accept-Language': 'zh-CN,zh;q=0.8,en;q=0.6,ja;q=0.4,zh-TW;q=0.2,mt;q=0.2',
        'Cache-Control': 'max-age=0',
        'Connection': 'keep-alive',
        'Cookie': 'sw_uuid=3609245170; ssuid=6984416384; IPLOC=CN3205; SUID=42E4F03A2208990A000000005CAACCB1; '
                  'SUV=0077C7C03AF0E4425CAACCBA3EAA1002; sg_uuid=4552023396; pgv_pvi=2560417792; '
                  'ld=5kllllllll2t$7hflllllVhmZ1GlllllnsR6yklllx6llllljZlll5@@@@@@@@@@; LSTMV=1233%2C730; LCLKINT=3357;'
                  ' ABTEST=7|1555656250|v1; SNUID=07A2B57C4540C099347CF15446DF5250; weixinIndexVisited=1; sct=28; '
                  'JSESSIONID=aaaEN5dN5s469asWKmQOw; '
                  'ppinf=5|1555657086|1556866686|dHJ1c3Q6MToxfGNsaWVudGlkOjQ6MjAxN3x1bmlxbmFtZTo3MjolRTQlQjglODAlRTQlQj'
                  'glQUElRTQlQjglOEQlRTQlQkYlOTclRTYlQjAlOTQlRTclOUElODQlRTYlOTglQjUlRTclQTclQjB8Y3J0OjEwOjE1NTU2NTcwODZ'
                  '8cmVmbmljazo3MjolRTQlQjglODAlRTQlQjglQUElRTQlQjglOEQlRTQlQkYlOTclRTYlQjAlOTQlRTclOUElODQlRTYlOTglQjUlR'
                  'TclQTclQjB8dXNlcmlkOjQ0Om85dDJsdU1uRGNFMkVZQ2FMdElITGU2MlZiOXdAd2VpeGluLnNvaHUuY29tfA; '
                  'pprdig=C9hb1VEIXL_gj8VuGmwYAWeDaqQwG5sz6aifKnY1z61mCwAiAliOr_i0ttUjeP0YQy4u24ea3W4YlciR_pOkna68yh'
                  'R-uVUZqG41lBUan5t34Qtq8InIhLWmrnR_HOhTNbvKWFtpS86DEg2E6CZO2zrf-6Sivzeu3l9ZDsRMV7M; '
                  'sgid=00-45162031-AVy5cX5f8XYsibaupx6TZIiak; ppmdig=1555663237000000b769348c00cc240f380a45206d091d03',
        'Host': 'weixin.sogou.com',
        'Upgrade-Insecure-Requests': '1',
        'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36'
    }
    session=Session()
    queue=RedisQueue()
    mysql=MySQL()

    def get_proxy(self):
        """
        从代理池获取代理
        :return:
        """
        try:
            response=requests.get(PROXY_POOL_URL)
            if response.status_code==200:
                print('Get Proxy',response.text)
                return response.text
        except requests.ConnectionError:
            return None

    def start(self):
        """
        初始化工作
        :return:
        """
        # 全局更新Headers
        self.session.headers.update(self.headers)
        start_url=self.base_url+'?'+urlencode({'query':self.keyword,'type':2})
        weixin_request=WeixinRequest(url=start_url,callback=self.parse_index,need_proxy=True)

        # 调度第一个请求
        self.queue.add(weixin_request)

    def parse_index(self,response):
        """
        解析索引页
        :param response: 响应
        :return: 新的响应
        """
        doc=pq(response.text)
        items=doc('.news-box .news-list li .txt-box h3 a').items()
        for item in items:
            url=item.attr('href')
            weixin_request=WeixinRequest(url=url,callback=self.parse_detail)
            yield weixin_request
        next=doc('#sogou_next').attr('href')
        if next:
            url=self.base_url+str(next)
            weixin_request=WeixinRequest(url=url,callback=self.parse_index,need_proxy=True)
            yield weixin_request

    def parse_detail(self,response):
        """
        解析详情页
        :param response:
        :return:
        """
        doc=pq(response.text)
        data = {
            'title': doc('.rich_media_title').text(),
            'content': doc('.rich_media_content').text(),
            'date': doc('#post-date').text(),
            'nickname': doc('#js_profile_qrcode > div > strong').text(),
            'wechat': doc('#js_profile_qrcode > div > p:nth-child(3) > span').text()
        }
        yield data

    def request(self,weixin_request):
        """
        执行请求
        :param weixin_request: 请求
        :return: 响应
        """
        try:
            if weixin_request.need_proxy:
                proxy=self.get_proxy()
                if proxy:
                    proxies={
                        'http':'http://'+proxy,
                        'https':'https://'+proxy
                    }
                    return  self.session.send(weixin_request.prepare())
        except:
            pass