import random
from typing import List, Tuple

import torch
from torch import nn, optim

from nlg_analysis.cfg import TrainConfig
from nlg_analysis.models.base_model import BaseModel
from nlg_analysis.models.conversation_side import ConversationSide
from nlg_analysis.models.seq2seq_decoder import Seq2SeqDecoder
from nlg_analysis.models.seq2seq_encoder import Seq2SeqEncoder
from nlg_analysis.models.seq2seq_utils import (
    prepareData,
    tensorFromSentence,
    tensorsFromPair,
)

class Seq2SeqModel(BaseModel):
    """Model based on encoder decoder architecture."""

    def __init__(
        self,
        questions_path: str,
        answers_path: str,
        path2encoder: str = None,
        path2decoder: str = None,
    ) -> None:
        super(Seq2SeqModel, self).__init__()
        self.sos_token = 0
        self.eos_token = 1
        self.questions_path = questions_path
        self.answers_path = answers_path
        if path2encoder is not None:
            self.path2encoder = path2encoder
        else:
            self.path2encoder = None
        if path2decoder is not None:
            self.path2decoder = path2decoder
        else:
            self.path2decoder = None

    def generate_transcript(self, questions: List) -> str:
        """Generate conversation transcript."""
        if self.path2decoder is None and self.path2encoder is None:
            raise ValueError
        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
        hidden_size = 256
        question_side, answer_side, pairs = prepareData(
            self.questions_path, self.answers_path
        )
        encoder = Seq2SeqEncoder(
            question_side.n_words, hidden_size, device=device
        )
        encoder.load_state_dict(torch.load(self.path2encoder))
        decoder = Seq2SeqDecoder(
            hidden_size, answer_side.n_words, dropout_p=0.1, device=device
        )
        decoder.load_state_dict(torch.load(self.path2decoder))

        trans_txt = ""
        for q in questions:
            output_words, attentions = self.evaluate(
                encoder=encoder,
                decoder=decoder,
                sentence=q,
                device=device,
                answer_side=answer_side,
                question_side=question_side,
            )
            trans_txt += q + "\n" + " ".join(output_words[:-1]) + "\n"
        return trans_txt

    @staticmethod
    def approach() -> str:
        """Return the name of implemented approach."""
        return "Seq2Seq"

    def train(
        self,
        encoder_output_path: str,
        decoder_output_path: str,
        train_config: TrainConfig,
    ) -> None:
        """Train model and save to given output path."""
        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
        learning_rate = train_config.learning_rate
        hidden_size = train_config.hidden_size
        n_iters = train_config.n_iters
        question_side, answer_side, pairs = prepareData(
            self.questions_path, self.answers_path
        )
        encoder = Seq2SeqEncoder(
            question_side.n_words, hidden_size, device
        ).to(device)
        attn_decoder = Seq2SeqDecoder(
            hidden_size, answer_side.n_words, device, dropout_p=0.1
        ).to(device)

        encoder_optimizer = optim.SGD(encoder.parameters(), lr=learning_rate)
        decoder_optimizer = optim.SGD(
            attn_decoder.parameters(), lr=learning_rate
        )
        training_pairs = [
            tensorsFromPair(
                random.choice(pairs),
                input_side=question_side,
                output_side=answer_side,
                eos_token=self.eos_token,
                device=device,
            )
            for _ in range(n_iters)
        ]
        criterion = nn.NLLLoss()

        for i in range(1, n_iters + 1):
            training_pair = training_pairs[i - 1]
            input_tensor = training_pair[0]
            target_tensor = training_pair[1]

            self.trainIter(
                input_tensor,
                target_tensor,
                encoder,
                attn_decoder,
                encoder_optimizer,
                decoder_optimizer,
                criterion,
                device,
                teacher_forcing_ratio=train_config.teacher_forcing_ratio,
            )

        torch.save(encoder.state_dict(), encoder_output_path)
        torch.save(attn_decoder.state_dict(), decoder_output_path)

    def trainIter(
        self,
        input_tensor: torch.Tensor,
        target_tensor: torch.Tensor,
        encoder: Seq2SeqEncoder,
        decoder: Seq2SeqDecoder,
        encoder_optimizer: torch.optim.Optimizer,
        decoder_optimizer: torch.optim.Optimizer,
        criterion: nn.Module,
        device: torch.device,
        max_length: int = 32,
        teacher_forcing_ratio: float = 0.5,
    ) -> float:
        """Run one epoch."""
        encoder_hidden = encoder.initHidden()

        encoder_optimizer.zero_grad()
        decoder_optimizer.zero_grad()

        input_length = input_tensor.size(0)
        target_length = target_tensor.size(0)

        encoder_outputs = torch.zeros(
            max_length, encoder.hidden_size, device=device
        )

        loss = 0

        for ei in range(input_length):
            encoder_output, encoder_hidden = encoder(
                input_tensor[ei], encoder_hidden
            )
            encoder_outputs[ei] = encoder_output[0, 0]

        decoder_input = torch.tensor([[self.sos_token]], device=device)

        decoder_hidden = encoder_hidden

        use_teacher_forcing = (
            True if random.random() < teacher_forcing_ratio else False
        )

        if use_teacher_forcing:
            # Teacher forcing: Feed the target as the next input
            for di in range(target_length):
                decoder_output, decoder_hidden, decoder_attention = decoder(
                    decoder_input, decoder_hidden, encoder_outputs
                )
                loss += criterion(decoder_output, target_tensor[di])
                decoder_input = target_tensor[di]

        else:
            # Without teacher forcing:
            # use its own predictions as the next input
            for di in range(target_length):
                decoder_output, decoder_hidden, decoder_attention = decoder(
                    decoder_input, decoder_hidden, encoder_outputs
                )
                topv, topi = decoder_output.topk(1)
                decoder_input = (
                    topi.squeeze().detach()
                )  # detach from history as input

                loss += criterion(decoder_output, target_tensor[di])
                if decoder_input.item() == self.eos_token:
                    break

        loss.backward()

        encoder_optimizer.step()
        decoder_optimizer.step()

        return loss.item() / target_length

    def evaluate(
        self,
        encoder: Seq2SeqEncoder,
        decoder: Seq2SeqDecoder,
        sentence: str,
        question_side: ConversationSide,
        answer_side: ConversationSide,
        device: torch.device,
        max_length: int = 32,
    ) -> Tuple[List[str], torch.Tensor]:
        """Create text answer to given query."""
        with torch.no_grad():
            input_tensor = tensorFromSentence(
                question_side,
                sentence,
                eos_token=self.eos_token,
                device=device,
            )
            input_length = input_tensor.size()[0]
            encoder_hidden = encoder.initHidden()

            encoder_outputs = torch.zeros(
                max_length, encoder.hidden_size, device=device
            )

            for ei in range(input_length):
                encoder_output, encoder_hidden = encoder(
                    input_tensor[ei], encoder_hidden
                )
                encoder_outputs[ei] += encoder_output[0, 0]

            decoder_input = torch.tensor([[self.sos_token]], device=device)

            decoder_hidden = encoder_hidden

            decoded_words = []
            decoder_attentions = torch.zeros(max_length, max_length)

            for di in range(max_length):
                decoder_output, decoder_hidden, decoder_attention = decoder(
                    decoder_input, decoder_hidden, encoder_outputs
                )
                decoder_attentions[di] = decoder_attention.data
                topv, topi = decoder_output.data.topk(1)
                if topi.item() == self.eos_token:
                    decoded_words.append("<EOS>")
                    break
                else:
                    decoded_words.append(answer_side.index2word[topi.item()])

                decoder_input = topi.squeeze().detach()

            return decoded_words, decoder_attentions[: di + 1]


import sys
import cv2
import numpy as np

W = 192
H = 230

np.set_printoptions(threshold=np.inf)

def binRet(mImg):
	with open(sys.argv[1], 'rb') as f:
		for i in range(H):
			for j in range(W):
				l = map(lambda x: "%02x" % ord(x), f.read(1))
				if not l: break
				s = "".join(l)
				mImg[i][j] = int(s, 16)
	return mImg

if __name__=="__main__":
	img = np.zeros((H, W, 1), np.uint8)
	img = binRet(img)
	cv2.namedWindow('image')
	cv2.imwrite('mal_Img.png', img)

	while(1):
		cv2.imshow('image', img)
		k = cv2.waitKey(1) & 0xFF
		if k == 27: break

	cv2.destroyAllWindows()


import gym
import environments
import agent
import config.config
import utills
import modules
import torch

class episode_run(object) :

    def __init__(self,config,logger , agent) :
        self.config = config
        self.logger = logger
        self.agent = agent
        self.writer = utills.result.Writer(config)
        self.env = config.env
        self.env_name = config.env_name

        self.logger.print_info('let start %s' % self.env_name)

        self.hyperparameters = config.hyperparameters

        self.batch_size = self.hyperparameters['batch_size']
        self.buffer_size = self.hyperparameters['buffer_size']

        if config.replay :
            if config.replay_buffer == 'replay_buffer' :
                self.memory = modules.memory.replay_buffer(self.batch_size, self.buffer_size)

            if config.replay_buffer == 'per' :
                self.memory = modules.memory.per(self.batch_size, self.buffer_size)

        self.step = 0
        self.episode = 0
        self.Timer = utills.timer.timer()

        self.score = []
        self.episode_lis = []

        self.run()

    def run(self) :

        done = False
        if self.config.evaluate :
            self.test(self.config.test_epi)
            return

        for self.episode in range(self.config.max_epi) :
            state = self.env.reset()
            self.step = 0
            total_reward = 0
            self.logger.reset_stat()
            self.logger.log_stat('episode',self.episode, 0)
            self.Timer.start_episode()
            train_time = 0
            while not done :
                self.step = self.step + 1
                self.env.render()

                if self.config.epsilon :
                    epsilon = self.agent.epsilon(self.episode, self.config.max_epi)
                    action = self.agent.select_action(state,self.episode)
                else :
                    action = self.agent.select_action(state,self.episode)
                next_state, reward, done, info = self.agent.step([action])

                if config.replay :
                    if config.replay_buffer == 'replay_buffer' :
                        self.memory.push(state, action, reward, next_state,done)

                    if config.replay_buffer == 'per' :
                        td_error = self.agent.td_error(state, next_state)
                        self.memory.push(td_error,state, action, reward,next_state, done)

                state = next_state
                total_reward = total_reward + reward

                if self.step % self.config.update_interval == 0 :

                    if config.replay :
                        if len(self.memory) > (self.batch_size  * 2):
                            batch = self.memory.make_batch()
                            loss = self.agent.update(batch)
                            self.logger.log_stat('loss',loss,self.step)
                    else :
                        loss = self.agent.update(state, action, reward, next_state, done)
                        self.logger.log_stat('loss',loss,self.step)

            episode_t = self.Timer.finish_episode()
            train_time = train_time + self.Timer.cal_time(episode_t)

            self.writer.add('score',total_reward,self.episode)

            self.logger.log_stat('episode_len',self.step,0)
            self.logger.log_stat('total_reward',total_reward,0)

            if self.config.test_interval != None or self.episode % self.config.test_interval == 0 :
                self.test(self.config.test_epi)

            if self.episode % self.config.logging_interval == 0 :

                hour,minute,sec = self.Timer.cal_time(episode_t)
                episode_time = 'episode {} : operate {} hours, {} minute, {} seconde'.format(self.episode,hour,minute,sec)
                self.logger.print_info(episode_time)

                left_epi = self.config.max_ep - self.episode
                left_time = left_epi * train_time / self.config.logging_interval
                hour,minute,sec = self.Timer.cal_time(left_time)
                left_time_ = 'left train time : {} hours, {} minute, {} seconde'.format(self.episode,hour,minute,sec)

                self.logger.print_info(left_time_)

                self.logger.print_recent_stats()

        t = self.Timer.finish_train()
        hour,minute,sec = self.Timer.cal_time(t)
        train_time = 'total operate time : {} hours, {} minute, {} seconde'.format(self.episode,hour,minute,sec)
        self.logger.print_info(train_time)

        self.episode_lis.append(self.episode)
        self.score.append(total_reward)

    def test(self,test_epi) :
        pass


import cv2 as cv
import numpy as np
# Get a blank image
blank_img = np.zeros((512, 512, 3), np.uint8)

# Changing Bakground of a Blannk Image
blank_img[:] = (0, 255, 0)

# Draw a line
cv.line(blank_img, (0, 0), (511, 511), (255, 0, 0), 5)

# Draw a rectangle
cv.rectangle(blank_img, (384, 0), (510, 128), (0, 0, 0), 3)

# Draw a circle
cv.circle(blank_img, (447, 63), 63, (0, 0, 255), -1)

# Write text on image
cv.putText(blank_img, 'OpenCV', (10, 500), cv.FONT_HERSHEY_SIMPLEX, 4, (255, 255, 255), 2, cv.LINE_AA)

cv.imshow('Image',blank_img)
cv.waitKey(0)


from typing import Union

import numpy as np

import _alloptical_utils as Utils
from _utils_.io import import_cls

try:
    main = import_cls(pkl_path='/home/pshah/mnt/qnap/Analysis/allopticalseizures/PhotostimResponsesQuantificationSLMtargets.pkl')
except:
    from _analysis_.ClassPhotostimResponseQuantificationSLMtargets import \
        PhotostimResponsesQuantificationSLMtargets as main

from _main_.AllOpticalMain import alloptical
from _main_.Post4apMain import Post4ap
from funcsforprajay import plotting as pplot

"##### -------------------- ALL OPTICAL PHOTOSTIM ANALYSIS #############################################################"

# %% c.0) feb 19- 2022: TRYING NEW CLASS DRIVEN APPROACH FOR EACH ANALYSIS

# @Utils.run_for_loop_across_exps(run_pre4ap_trials=True, run_post4ap_trials=True, allow_rerun=True)
# def delete_old(**kwargs):
#     expobj: Union[alloptical, Post4ap] = kwargs['expobj']
#     delattr(expobj, 'PhotostimResponsesQuantificationSLMtargets')
#     expobj.save()
# delete_old()

# expobj: Post4ap = Utils.import_expobj(prep='RL108', trial='t-013')
# expobj.PhotostimResponsesSLMTargets = main(expobj)
# expobj.PhotostimResponsesSLMTargets.collect_photostim_responses_exp(expobj=expobj)
# expobj.PhotostimResponsesSLMTargets.create_anndata_SLMtargets(expobj=expobj)
# expobj.PhotostimResponsesSLMTargets.plot_photostim_responses_magnitude(expobj=expobj, stims='all')
# expobj.save()

# %%

@Utils.run_for_loop_across_exps(run_pre4ap_trials=1, run_post4ap_trials=1, allow_rerun=0)
def run__initPhotostimResponseQuant(**kwargs):
    expobj: Union[alloptical, Post4ap] = kwargs['expobj']
    expobj.PhotostimResponsesSLMTargets = main(expobj)
    expobj.save()

@Utils.run_for_loop_across_exps(run_pre4ap_trials=1, run_post4ap_trials=1, allow_rerun=0)
def run__collect_photostim_responses_exp(**kwargs):
    expobj: Union[alloptical, Post4ap] = kwargs['expobj']
    expobj.PhotostimResponsesSLMTargets.collect_photostim_responses_exp(expobj=expobj)
    expobj.save()

@Utils.run_for_loop_across_exps(run_pre4ap_trials=1, run_post4ap_trials=1, allow_rerun=0)
def run__create_anndata_SLMtargets(**kwargs):
    expobj: alloptical = kwargs['expobj']
    expobj.PhotostimResponsesSLMTargets.create_anndata_SLMtargets(expobj=expobj)
    expobj.save()

# %% c.1) plotting mean photostim response magnitude
def full_plot_mean_responses_magnitudes():
    """create plot of mean photostim responses magnitudes for all three exp groups"""

    @Utils.run_for_loop_across_exps(run_pre4ap_trials=True, run_post4ap_trials=False, allow_rerun=0)
    def pre4apexps_collect_photostim_responses(**kwargs):
        expobj: alloptical = kwargs['expobj']
        if 'pre' in expobj.exptype:
            # all stims
            mean_photostim_responses = expobj.PhotostimResponsesSLMTargets.collect_photostim_responses_magnitude(
                stims='all')
            return np.mean(mean_photostim_responses)

    mean_photostim_responses_baseline = pre4apexps_collect_photostim_responses()

    @Utils.run_for_loop_across_exps(run_pre4ap_trials=False, run_post4ap_trials=True, allow_rerun=0)
    def post4apexps_collect_photostim_responses(**kwargs):
        expobj: Post4ap = kwargs['expobj']
        if 'post' in expobj.exptype:
            # interictal stims
            mean_photostim_responses_interictal = expobj.PhotostimResponsesSLMTargets.collect_photostim_responses_magnitude(
                stims=expobj.stim_idx_outsz)

            # ictal stims
            mean_photostim_responses_ictal = expobj.PhotostimResponsesSLMTargets.collect_photostim_responses_magnitude(
                stims=expobj.stim_idx_insz)

            return np.mean(mean_photostim_responses_interictal), np.mean(mean_photostim_responses_ictal)

    func_collector = post4apexps_collect_photostim_responses()

    if len(func_collector) > 0:
        mean_photostim_responses_interictal, mean_photostim_responses_ictal = np.asarray(func_collector)[:, 0], np.asarray(
            func_collector)[:, 1]

        return mean_photostim_responses_baseline, mean_photostim_responses_interictal, mean_photostim_responses_ictal

# %% RUN SCRIPT
if __name__ == '__main__':
    run__initPhotostimResponseQuant()
    run__collect_photostim_responses_exp()
    run__create_anndata_SLMtargets()
    main.allexps_plot_photostim_responses_magnitude()

    main.mean_photostim_responses_baseline, main.mean_photostim_responses_interictal, main.mean_photostim_responses_ictal = full_plot_mean_responses_magnitudes()
    main.saveclass()

    pplot.plot_bar_with_points(
        data=[main.mean_photostim_responses_baseline, main.mean_photostim_responses_interictal,
              main.mean_photostim_responses_ictal],
        x_tick_labels=['baseline', 'interictal', 'ictal'], bar=False, colors=['blue', 'green', 'purple'],
        expand_size_x=0.4, title='Average Photostim resopnses', y_label='% dFF (delta(trace(dFF))')


import json
import logging

import unittest.mock as mock
import httpretty
from ddt import data, ddt
from django.conf import settings
from django.test import TestCase
from django.test.utils import override_settings
from requests.exceptions import ConnectionError as RequestsConnectionError
from requests.exceptions import Timeout
from testfixtures import LogCapture
from waffle.testutils import override_flag, override_switch

from analytics_dashboard.courses.tests.test_views import ViewTestMixin
from analytics_dashboard.courses.tests.utils import assert_dict_contains_subset, CourseSamples

@httpretty.activate
@override_flag('display_learner_analytics', active=True)
@ddt
class LearnersViewTests(ViewTestMixin, TestCase):
    TABLE_ERROR_TEXT = 'We are unable to load this table.'
    viewname = 'courses:learners:learners'

    def _register_uris(self, learners_status, learners_payload, course_metadata_status, course_metadata_payload):
        httpretty.register_uri(
            httpretty.GET,
            f'{settings.DATA_API_URL}/learners/',
            body=json.dumps(learners_payload),
            status=learners_status
        )
        httpretty.register_uri(
            httpretty.GET,
            '{data_api_url}/course_learner_metadata/{course_id}/'.format(
                data_api_url=settings.DATA_API_URL,
                course_id=CourseSamples.DEMO_COURSE_ID,
            ),
            body=json.dumps(course_metadata_payload),
            status=course_metadata_status
        )
        self.addCleanup(httpretty.reset)

    def _get(self):
        return self.client.get(self.path(course_id=CourseSamples.DEMO_COURSE_ID))

    def _assert_context(self, response, expected_context_subset):
        default_expected_context_subset = {
            'learner_list_url': '/api/learner_analytics/v0/learners/',
            'course_learner_metadata_url': '/api/learner_analytics/v0/course_learner_metadata/{course_id}/'.format(
                course_id=CourseSamples.DEMO_COURSE_ID
            ),
        }
        assert_dict_contains_subset(response.context, dict(list(expected_context_subset.items())))
        assert_dict_contains_subset(
            response.context['js_data']['course'],
            dict(list(default_expected_context_subset.items())),
        )

    def get_mock_data(self, *args, **kwargs):
        pass

    def test_success(self):
        learners_payload = {'arbitrary_learners_key': ['arbitrary_value_1', 'arbitrary_value_2']}
        course_metadata_payload = {'arbitrary_metadata_value': {'arbitrary_value_1': 'arbitrary_value_2'}}
        self._register_uris(200, learners_payload, 200, course_metadata_payload)
        response = self._get()
        self._assert_context(response, {
            'learner_list_json': learners_payload,
            'course_learner_metadata_json': course_metadata_payload
        })
        self.assertNotContains(response, self.TABLE_ERROR_TEXT)
        return response

    @override_flag('display_learner_analytics', active=False)
    def test_success_if_hidden(self):
        self.test_success()

    @override_switch('enable_learner_download', active=False)
    def test_disable_learner_download_button(self):
        response = self.test_success()
        self.assertNotIn('learner_list_download_url', response.context['js_data']['course'])

    @override_switch('enable_learner_download', active=True)
    @override_settings(LEARNER_API_LIST_DOWNLOAD_FIELDS=None)
    def test_enable_learner_download_button(self):
        response = self.test_success()
        self.assertEqual(response.context['js_data']['course']['learner_list_download_url'],
                         '/api/learner_analytics/v0/learners.csv')

    @override_switch('enable_learner_download', active=True)
    @override_settings(LEARNER_API_LIST_DOWNLOAD_FIELDS='username,email')
    def test_enable_learner_download_button_with_fields(self):
        response = self.test_success()
        self.assertEqual(response.context['js_data']['course']['learner_list_download_url'],
                         '/api/learner_analytics/v0/learners.csv?fields=username%2Cemail')

    @data(Timeout, RequestsConnectionError, ValueError)
    def test_data_api_error(self, RequestExceptionClass):
        learners_payload = {'should_not': 'return this value'}
        course_metadata_payload = learners_payload
        self._register_uris(200, learners_payload, 200, course_metadata_payload)

        # False positive https://github.com/PyCQA/pylint/issues/289
        # pylint: disable=bad-continuation
        with mock.patch(
            'analytics_dashboard.learner_analytics_api.v0.clients.LearnerApiResource.get',
            mock.Mock(side_effect=RequestExceptionClass),
        ):
            with LogCapture(level=logging.ERROR) as lc:
                response = self._get()
                self._assert_context(response, {
                    'learner_list_json': 'Failed to reach the Learner List endpoint',
                    'course_learner_metadata_json': 'Failed to reach the Course Learner Metadata endpoint'
                })
                lc.check(
                    (
                        'analytics_dashboard.courses.views.learners',
                        'ERROR',
                        'Failed to reach the Learner List endpoint',
                    ),
                    (
                        'analytics_dashboard.courses.views.learners',
                        'ERROR',
                        'Failed to reach the Course Learner Metadata endpoint',
                    ),
                )


import sqlalchemy
from datetime import datetime
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship
from ecoalgorithm._db_connect import db
import json
from uuid import uuid4
from collections import OrderedDict
from abc import abstractmethod
from typing import List, Dict, Set
from warnings import warn
from . import _helpers
from random import random

Base = declarative_base()

class SpeciesBase(Base):
    __tablename__ = 'individual'
    __uid = sqlalchemy.Column(sqlalchemy.INTEGER, primary_key=True)
    _guid = sqlalchemy.Column(sqlalchemy.String(36), index=True, nullable=False)
    _gen_num = sqlalchemy.Column(
        sqlalchemy.INTEGER,
        sqlalchemy.ForeignKey('generation.gen_num', ondelete="CASCADE"),
        nullable=False
    )
    _class_name = sqlalchemy.Column(sqlalchemy.String, nullable=False)
    _success = sqlalchemy.Column(sqlalchemy.Float)
    _mother_id = sqlalchemy.Column(sqlalchemy.String(36), index=True)
    _father_id = sqlalchemy.Column(sqlalchemy.String(36), index=True)
    _kwargs = sqlalchemy.Column(sqlalchemy.String, nullable=False)
    _alive = sqlalchemy.Column(sqlalchemy.String(1), nullable=False, default='T')

    _offspring_count = 5

    _validated = False

    __table_args__ = (
        sqlalchemy.Index('ix_gen_success', '_gen_num', '_success'),
        {'sqlite_autoincrement': True},
    )

    @classmethod
    def get_offspring_count(cls) -> int:
        """
        get the number of offspring produced by this species
        :return: the number of offspring returned by breed
        :rtype: int
        """
        return cls._offspring_count

    @classmethod
    def set_offspring_count(cls, new_count: int):
        """
        Set the number of offspring produced by this species
        :param new_count: new offspring count
        :type new_count: int
        :return:
        """
        if new_count < 2:
            raise Exception('must produce 2 or more offspring')
        cls._offspring_count = new_count

    @classmethod
    def validate_class(cls) -> bool:
        """
        Little class method to let the creator of subclasses know that it checks out

        :return:
        """

        if cls._validated:
            return True

        if not issubclass(cls, SpeciesBase):
            raise AssertionError("'{0}' is not subclass of {1}".format(
                cls.__name__, SpeciesBase.__name__
            ))

        ind1 = cls()
        ind2 = cls()

        ind1.mature()
        ind2.mature()

        if not ind1.is_mature or not ind2.is_mature:
            raise AssertionError("'success' property not set in {0} class mature method".format(cls.__name__))

        if not ind1.is_alive:
            ind1.success = 10

        if not ind2.is_alive:
            ind2.success = 10

        progeny = _helpers.breed(ind1, ind2)

        assert len(progeny) == cls._offspring_count

        for p in progeny:
            try:
                assert p.class_name == cls.__name__
            except AssertionError:
                raise AssertionError('mate must return offspring of the same class: {0} returned {1}'.format(
                    cls.__name__, p.class_name
                ))

        p = ind1.params

        try:
            cls(**p)
        except TypeError as ex:
            warn("\nArgument not in constructor for class " + ind1.db_class_name + "\n" + str(ex.args))
            raise ex

        cls_keys = [k for k in ind1.__dict__.keys() if not k.startswith('_')]

        if len(p) != len(cls_keys):
            raise AssertionError("number of __init__ args must equal the number of public species attributes")

        cls._validated = True
        return cls._validated

    def __init__(self):
        """
        The the init method of subclasses is called in two contexts

        First, as one would expect by instantiating a new instance.

        Atlernatively, objects retrieved from the database are always of
        type SpeciesBase.  However, using a class lookup based on the attribute
        from the database,  the class can be altered by means of the following

        species_instance.__class__ = <subclass>
        species instance.__init__()

        The call to init adds the subclass specfic attributes as created
        by the subclass constructor followed by the call to the super
        constructor.

        Note the fall through.

        If the uid is None, that means it is not yet in the database
        checked by property is_in_db

        Initialize the object normally

        If the uid exists, the object is already persisted to the database and
        therefore has the _kwargs column populuated.  These are used to set the
        default random attributes to those such that the instance matches what
        exists in the database.

        Pretty neat!

        """
        if not self.is_in_db:
            self._guid = str(uuid4())
            self._gen_num = None
            self._class_name = self.__class__.__name__
            self._mother_id = None
            self._father_id = None
            self._kwargs = json.dumps(self.params)
            self._success = None
            self._alive = 'T'
            self._success_set = False
        else:
            self._set_attributes(**json.loads(self._kwargs))
            self._success_set = True

    def update_params(self):
        self._kwargs = json.dumps(self.params)

    @abstractmethod
    def mature(self):
        """
        mature the individual
        """
        raise NotImplementedError('mature must be overridden by subclasses')

    def mutate(self):
        """
        optional - apply random mutation
        """
        pass

    @abstractmethod
    def mate(self, other_individual: 'SpeciesBase') -> 'SpeciesBase':
        """

        :param other_individual:
        :type other_individual: self.__class__
        :return:
        :rtype: SpeciesBase
        """
        raise NotImplementedError('mate must be overridden by subclasses')

    def _set_attributes(self, **kwargs: dict):
        """
        set the public attributes given a kwarg dict

        :param kwargs:
        :type kwargs: dict[str, object]
        """
        attributes_set_list = []

        for k, v in kwargs.items():
            if k in self.__dict__ and not k.startswith('_'):
                self.__setattr__(k, v)
                attributes_set_list.append(k)

        public_attributes = [ky for ky in self.__dict__.keys() if not ky.startswith('_')]

        for atr in public_attributes:
            if atr not in public_attributes:
                raise AssertionError("attribute {0} not set from database".format(atr))

    @property
    def guid(self) -> str:
        return self._guid

    @property
    def params(self) -> Dict[str, object]:
        """
        Get the object parameters represented as a dictionary

        :return:
        """

        ordered_keys = [k for k in self.__dict__.keys() if not k.startswith('_')]
        ordered_keys.sort()

        params = OrderedDict()

        for k in ordered_keys:
            params[k] = self.__getattribute__(k)

        return params

    @property
    def _is_base_class(self) -> bool:
        return self.__class__.__name__ == SpeciesBase.__name__

    @property
    def class_name(self) -> str:
        """
        Get the class name

        :return: class name
        :rtype: str
        """
        if self._is_base_class:
            return SpeciesBase.__name__
        else:
            return self._class_name

    @property
    def db_class_name(self) -> str:
        return self._class_name

    @property
    def gen_num(self) -> int or None:
        """

        :return:
        :rtype: int|None
        """

        return self._gen_num

    @gen_num.setter
    def gen_num(self, gen: int):
        """

        :param gen: the generation
        :type gen: int
        :return:
        """

        if self._gen_num is None:
            self._gen_num = gen

    @property
    def success(self) -> float or None:
        return self._success

    @success.setter
    def success(self, success: float or None):
        self._success_set = True
        self._success = success
        self._alive = 'T' if success is not None else 'F'

    @property
    def is_mature(self) -> bool:
        return self._success_set

    @property
    def is_alive(self) -> bool:
        return self._alive == 'T'

    @classmethod
    def get_by_guid(cls, guid: str) -> 'SpeciesBase':
        """

        :param guid:
        :type guid: str|None
        :return:
        :rtype: SpeciesBase
        """
        if guid is None:
            return None

        ind = db.sess.query(cls).filter(cls._guid == guid).first()

        if not ind:
            return None

        ind.__class__ = cls
        ind.__init__()
        return ind

    @property
    def mother(self) -> 'SpeciesBase' or None:
        """

        :return:
        :rtype: SpeciesBase
        """
        return SpeciesBase.get_by_guid(self._mother_id)

    @property
    def father(self) -> 'SpeciesBase' or None:
        """

        :return:
        :rtype: SpeciesBase
        """
        return SpeciesBase.get_by_guid(self._father_id)

    @property
    def is_in_db(self):
        return self.__uid is not None

class Generation(Base):
    __tablename__ = 'generation'
    uid = sqlalchemy.Column(sqlalchemy.INTEGER, primary_key=True, index=True)
    gen_num = sqlalchemy.Column(sqlalchemy.INTEGER, index=True, unique=True, nullable=False)
    gen_time = sqlalchemy.Column(sqlalchemy.DateTime)
    mod_time = sqlalchemy.Column(sqlalchemy.DateTime, default=datetime.utcnow())
    _individuals = relationship(
        SpeciesBase,
        enable_typechecks=False,
        backref='generation',
        order_by="desc(SpeciesBase._success)"
    )
    """
    :type: list[SpeciesBase]
    """

    __table_args__ = {'sqlite_autoincrement': True}

    @classmethod
    def get_current_generation_number(cls) -> int or None:

        last = db.sess.query(cls.gen_num).order_by(sqlalchemy.desc(cls.gen_num)).first()
        return None if last is None else last[0]

    def __init__(self, species_set: Set[SpeciesBase.__class__]):
        """
        create a new generation

        """

        self._next_generation = None
        """
        :type: Generation
        """

        if self.uid is None:
            self.gen_time = self.mod_time = datetime.utcnow()
            the_generation_number = self.get_current_generation_number()
            self.gen_num = 1 if the_generation_number is None else the_generation_number + 1

        self._species_set = species_set
        """
        :type: set[SpeciesBase.__class__]
        """

        self._species_lookup = {}
        """
        :type: dict[str, SpeciesBase.__class__]
        """

        for c in self._species_set:
            if c.__name__ == SpeciesBase.__name__:
                raise Exception('Cannot add Species base to set')
            self._species_lookup[c.__name__] = c

        self._all_species_picker = _helpers.IndividualPicker(self.individuals)

        self._by_species_picker = {
            k: _helpers.IndividualPicker(
                [ind for ind in self.individuals if ind.class_name == k]) for k in self._species_lookup.keys()}
        """
        :type: Dict[str, _helpers.IndividualPicker]
        """

        self._next_gen_individuals = []
        """
        :type: List[SpeciesBase]
        """

    def add_individuals(self, new_individuals: List[SpeciesBase] or SpeciesBase) -> Set[type]:

        if self._next_generation is not None:
            raise Exception('cannot add more individuals after the next generation is created')

        if type(new_individuals) is not list:
            new_individuals = [new_individuals]

        self._individuals.extend(new_individuals)

        for ind in new_individuals:
            assert ind.class_name != SpeciesBase.__class__.__name__

            self._species_set.add(ind.__class__)

        _helpers.mature_all(new_individuals)

        self.__init__(self._species_set)

        return self._species_set

    def populate_next_generation(self, max_population, keep_all) -> bool:

        if not self._all_species_picker.has_two_alive and not keep_all:
            """
            No successful individuals
            """
            return False

        if keep_all:
            for class_name, picker in self._by_species_picker.items():
                # create new individuals using the class constructor
                the_class = self._species_lookup[class_name]

                if picker.has_two_alive:
                    for i in range(2):
                        female = picker.pick_female()
                        male = picker.pick_male(female)
                        self._next_gen_individuals.extend(_helpers.breed(female, male))
                    # add two new to prevent inbreeding
                    self._next_gen_individuals.extend([the_class(), the_class()])
                elif picker.count_all > 0:
                    for i in range(the_class.get_offspring_count()):
                        self._next_gen_individuals.append(the_class())

        if not self._all_species_picker.has_two_alive:
            while len(self._next_gen_individuals) < max_population:
                for c in self._species_set:
                    self._next_gen_individuals.append(c())

        while len(self._next_gen_individuals) < max_population:
            female = self._all_species_picker.pick_female()

            species_picker = self._by_species_picker[female.class_name]

            if species_picker.has_two_alive:
                male = species_picker.pick_male(female)
            else:
                male = self._species_lookup[female.class_name]()
                male.mature()

            self._next_gen_individuals.extend(_helpers.breed(female, male))

        if len(self._next_gen_individuals) > max_population:
            self._next_gen_individuals = self._next_gen_individuals[:max_population]

        for ind in self._next_gen_individuals:
            # TODO add random mutate
            if random() < 0.5:
                ind.mutate()
            pass

        return True

    def save(self):

        if len(self.individuals) == 0:
            raise AssertionError('there must at least one individual in the generation')

        _helpers.mature_all(self.individuals)

        if self.uid is None:
            db.sess.add(self)

        self.mod_time = datetime.utcnow()

        db.sess.commit()

    @property
    def best_individual(self) -> SpeciesBase or None:
        return self._all_species_picker.best_individual

    @property
    def best_success(self) -> float or None:
        best_ind = self.best_individual

        return None if best_ind is None else best_ind.success

    @property
    def summary_short(self):
        best = self.best_individual
        if best:
            return "Generation {0}. Best: {1}, success: {2}\n\t{3}".format(
                self.gen_num, best.class_name, best.success,
                dict(best.params)
            )
        else:
            return "Generation {0}. None Successful".format(
                self.gen_num
            )

    @property
    def summary_long(self):
        out_string = "Generation {0}\n".format(self.gen_num)

        # print(self._by_species_picker)

        best_list = []
        none_list = []

        for k, v in self._by_species_picker.items():
            best = v.best_individual
            if best:
                best_list.append((k, best.success, dict(best.params), v.count_all, v.count_alive, v.count_dead))
            else:
                none_list.append((k, v.count_all, v.count_alive, v.count_dead))

        best_list.sort(key=lambda x: x[1], reverse=True)

        for b in best_list:
            out_string += "\t{0}, success: {1}, {2}\n\t\tPopulation count: {3}, alive: {4}, dead: {5}\n".format(*b)

        for n in none_list:
            out_string += "\t{0}, None Successful\n\t\tPopulation count: {1}, alive: {2}, dead: {3}\n".format(*n)

        return out_string

    @property
    def individuals(self) -> List[SpeciesBase]:

        for ind in self._individuals:
            if ind.class_name == SpeciesBase.__name__:
                try:
                    ind.__class__ = self._species_lookup[ind.db_class_name]
                    ind.__init__()
                except KeyError:
                    raise KeyError('Class {0} not found in species set'.format(ind.db_class_name))

        return self._individuals

    @property
    def next_generation(self):
        if self._next_generation is None:

            if len(self._next_gen_individuals) == 0:
                raise AssertionError('Next generation not yet populated')

            _helpers.mature_all(self._next_gen_individuals)

            self._next_generation = Generation(self._species_set)
            self._next_generation.add_individuals(self._next_gen_individuals)
            self._next_generation.save()

        return self._next_generation

def create_db():
    Base.metadata.drop_all(bind=db.engine)
    Base.metadata.create_all(bind=db.engine)

__all__ = [create_db, Generation, SpeciesBase]

if __name__ == '__main__':
    create_db()


# nodely >>> putMORE Node.js into Python
#
# Copyright (C) 2017 Stefan Zimmermann <user@zimmermann.co>
#
# nodely is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# nodely is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with nodely.  If not, see <http://www.gnu.org/licenses/>.

"""
nodely.bin

Command proxy to the ``node_modules/.bin/`` directory
"""

from subprocess import CalledProcessError, PIPE
from types import ModuleType
import os
import platform
import sys

from path import Path
import zetup

from nodely import NODE_MODULES_DIR

from .error import NodeCommandError

zetup.module(
    __name__, ['Command'],
    __getitem__=lambda cmdname: __getitem__(cmdname),
    __getattr__=lambda name: __getattr__(name),
    __dir__=lambda: __dir__())

#: Running on Windows?
WIN = platform.system() == 'Windows'

def __getitem__(cmdname):
    """
    Get a :class:`nodely.bin.Command` instance for given `cmdname`.

    :raises OSError: if executable can not be found
    """
    return Command(cmdname)

def __getattr__(name):
    """
    Get a :class:`nodely.bin.Command` instance for given command `name`.

    :raises OSError: if executable cannot be found
    """
    try:
        return __getitem__(name)

    except (IOError, OSError) as exc:
        raise AttributeError(str(exc))

def __dir__():
    cmdnames = (f.basename() for f in (NODE_MODULES_DIR / '.bin').files())
    if WIN:  # pragma: no cover
        cmdnames = (cmd for cmd in cmdnames if cmd.ext.lower() != '.cmd')
    return list(cmdnames)

class Command(zetup.object, Path):
    """
    A Node.js executable from current Python environment.

    Residing in ``node_modules/.bin/``
    """

    def __new__(cls, name):
        """
        Check existance and store absolute path in ``path.Path`` base.

        :param name:
            The basename of the executable in ``node_modules/.bin``
        :raises OSError:
            if executable cannot be found
        """
        if WIN:  # pragma: no cover
            name += '.cmd'
        cmd = Path.__new__(cls, NODE_MODULES_DIR / '.bin' / name)
        cmd.atime  # HACK: raises OSError if not existing
        return cmd

    def __init__(self, name):
        """:meth:`.__new__` does all the work :)."""
        pass

    @property
    def name(self):
        """Get the name of this Node.js command."""
        name = Path(self).basename()
        if WIN:  # pragma: no cover
            # remove .cmd file extension
            name = name.splitext(  # pylint: disable=no-value-for-parameter
            )[0]
        return str(name)

    def Popen(self, cmdargs=None, **options):
        """
        Create a ``subprocess`` for this Node.js command.

        :param cmdargs:
            Optional sequence of command argument strings.
        :param options:
            General options for ``zetup.call``, including all options for
            ``subprocess.call``.
        """
        command = [str(self)]
        if cmdargs is not None:
            command += cmdargs
        return zetup.Popen(command, **options)

    def call(self, cmdargs=None, **options):
        """
        Call this Node.js command.

        :param cmdargs:
            Optional sequence of command argument strings.
        :param options:
            General options for ``zetup.call``, including all options for
            ``subprocess.call``.
        """
        command = [str(self)]
        if cmdargs is not None:
            command += cmdargs
        return zetup.call(command, **options)

    def check_call(self, cmdargs=None, **options):
        """
        Call this Node.js command and check its return code.

        :param cmdargs:
            Optional sequence of command argument strings.
        :param options:
            General options for ``zetup.call``, including all options for
            ``subprocess.call``.

        :raises subprocess.CalledProcessError:
            When return code is not zero.
        """
        command = [str(self)]
        if cmdargs is not None:
            command += cmdargs

        returncode = zetup.call(command, **options)
        if returncode:
            raise NodeCommandError(returncode, command, os.getcwd())

    def check_output(self, cmdargs=None, **options):
        """
        Call this Node.js command, check return code, and return its stdout.

        :param cmdargs:
            Optional sequence of command argument strings.
        :param options:
            General options for ``zetup.call``, including all options for
            ``subprocess.call``.

        :raises subprocess.CalledProcessError:
            When return code is not zero.
        """
        command = [str(self)]
        if cmdargs is not None:
            command += cmdargs

        process = zetup.Popen(command, stdout=PIPE, **options)
        out, _ = process.communicate()
        if process.returncode:
            raise NodeCommandError(process.returncode, command, os.getcwd())

        return out

    def __call__(self, *cmdargs, **kwargs):
        """
        Alternative to :meth:`.check_output`.

        Takes `cmdargs` as varargs instead of a sequence parameter
        """
        return self.check_output(cmdargs, **kwargs)

    def __repr__(self):
        return "{}[{!r}]".format(__name__, self.name)


# -*- coding: utf-8 -*-
import collections
import json
from builtins import print
import time

import scrapy
import self as self

from BZhanSpider.replybean import replybean
from BZhanSpider.items import BzhanspiderItem
from selenium import webdriver

from BZhanSpider.middlewares import SeleniumMiddleware
from urllib import request, parse

"""
name:scrapy唯一定位实例的属性，必须唯一
allowed_domains：允许爬取的域名列表，不设置表示允许爬取所有
start_urls：起始爬取列表
start_requests：它就是从start_urls中读取链接，然后使用make_requests_from_url生成Request，
                这就意味我们可以在start_requests方法中根据我们自己的需求往start_urls中写入
                我们自定义的规律的链接
parse：回调函数，处理response并返回处理后的数据和需要跟进的url
log：打印日志信息
closed：关闭spider
"""

class BzhanSpider(scrapy.Spider):
    name = 'BZhan'
    allowed_domains = ['www.bilibili.com/']

    start_url = input('请输入要抓取B站视频信息的网址, 按回车结束: \n')
    start_url_parse = parse.urlparse(start_url)
    start_url_path = start_url_parse.path
    start_urls = [start_url]
    # start_urls = ['http://www.bilibili.com/video/av48323686/']

    # # https: // api.bilibili.com / x / web - interface / view?aid = 48465492
    # # start_urls = ['https://api.bilibili.com/x/web-interface/view?aid=48465492']#播放界面数据类型json\
    # page_data_url = 'http://www.bilibili.com/video/av48323686'#界面数据，无法获取数量
    # data_url = 'https://api.bilibili.com/x/web-interface/view?aid=48465492'#数据数量(弹幕、播放、点赞等)
    # url = 'https://api.bilibili.com/x/web-interface/archive/stat?aid=48465492'#数量json
    # comment_url = 'https://api.bilibili.com/x/v2/reply?callback=jQuery17202434448397650979_1554717195061&jsonp=jsonp&pn=2&type=1&oid=48465492&sort=0&_=1554718673887'
    # # start_urls = [start_url]
    # # video / av48323686
    # video_comment_url = 'https://api.bilibili.com/x/v2/reply?callback=jQuery17208525034767588849_1554860847400&jsonp=jsonp&pn=2&type=1&oid=48323686&sort=0&_=1554863787356'
    reply_comment_base_url = 'https://api.bilibili.com/x/v2/reply/reply?jsonp=jsonp&pn=%s&type=1&oid=%s&ps=10&root=%s'
    video_comment_url = 'https://api.bilibili.com/x/v2/reply?jsonp=jsonp&pn=%s&type=1&oid=%s&sort=0'

    is_save = False
    reply_comment_type = -1
    # 视频评论
    video_comment = 0
    # 网友的回复
    net_friend_reply_comment = 1
    # up主的回复
    up_reply_comment = 2
    # 视频所有评论及回复，用于判断是否爬到所有评论及回复来保存数据
    total_reply_count = 0

    # 视频的id
    # 从start_url分解出来的path里面过滤
    if start_url_path is not None and len(start_url_path) > 0:
        start_url_path_split = start_url_path.split('/')[2]  # 分割后获取形如av48323686这个块
        oid = start_url_path_split[2:]  # 去掉av留下视频id
        print('-------------------视频id：' + oid + ' -------------------')
    else:
        print('-------------------url不正确，请重新输入-------------------')

    # 视频评论列表
    video_reply_map = collections.OrderedDict()
    # 评论下面的恢复列表，网友互动内容
    net_friend_reply_map = collections.OrderedDict()
    # 视频owner，up主的恢复列表
    up_reply_map = collections.OrderedDict()
    """
    start_urls主请求的回调，在里面解析数据并存储到items中去
    """

    def parse(self, response):  # 默认回调
        print('--------------parse start-----------------')
        # video_comment_url = 'https://api.bilibili.com/x/v2/reply?jsonp=jsonp&pn=1&type=1&oid=48323686&sort=0'

        # print(response.text)
        item = BzhanspiderItem()
        # title = response.css('.tit tr-fix::text').extract_first()
        # title = response.xpath('/html/body/div[3]/div/div[1]/div[1]/h1/span/text()').extract_first()

        title = response.xpath('/html/body/div[3]/div/div[1]/div[1]/h1/span/text()').extract_first()
        column = response.xpath('//span[@class="a-crumbs"]/a[@target="_blank"]/text()').extract_first()
        publish_date = response.xpath('//div[@class="video-data"]/span[2]/text()').extract_first()
        ranking = ''
        play_count = response.xpath('//div[@class="video-data"]/span[contains(@class,"view")]/text()').extract_first()
        danmu_count = response.xpath('//div[@class="video-data"]/span[@class="dm"]/text()').extract_first()
        like_count = response.xpath('/html/body/div[3]/div/div[1]/div[3]/div[1]/span[1]/text()').extract_first()
        corn_count = response.xpath('/html/body/div[3]/div/div[1]/div[3]/div[1]/span[2]/text()').extract_first()
        favorite_count = response.xpath('/html/body/div[3]/div/div[1]/div[3]/div[1]/span[3]/text()').extract_first()
        text_introduce = response.xpath('/html/body/div[3]/div/div[1]/div[4]/div[1]/text()').extract()
        keywords_tag = response.xpath('/html/body/div[3]/div/div[1]/div[5]/ul//text()').extract()
        video_time = response.xpath('//span[@class="bilibili-player-video-time-total"]/text()').extract_first()
        print('-------------------' + str(video_time))
        # comment_page_count = response.xpath('//div[@class="header-page paging-box"]/a[@class="tcd-number"]/text()').extract()[-1]
        # print(comment_page_count)
        #
        # if comment_page_count is not None:
        #     comment_page_count = int(comment_page_count)
        # else:
        #     comment_page_count = 0
        comment_page_count = 3
        if len(text_introduce) > 0:
            text_introduce = '\n'.join(text_introduce)
        else:
            text_introduce = ''
        if len(keywords_tag) > 0:
            keywords_tag = '\n'.join(keywords_tag)
        else:
            keywords_tag = ''
        # resp = json.loads(response.body, encoding='utf-8')
        # data = resp['data']
        # stat = data['stat']
        # title = data['title']
        # column = ''
        # publish_date = data['pubdate']
        # ranking = ''
        # play_count = stat['view']
        # danmu_count = stat['danmaku']
        # like_count = stat['favorite']
        # corn_count = stat['coin']
        # text_introduce = data['desc']
        # keywords_tag = ''

        # 评论先拿第一组数据测试
        up_comments = ''

        # yield item

        # 获取up主的网页url
        # 取指定第一个加[0], 否则会有type问题
        up_page_url = response.xpath('/html/body/div[3]/div/div[2]/div[1]/div[2]/div[1]/a[1]/@href').extract()[0]
        # 获取url绝对路径
        url = response.urljoin(up_page_url)
        print('-------------------' + str(up_page_url))
        print(url)

        # 视频owner的id，mid
        mid = ''

        # 请求评论的接口直接获取json数据进行解析
        if comment_page_count > 0:
            current_page = 1
            yield scrapy.Request(url=self.video_comment_url % (str(current_page), self.oid),
                                 callback=self.video_comment_parse,
                                 dont_filter=True, meta={'is_request_without_browser': True,
                                                         'comment_page_count': comment_page_count,
                                                         'current_page': current_page,
                                                         'mid': mid})

        # DEBUG: Filtered offsiterequest to因为Request中请求的URL和allowed_domains中定义的域名冲突，所以将Request中请求的URL过滤掉了，无法请求
        # 在Request请求参数中，设置dont_filter = True, Request中请求的URL将不通过allowed_domains过滤。
        yield scrapy.Request(url=url, callback=self.up_page_parse, dont_filter=True, meta={'title': title,
                                                                                'column': column,
                                                                                'publish_date': publish_date,
                                                                                'ranking': ranking,
                                                                                'play_count': play_count,
                                                                                'danmu_count': danmu_count,
                                                                                'like_count': like_count,
                                                                                'corn_count': corn_count,
                                                                                'favorite_count': favorite_count,
                                                                                'text_introduce': text_introduce,
                                                                                'keywords_tag': keywords_tag,
                                                                                'up_comments': up_comments,
                                                                                'video_time': video_time,
                                                                                })

        page_data_url = 'http://www.bilibili.com/video/av48323686'  # 界面数据，无法获取数量
        # yield scrapy.Request(url= page_data_url, callback=self.parse_video_page, dont_filter=True)
        print('-------------------parse end-------------------')

    """
    请求当前页的数据
    """

    def parse_video_page(self, response):
        pass

    """
    up主的主页请求回调，处理up主的主页的数据
    """

    def up_page_parse(self, response):
        print('-------------------up_page_parse start-------------------')
        # print(response.text)
        # return
        status = response.status
        if status != 200:
            print('-------------------request failed, status: ' + status)
            return
        item = BzhanspiderItem()

        title = response.meta['title']
        column = response.meta['column']
        publish_date = response.meta['publish_date']
        ranking = response.meta['ranking']
        play_count = response.meta['play_count']
        danmu_count = response.meta['danmu_count']
        like_count = response.meta['like_count']
        corn_count = response.meta['corn_count']
        favorite_count = response.meta['favorite_count']
        text_introduce = response.meta['text_introduce']
        keywords_tag = response.meta['keywords_tag']
        up_comments = response.meta['up_comments']
        video_time = response.meta['video_time']

        up_name = response.xpath('//*[@id="h-name"]/text()').extract_first()
        up_introduction = response.xpath(
            '/html/body/div[2]/div[1]/div[1]/div[2]/div[2]/div/div[2]/div[2]/h4/text()').extract_first()
        up_certification = response.xpath('//*[@id="h-ceritification"]/text()').extract_first()
        up_focus_count = response.xpath('//*[@id="n-gz"]/text()').extract_first()
        up_fans_count = response.xpath('//*[@id="n-fs"]/text()').extract_first()
        up_play_count = response.xpath('//*[@id="n-bf"]/text()').extract_first()

        # res = json.loads(response.body.decode('utf-8'))

        # up_name = res['h-name']
        # up_certification = res['h-ceritification']
        print("title---------------------" + str(title))
        print("column---------------------" + str(column))
        print("publish_date---------------------" + str(publish_date))
        print("ranking---------------------" + str(ranking))
        print("play_count---------------------" + str(play_count))
        print("danmu_count---------------------" + str(danmu_count))
        print("like_count---------------------" + str(like_count))
        print("corn_count---------------------" + str(corn_count))
        print("favorite_count---------------------" + str(favorite_count))
        print("text_introduce---------------------" + str(text_introduce))
        print("keywords_tag---------------------" + str(keywords_tag))
        print("up_comments---------------------" + str(up_comments))
        print("up_name---------------------" + str(up_name))
        print("up_introduction---------------------" + str(up_introduction))
        print("up_certification---------------------" + str(up_certification))
        print("up_focus_count---------------------" + str(up_focus_count))
        print("up_fans_count---------------------" + str(up_fans_count))
        print("up_play_count---------------------" + str(up_play_count))
        print("video_time---------------------" + str(video_time))

        item['title'] = title
        item['column'] = column
        item['publish_date'] = publish_date
        item['ranking'] = ranking
        item['play_count'] = play_count
        item['danmu_count'] = danmu_count
        item['like_count'] = like_count
        item['corn_count'] = corn_count
        item['favorite_count'] = favorite_count
        item['text_introduce'] = text_introduce
        item['keywords_tag'] = keywords_tag
        item['up_comments'] = up_comments

        item['up_name'] = up_name
        item['up_introduction'] = up_introduction
        item['up_certification'] = up_certification
        item['up_focus_count'] = up_focus_count
        item['up_fans_count'] = up_fans_count
        item['up_play_count'] = up_play_count
        item['video_time'] = video_time
        # self.is_save = True

        # 回调item数据给pipelines
        yield item

        # #关闭浏览器
        # SeleniumMiddleware.__del__(self)

        print('-------------------up_page_parse end-------------------')

    """
    请求视频的评论，分为网友评论和up主评论分类存储
    """

    def video_comment_parse(self, response):
        print('-------------------video_comment_parse start-------------------')
        if response is None or response.body is None:
            return

        status = response.status
        if status != 200:
            print('-------------------request failed, status: ' + status)
            return
        # 回复评论的的url
        reply_comment_url = 'https://api.bilibili.com/x/v2/reply/reply?callback=jQuery17208938958981882081_1554882206023&jsonp=jsonp&pn=1&type=1&oid=48323686&ps=10&root=1503675439&=1554890310602'
        reply_comment_url = 'https://api.bilibili.com/x/v2/reply/reply?jsonp=jsonp&pn=1&type=1&oid=48323686&ps=10&root=1503675439'
        oid = ''
        up_mid = ''

        item = BzhanspiderItem()
        try:
            comment_page_count = 0
            current_page = -1
            if 'oid' in response.meta:
                oid = response.meta['oid']
            if 'up_mid' in response.meta:
                up_mid = response.meta['up_mid']
            resp = json.loads(response.body, encoding='utf-8')
            if 'data' not in resp:
                return
            data = resp['data']
            # 热门评论
            # hots = data['hots']
            replies = data['replies']
            page = data['page']
            upper = data['upper']
            """所有评论及回复只赋值一次，避免多次赋值导致爬取的和获取的acount数量不匹配导致不能存储"""
            if self.total_reply_count == 0 and page['acount'] > 0:
                self.total_reply_count = page['acount']
            print('-------------------评论及回复总数： ' + str(self.total_reply_count))
            current_page = page['num']
            item = BzhanspiderItem()

            comment_page_count = calculate_comment_page_count(page, True)
            hot_comments_url = 'https://api.bilibili.com/x/v2/reply?jsonp=jsonp&pn=1&type=1&oid=48323686&sort=0'

            """遍历当前页的评论并分类保存"""
            if replies is not None:
                for reply in replies:
                    bean = get_reply_bean(reply, oid)
                    # print(bean.to_string())
                    # 视频评论保存
                    if bean.get_mid() == str(upper['mid']):
                        self.up_reply_map[bean.get_rpid_str()] = bean
                    else:
                        self.video_reply_map[bean.get_rpid_str()] = bean

                    # 判断评论下面的回复是否存在，存在的话，继续遍历回复列表在comment_reply_parse里面处理所有回复
                    comment_replies = reply['replies']
                    if comment_replies is not None:
                        comment_reply_current_page = 1  # 默认请求第一页的的回复
                        yield scrapy.Request(url=self.reply_comment_base_url % (
                            str(comment_reply_current_page), self.oid, bean.get_rpid_str()),
                                             callback=self.comment_reply_parse,
                                             dont_filter=True,
                                             meta={'is_request_without_browser': True,
                                                   'comment_reply_current_page': page,
                                                   'root': bean.get_rpid_str()})

            # self.add_comments_to_map(hots)
            # self.add_comments_to_map(replies)
            page = current_page + 1
            if comment_page_count >= page > 0:
                yield scrapy.Request(url=self.video_comment_url % (str(page), self.oid),
                                     callback=self.video_comment_parse,
                                     dont_filter=True,
                                     meta={'is_request_without_browser': True,
                                           'comment_page_count': comment_page_count,
                                           'current_page': page})
            else:
                if Is_yield_comments_and_reply_to_pipelines(self.total_reply_count, self.video_reply_map,
                                                            self.net_friend_reply_map,
                                                            self.up_reply_map):
                    # if 'video_reply_map' in item:
                    item['video_reply_map'] = self.video_reply_map

                    # if 'up_reply_map' in item:
                    item['up_reply_map'] = self.up_reply_map

                    # if 'net_friend_reply_map' in item:
                    item['net_friend_reply_map'] = self.net_friend_reply_map
                    yield item
                # # print(self.video_reply_map)
                # # if 'video_reply_map' in item:
                # item['video_reply_map'] = self.video_reply_map
                # length = len(self.video_reply_map) + len(self.net_friend_reply_map) + len(self.up_reply_map)
                # if length >= 42:
                #     print('-------------------video_comment_parse------------------抓取完成， 返回评论 map 总和: ' + str(length) + '-------------------------------------------------------')
                #     yield item
        except TimeoutError as e:
            print('-------------------评论抓取失败, 失败原因' + str(e))
        print('-------------------video_comment_parse end-------------------')

    """
    处理评论下的回复，包括网友回复、up主回复，分类存储
    """

    def comment_reply_parse(self, response):
        print('-------------------comment_reply_parse start-------------------')
        try:
            oid = ''
            up_mid = ''
            root = ''  # 对应评论的rpid，用于url请求拿到指定rpid下面的回复列表
            comment_reply_current_page = -1  # 回复列表页数
            comment_reply_page_count = 0  # 回复列表当前页数

            item = BzhanspiderItem()
            if 'comment_reply_current_page' in response.meta:
                comment_reply_current_page = response.meta['comment_reply_current_page']
            if 'oid' in response.meta:
                oid = response.meta['oid']
            if 'up_mid' in response.meta:
                up_mid = response.meta['up_mid']
            if 'root' in response.meta:
                root = response.meta['root']

            resp = json.loads(response.body, encoding='utf-8')
            data = resp['data']
            replies = data['replies']
            page = data['page']
            upper = data['upper']

            comment_reply_current_page = page['num']
            comment_reply_page_count = calculate_comment_page_count(page, False)

            print('-------------------comment_reply_current_page: ' + str(comment_reply_current_page))
            print('-------------------comment_reply_page_count: ' + str(comment_reply_page_count))

            """遍历当前评论的回复并分类保存"""
            if replies is not None:
                for reply in replies:
                    bean = get_reply_bean(reply, oid)
                    # print(bean.to_string())

                    # 评论下的回复进行保存，分网友和up主两类
                    if bean.get_mid() == str(upper['mid']):
                        self.up_reply_map[bean.get_rpid_str()] = bean
                    else:
                        self.net_friend_reply_map[bean.get_rpid_str()] = bean

                    comment_replies = reply['replies']  # 评论下面的回复

                    if comment_replies is not None:
                        page = 1
                        if comment_reply_page_count >= page > 0:
                            yield scrapy.Request(
                                url=self.reply_comment_base_url % (str(page), self.oid, bean.get_root_str()),
                                callback=self.comment_reply_parse,
                                dont_filter=True,
                                meta={'is_request_without_browser': True,
                                      'root': bean.get_root_str()})

            page = comment_reply_current_page + 1
            if comment_reply_page_count >= page > 0:
                yield scrapy.Request(url=self.reply_comment_base_url % (str(page), self.oid, root),
                                     callback=self.comment_reply_parse,
                                     dont_filter=True,
                                     meta={'is_request_without_browser': True,
                                           'current_page': page,
                                           'root': root})
            else:
                if Is_yield_comments_and_reply_to_pipelines(self.total_reply_count, self.video_reply_map,
                                                            self.net_friend_reply_map,
                                                            self.up_reply_map):
                    # if 'video_reply_map' in item:
                    item['video_reply_map'] = self.video_reply_map

                    # if 'up_reply_map' in item:
                    item['up_reply_map'] = self.up_reply_map

                    # if 'net_friend_reply_map' in item:
                    item['net_friend_reply_map'] = self.net_friend_reply_map
                    yield item
                # if 'up_reply_map' in item:
                # item['up_reply_map'] = self.up_reply_map
                # item['net_friend_reply_map'] = self.net_friend_reply_map
                # # yield item
                # length = len(self.video_reply_map) + len(self.net_friend_reply_map) + len(self.up_reply_map)
                # if length >= 42:
                #     # print('--------------------------comment_reply_parse--------------抓取完成， 返回评论 map 总和: ' + str(length) + '-------------------------------------------------------')
                #     yield item
        except TimeoutError as e:
            print('-------------------评论下的回复抓取失败, 失败原因' + str(e))
        print('-------------------comment_reply_parse end-------------------')

    # def add_comments_to_map(replies):
    #     if 0 >= len(replies):
    #         return
    #
    #     for reply in replies:
    #         ctime = reply['ctime']
    #         time = time.strftime('%Y-%m-%d %H:%M:%S', reply['ctime'])
    #         content = reply['content']
    #         message = content['message']
    #
    #     pass

"""
将数据存储javabean格式的数据结构并返回
"""

def get_reply_bean(reply, oid):
    # reply id
    rpid_str = reply['rpid_str']
    # 根评论的reply id，对应rpid_str
    root_str = reply['root_str']
    # 上一级评论(可能是回复的回复多层嵌套)
    parent_str = reply['parent_str']
    # reply 时间
    ctime = reply['ctime']
    # 时间戳转换为时间元祖
    time_tuples = time.localtime(ctime)
    # 再将时间元祖格式化
    date = time.strftime('%Y-%m-%d %H:%M:%S', time_tuples)
    # reply 楼层，第几楼
    floor = reply['floor']
    # reply 用户id，mid
    mid = reply['mid']
    member = reply['member']
    # reply 用户的用户名
    uname = member['uname']
    # 用户评论内容
    content = reply['content']
    message = content['message']
    # print('rpid_str: ' + str(rpid_str) + ', message: ' + str(message)
    #       + ', date: ' + str(date) + ', mid: ' + str(mid) + ', floor: ' + str(floor))

    return replybean(str(oid), str(rpid_str), str(date), str(floor), str(mid), str(uname),
                     str(message), str(root_str), str(parent_str))

"""
计算评论页数
"""

def calculate_comment_page_count(page, is_video_comment=True):
    acount = 0
    if 'acount' in page:
        acount = page['acount']  # 所有评论数(包括视频评论和评论下的回复)
    count = page['count']  # 总评论数
    num = page['num']  # 当前页是第几页
    size = page['size']  # 每一页的个数
    page_count = count // size
    if count % size > 0:
        page_count += 1
    if is_video_comment:
        print('-------------------爬取中---评论: 所有评论数(包括视频评论和评论下的回复)：' + str(acount) + ', 视频评论数：' + str(count)
              + ', 总页数：' + str(page_count) + ', 当前第' + str(num) + '页， 每一页个数：' + str(size))
    else:
        print('-------------------爬取中---回复: 当前视频评论下的回复数：' + str(count) + ', 总页数：' + str(page_count)
              + ', 当前第' + str(num) + '页， 每一页个数：' + str(size))
    return page_count

def Is_yield_comments_and_reply_to_pipelines(comment_reply_total_count, video_reply_map, net_friend_reply_map,
                                             up_reply_map):
    # print(self.video_reply_map)
    item = BzhanspiderItem()
    length = len(video_reply_map) + len(net_friend_reply_map) + len(up_reply_map)
    print('-------------------评论及回复是否保存判断， 总评论及回复数:' + str(comment_reply_total_count)
          + '， 当前爬到的评论及回复总数量: ' + str(length) + '\n-------------------当前爬到的视频回复数量：' + str(len(video_reply_map))
          + ', 当前爬到的回复总数' + str(len(net_friend_reply_map) + len(up_reply_map))
          + '--------------------------------------\n')

    if length >= comment_reply_total_count:
        print('-------------------所有评论及回复爬取完成，保存-------------------\n')
        return True
    else:
        print('-------------------所有评论及回复爬取未完成，继续爬取ing-------------------\n')
        return False

@staticmethod
def close(spider, reason):
    print('-------------------BZhanSpider close-------------------')
    yield spider.item


# -*- coding: utf-8 -*-
"""
Web scraping on BaiDubaike.
1. Set a first page, then random find website in it.
2. Run it for fixed times to see where you got the website.
3. If the website cannot be open, we ignore this, rerandom again.

@author: mayao
"""

from bs4 import BeautifulSoup
from urllib.request import urlopen
from urllib import error
import re
import random

base_url = "https://baike.baidu.com"
his = ["/item/%E7%BD%91%E7%BB%9C%E7%88%AC%E8%99%AB/5162711"]

for i in range(20):
    url = base_url + his[-1]
    try:
        html = urlopen(url).read().decode('utf-8')
    except error.URLError:  #IF invalid website, ignore and restart again
        his.pop()
        his.append(random.sample(sub_urls, 1)[0]['href'])
        continue
    soup = BeautifulSoup(html, features='lxml')
    print(i, soup.find('h1').get_text(), '    url: ', his[-1])

    # find valid urls
    sub_urls = soup.find_all("a", {"target": "_blank", "href": re.compile("/item/(%.{2})+$")})

    if len(sub_urls) != 0:
        his.append(random.sample(sub_urls, 1)[0]['href'])
    else:
        # no valid sub link found
        his.pop()


from datetime import timedelta

class Config:
	MAX_MOVE_TIME_IN_SECONDS = 20
	MAX_NAME_LENGTH = 12
	MAX_CLIENTS_PER_ROOM = 6
	MAX_BOTS_PER_ROOM = 6

	ROOMS_TABLE = "MartianDice-Rooms-dev"
	GAMES_TABLE = "MartianDice-Games-dev"
	ROOM_EXPIRATION = timedelta(days = 1)

def is_bot_name(name):
	return name.startswith("Bot-") or name == "Bot"


import numpy as np
import tensorflow as tf
import json
import timeit
from matplotlib import pyplot as plt
from playsound import playsound
hParams = {
    'experimentName': None,
    'datasetProportion': 1,
    'valProportion': .1,
    'numEpochs': 20,
    'denseLayers': None,
    "optimizer" : None,
    "convLayers" : []
    }
extraV1 = {
    "itemsToPlot" : [
        'C[32,64] d0.0 D[128,10] rms',
        'C[32,64] d0.2 D[128,10] rms',
        'C[32,64] d0.0 D[128,10] adam',
        'C[32,64] d0.2 D[128,10] adam',
    ]
}
extraV2 = {
    "itemsToPlot" : [
        'C[32,64] d0.0 D[128,10] adam',
        'C[32,64] d0.02 D[128,10] adam',
        'C[32,64] d0.05 D[128,10] adam',
        'C[32,64] d0.1 D[128,10] adam',
        'C[32,64] d0.2 D[128,10] adam',
        'C[32,64] d0.3 D[128,10] adam',
    ]
}
extraV3 = {
    "itemsToPlot" : [
        'C[32] d0.2 D[128,10] adam',
        'C[32,64] d0.2 D[128,10] adam',
        'C[32,64,128] d0.2 D[128,10] adam',
    ]
}
extraV4 = {
    "itemsToPlot" : [
        'C[32,64] d0.2 D[128,10] adam',
        'C[32,64] d0.2 D[256,128,10] adam',
        'C[32,64] d0.2 D[512,256,128,10] adam',
    ]
}
Counter = { "X" : 1}
def runExp(expNames = []):
    x_train, y_train, x_val, y_val, x_test, y_test = get10ClassData(flatten=False)
    for currExp in expNames:
        getHParams(currExp)
        trainRes, testRes = cnnGray([x_train, y_train, x_val, y_val, x_test, y_test])
        writeExperimentalResults(trainRes,testRes)
def getHParams(expName=None):
    shortTest = False # hardcode to True to run a quick debugging test
    if shortTest:
        print("+++++++++++++++++ WARNING: SHORT TEST +++++++++++++++++")
        hParams['datasetProportion'] = 0.01
        hParams['numEpochs'] = 2
    if (expName is None):
        # Not running an experiment yet, so just return the "common" parameters
        return hParams
    hParams["experimentName"] = expName
    if (expName == 'C[32,64] d0.0 D[128,10] rms'):
        dropProp = 0.0
        hParams['convLayers'] = [
        {'conv_numFilters': 32, 'conv_f': 3, 'conv_p': 'same', 'conv_act': 'relu', 'pool_f': 2, 'pool_s': 2, 'drop_prop': dropProp},
        {'conv_numFilters': 64, 'conv_f': 3, 'conv_p': 'same', 'conv_act': 'relu', 'pool_f': 2, 'pool_s': 2, 'drop_prop': dropProp},
        ]
        hParams['denseLayers'] = [128, 10]
        hParams['optimizer'] = 'rmsprop'
    elif (expName == 'C[32,64] d0.2 D[128,10] rms'):
        dropProp = 0.2
        hParams['convLayers'] = [
        {'conv_numFilters': 32, 'conv_f': 3, 'conv_p': 'same', 'conv_act': 'relu', 'pool_f': 2, 'pool_s': 2, 'drop_prop': dropProp},
        {'conv_numFilters': 64, 'conv_f': 3, 'conv_p': 'same', 'conv_act': 'relu', 'pool_f': 2, 'pool_s': 2, 'drop_prop': dropProp},
        ]
        hParams['denseLayers'] = [128, 10]
        hParams['optimizer'] = 'rmsprop'
    elif (expName == 'C[32,64] d0.0 D[128,10] adam'):
        dropProp = 0.0
        hParams['convLayers'] = [
        {'conv_numFilters': 32, 'conv_f': 3, 'conv_p': 'same', 'conv_act': 'relu', 'pool_f': 2, 'pool_s': 2, 'drop_prop': dropProp},
        {'conv_numFilters': 64, 'conv_f': 3, 'conv_p': 'same', 'conv_act': 'relu', 'pool_f': 2, 'pool_s': 2, 'drop_prop': dropProp},
        ]
        hParams['denseLayers'] = [128, 10]
        hParams['optimizer'] = 'adam'
    elif (expName == 'C[32,64] d0.2 D[128,10] adam'):
        dropProp = 0.2
        hParams['convLayers'] = [
        {'conv_numFilters': 32, 'conv_f': 3, 'conv_p': 'same', 'conv_act': 'relu', 'pool_f': 2, 'pool_s': 2, 'drop_prop': dropProp},
        {'conv_numFilters': 64, 'conv_f': 3, 'conv_p': 'same', 'conv_act': 'relu', 'pool_f': 2, 'pool_s': 2, 'drop_prop': dropProp},
        ]
        hParams['denseLayers'] = [128, 10]
        hParams['optimizer'] = 'adam'

    elif (expName == 'C[32,64] d0.02 D[128,10] adam'):
        dropProp = 0.02
        hParams['convLayers'] = [
        {'conv_numFilters': 32, 'conv_f': 3, 'conv_p': 'same', 'conv_act': 'relu', 'pool_f': 2, 'pool_s': 2, 'drop_prop': dropProp},
        {'conv_numFilters': 64, 'conv_f': 3, 'conv_p': 'same', 'conv_act': 'relu', 'pool_f': 2, 'pool_s': 2, 'drop_prop': dropProp},
        ]
        hParams['denseLayers'] = [128, 10]
        hParams['optimizer'] = 'adam'
    elif (expName == 'C[32,64] d0.05 D[128,10] adam'):
        dropProp = 0.05
        hParams['convLayers'] = [
        {'conv_numFilters': 32, 'conv_f': 3, 'conv_p': 'same', 'conv_act': 'relu', 'pool_f': 2, 'pool_s': 2, 'drop_prop': dropProp},
        {'conv_numFilters': 64, 'conv_f': 3, 'conv_p': 'same', 'conv_act': 'relu', 'pool_f': 2, 'pool_s': 2, 'drop_prop': dropProp},
        ]
        hParams['denseLayers'] = [128, 10]
        hParams['optimizer'] = 'adam'
    elif (expName == 'C[32,64] d0.1 D[128,10] adam'):
        dropProp = 0.1
        hParams['convLayers'] = [
        {'conv_numFilters': 32, 'conv_f': 3, 'conv_p': 'same', 'conv_act': 'relu', 'pool_f': 2, 'pool_s': 2, 'drop_prop': dropProp},
        {'conv_numFilters': 64, 'conv_f': 3, 'conv_p': 'same', 'conv_act': 'relu', 'pool_f': 2, 'pool_s': 2, 'drop_prop': dropProp},
        ]
        hParams['denseLayers'] = [128, 10]
        hParams['optimizer'] = 'adam'
    elif (expName == 'C[32,64] d0.3 D[128,10] adam'):
        dropProp = 0.3
        hParams['convLayers'] = [
        {'conv_numFilters': 32, 'conv_f': 3, 'conv_p': 'same', 'conv_act': 'relu', 'pool_f': 2, 'pool_s': 2, 'drop_prop': dropProp},
        {'conv_numFilters': 64, 'conv_f': 3, 'conv_p': 'same', 'conv_act': 'relu', 'pool_f': 2, 'pool_s': 2, 'drop_prop': dropProp},
        ]
        hParams['denseLayers'] = [128, 10]
        hParams['optimizer'] = 'adam'

    elif (expName == 'C[32] d0.2 D[128,10] adam'):
        dropProp = 0.2
        hParams['convLayers'] = [
        {'conv_numFilters': 32, 'conv_f': 3, 'conv_p': 'same', 'conv_act': 'relu', 'pool_f': 2, 'pool_s': 2, 'drop_prop': dropProp},
        ]
        hParams['denseLayers'] = [128, 10]
        hParams['optimizer'] = 'adam'
    elif (expName == 'C[32,64,128] d0.2 D[128,10] adam'):
        dropProp = 0.2
        hParams['convLayers'] = [
        {'conv_numFilters': 32, 'conv_f': 3, 'conv_p': 'same', 'conv_act': 'relu', 'pool_f': 2, 'pool_s': 2, 'drop_prop': dropProp},
        {'conv_numFilters': 64, 'conv_f': 3, 'conv_p': 'same', 'conv_act': 'relu', 'pool_f': 2, 'pool_s': 2, 'drop_prop': dropProp},
        {'conv_numFilters': 128, 'conv_f': 3, 'conv_p': 'same', 'conv_act': 'relu', 'pool_f': 2, 'pool_s': 2, 'drop_prop': dropProp},
        ]
        hParams['denseLayers'] = [128, 10]
        hParams['optimizer'] = 'adam'
    elif (expName == 'C[32,64,128,256] d0.2 D[128,10] adam'):
        dropProp = 0.2
        hParams['convLayers'] = [
        {'conv_numFilters': 32, 'conv_f': 3, 'conv_p': 'same', 'conv_act': 'relu', 'pool_f': 2, 'pool_s': 2,'drop_prop': dropProp},
        {'conv_numFilters': 64, 'conv_f': 3, 'conv_p': 'same', 'conv_act': 'relu', 'pool_f': 2, 'pool_s': 2,'drop_prop': dropProp},
        {'conv_numFilters': 128, 'conv_f': 3, 'conv_p': 'same', 'conv_act': 'relu', 'pool_f': 2, 'pool_s': 2,'drop_prop': dropProp},
        {'conv_numFilters': 256, 'conv_f': 3, 'conv_p': 'same', 'conv_act': 'relu', 'pool_f': 2, 'pool_s': 2,'drop_prop': dropProp},
        ]
        hParams['denseLayers'] = [128, 10]
        hParams['optimizer'] = 'adam'

    elif (expName == 'C[32,64] d0.2 D[256,128,10] adam'):
        dropProp = 0.2
        hParams['convLayers'] = [
            {'conv_numFilters': 32, 'conv_f': 3, 'conv_p': 'same', 'conv_act': 'relu', 'pool_f': 2, 'pool_s': 2,
             'drop_prop': dropProp},
            {'conv_numFilters': 64, 'conv_f': 3, 'conv_p': 'same', 'conv_act': 'relu', 'pool_f': 2, 'pool_s': 2,
             'drop_prop': dropProp},
        ]
        hParams['denseLayers'] = [256, 128, 10]
        hParams['optimizer'] = 'adam'
    elif (expName == 'C[32,64] d0.2 D[512,256,128,10] adam'):
        dropProp = 0.2
        hParams['convLayers'] = [
            {'conv_numFilters': 32, 'conv_f': 3, 'conv_p': 'same', 'conv_act': 'relu', 'pool_f': 2, 'pool_s': 2,
             'drop_prop': dropProp},
            {'conv_numFilters': 64, 'conv_f': 3, 'conv_p': 'same', 'conv_act': 'relu', 'pool_f': 2, 'pool_s': 2,
             'drop_prop': dropProp},
        ]
        hParams['denseLayers'] = [512,256,128,10]
        hParams['optimizer'] = 'adam'
    else:
        #error…
        pass
def buildValAccuracyPlot(extraV):
    itemsToPlot = extraV
    yList = []
    for item in itemsToPlot:
        print(item)
        hParamsl, trainResults, testResults = readExperimentalResults(item)
        yList.append(trainResults["val_accuracy"])
    plotCurves(x=np.arange(0, hParamsl['numEpochs']),
               yList=yList,
               xLabel="Epoch",
               yLabelList=itemsToPlot,
               title="validation accuracies")
def buildTestAccuracyPlot(extraV):
    pointLabels = extraV
    xList = []
    yList = []
    xLabel = "Parameter Count"
    yLabel = "Test Set Accuracy"
    title = "Test Accuracy for Various Models"
    filename = "test accuracies for " + str(pointLabels)
    for item in pointLabels:
        hParamsl, trainResults, testResults = readExperimentalResults(item)
        xList.append(hParamsl["paramCount"])
        yList.append((testResults[1]))
    plotPoints(xList, yList, pointLabels, xLabel, yLabel, title, filename)
#given function
def processResults():
    hParamsl, trainResults, testResults = readExperimentalResults(hParams["experimentName"])
    itemsToPlot = ['accuracy', 'val_accuracy']
    plotCurves(x=np.arange(0, hParamsl['numEpochs']),
    yList=[trainResults[item] for item in itemsToPlot],
    xLabel="Epoch",
    yLabelList=itemsToPlot,
    title=hParamsl['experimentName'])
    itemsToPlot = ['loss', 'val_loss']
    plotCurves(x=np.arange(0, hParamsl['numEpochs']),
    yList=[trainResults[item] for item in itemsToPlot],
    xLabel="Epoch",
    yLabelList=itemsToPlot,
    title=hParamsl['experimentName'])
#provided function
def plotPoints(xList, yList, pointLabels=[], xLabel="", yLabel="", title="", filename="pointPlot"):
    plt.figure()
    plt.scatter(xList,yList)
    plt.xlabel(xLabel)
    plt.ylabel(yLabel)
    plt.title(title)
    if pointLabels != []:
        for i, label in enumerate(pointLabels):
            plt.annotate(label, (xList[i], yList[i]))
    filepath = "results/" + str(Counter["X"]) + "/" + "test Accuracies" + ".png"
    plt.savefig(filepath)
    print("Figure saved in", filepath)
#provided function : plots our data
def plotCurves(x, yList, xLabel="", yLabelList=[], title=""):
    fig, ax = plt.subplots()
    y = np.array(yList).transpose()
    ax.plot(x, y)
    ax.set(xlabel=xLabel, title=title)
    plt.legend(yLabelList, loc='best', shadow=True)
    ax.grid()
    filepath = "results/" +str(Counter["X"]) + "/" +  str(title) + ".png"
    fig.savefig(filepath)
    print("Figure saved in", filepath)
#write expirement result
def writeExperimentalResults(trainResults,testResults):
    dict = {
        "hParams" : hParams,
        "trainResults" :trainResults,
        "testResults" : testResults
    }
    with open ("results/" + str(Counter["X"]) + "/" + str(hParams["experimentName"])+ ".txt" , "w") as resultFile:
        resultFile.write(json.dumps(dict))
#read data from our files
def readExperimentalResults(filePath):#Finish
    with open("results/" +  str(Counter["X"]) + "/" +  filePath + ".txt") as file:
        data = json.load(file)
        hParamsl = data["hParams"]
        return hParamsl, data["trainResults"], data["testResults"]
#shuffle data function
def correspondingShuffle(x,y):
    indices = tf.range(start=0, limit=tf.shape(x)[0], dtype=tf.int32)
    shuffled_indices = tf.random.shuffle(indices)
    x = tf.gather(x, shuffled_indices)
    y= tf.gather(y, shuffled_indices)
    return x,y
#get Data functions
def get10ClassData(proportion = 1.0,silent=True,flatten=True):
    (x_train, y_train), (x_test, y_test) =  tf.keras.datasets.fashion_mnist.load_data()
    if proportion != 1.0:
        x_train = x_train[:int(len(x_train) * proportion)]
        y_train = y_train[:int(len(y_train) * proportion)]
        x_test = x_test[:int(len(x_test) * proportion)]
        y_test = y_test[:int(len(y_test) * proportion)]
    #set rnage to be 0 - 1
    x_train = x_train / 255
    x_test = x_test / 255
    #reshape from 3d to 1d
    if flatten:
        x_train = tf.reshape(x_train,[x_train.shape[0],28*28])
        x_test = tf.reshape(x_test,[x_test.shape[0],28*28])
    #shuffle
    x_train, y_train = correspondingShuffle(x_train, y_train)
    x_test, y_test = correspondingShuffle(x_test, y_test)
    #split data into train, val and test:
    if hParams["valProportion"] > 0.0:
        maxValIndexX = int(x_train.shape[0] * hParams["valProportion"])
        maxValIndexY = int(y_train.shape[0] * hParams["valProportion"])
        x_val = x_train[:maxValIndexX]
        y_val = y_train[:maxValIndexY]
        x_train = x_train[maxValIndexX:]
        y_train = y_train[maxValIndexY:]
        if not silent:
            print(x_train)
            print("With shape : " + str(x_train.shape))
            print(y_train)
            print("With shape : " + str(y_train.shape))
            print(x_test)
            print("With shape : " + str(x_test.shape))
            print(y_test)
            print("With shape : " + str(y_test.shape))
        return x_train, y_train, x_val, y_val, x_test, y_test
    if not silent:
        print(x_train)
        print("With shape : " + str(x_train.shape))
        print(y_train)
        print("With shape : " + str(y_train.shape))
        print(x_test)
        print("With shape : " + str(x_test.shape))
        print(y_test)
        print("With shape : " + str(y_test.shape))
    return x_train, y_train,None, None, x_test, y_test
#Model that works with grey scale images
def cnnGrayHardcode(dataSubsets):
    x_train, y_train, x_val, y_val, x_test, y_test = dataSubsets
    num_channels = 1
    image_width = 28
    image_height = 28
    x_train = tf.reshape(x_train, (-1, image_width, image_height, num_channels))
    x_val = tf.reshape(x_val, (-1, image_width, image_height, num_channels))
    x_test = tf.reshape(x_test, (-1, image_width, image_height, num_channels))

    fourthModel = tf.keras.Sequential()

    fourthModel.add(tf.keras.layers.Conv2D(
        filters=32,
        kernel_size=(3, 3),
        activation="relu",
        input_shape=(image_width, image_height, num_channels)
    ))
    fourthModel.add(tf.keras.layers.MaxPooling2D(
        pool_size=(2, 2)
    ))
    fourthModel.add(tf.keras.layers.Dropout(.2))

    fourthModel.add(tf.keras.layers.Conv2D(
        filters=64,
        kernel_size=(3, 3),
        activation="relu"
    ))
    fourthModel.add(tf.keras.layers.MaxPooling2D(
        pool_size=(2, 2)
    ))
    fourthModel.add(tf.keras.layers.Dropout(.2))

    fourthModel.add(tf.keras.layers.Conv2D(
        filters=64,
        kernel_size=(3, 3),
        activation="relu"
    ))
    fourthModel.add(tf.keras.layers.MaxPooling2D(
        pool_size=(2, 2)
    ))
    fourthModel.add(tf.keras.layers.Dropout(.2))

    fourthModel.add(tf.keras.layers.Flatten())

    fourthModel.add(tf.keras.layers.Dense(128,activation="relu"))
    fourthModel.add(tf.keras.layers.Dense(10))

    startTime = timeit.default_timer()
    fourthModel.compile(loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), metrics="accuracy",
                         optimizer=hParams["optimizer"])
    elapsedTime = timeit.default_timer() - startTime
    print('\033[94m' + "Construction Time : " + str(elapsedTime) + "\033[0;0m")

    startTime = timeit.default_timer()
    hist = fourthModel.fit(x_train, y_train,
                            validation_data=(x_val, y_val) if hParams['valProportion'] != 0.0 else None,
                            epochs=hParams['numEpochs'], verbose=1)
    elapsedTime = timeit.default_timer() - startTime
    print('\033[94m' + "Training time : " + str(elapsedTime) + "\033[0;0m")
    hParams["paramCount"] = fourthModel.count_params()

    startTime = timeit.default_timer()
    accuracy = fourthModel.evaluate(x_test, y_test)
    elapsedTime = timeit.default_timer() - startTime
    print('\033[94m' + "Testing time : " + str(elapsedTime) + " With Accuracy : " + str(accuracy) + "\033[0;0m")
    print('\033[92m')
    fourthModel.summary()
    print(".count_params() = " + str(fourthModel.count_params()))
    print("\033[0;0m")
    return hist.history, accuracy
# model for gray scale images that can be configured
def cnnGray(dataSubsets):
    x_train, y_train, x_val, y_val, x_test, y_test = dataSubsets
    num_channels = 1
    image_width = 28
    image_height = 28
    x_train = tf.reshape(x_train, (-1, image_width, image_height, num_channels))
    x_val = tf.reshape(x_val, (-1, image_width, image_height, num_channels))
    x_test = tf.reshape(x_test, (-1, image_width, image_height, num_channels))

    fourthModel = tf.keras.Sequential()
    for x in hParams["convLayers"]:
        x2 = x['conv_numFilters']
        fourthModel.add(tf.keras.layers.Conv2D(
            filters=x2,
            kernel_size=(3, 3),
            activation="relu",
            input_shape=(image_width, image_height, num_channels)
        ))
        fourthModel.add(tf.keras.layers.MaxPooling2D(
            pool_size=(2, 2)
        ))
        x3 = x["drop_prop"]
        fourthModel.add(tf.keras.layers.Dropout(x3))

    fourthModel.add(tf.keras.layers.Flatten())

    for x in range(len(hParams["denseLayers"]) - 1):

        fourthModel.add(tf.keras.layers.Dense(hParams["denseLayers"][x], activation="relu"))

    fourthModel.add(tf.keras.layers.Dense(10))

    startTime = timeit.default_timer()
    fourthModel.compile(loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), metrics="accuracy",
                        optimizer=hParams["optimizer"])
    elapsedTime = timeit.default_timer() - startTime
    print('\033[94m' + "Construction Time : " + str(elapsedTime) + "\033[0;0m")

    startTime = timeit.default_timer()
    hist = fourthModel.fit(x_train, y_train,
                           validation_data=(x_val, y_val) if hParams['valProportion'] != 0.0 else None,
                           epochs=hParams['numEpochs'], verbose=1)
    elapsedTime = timeit.default_timer() - startTime
    print('\033[94m' + "Training time : " + str(elapsedTime) + "\033[0;0m")
    hParams["paramCount"] = fourthModel.count_params()

    startTime = timeit.default_timer()
    accuracy = fourthModel.evaluate(x_test, y_test)
    elapsedTime = timeit.default_timer() - startTime
    print('\033[94m' + "Testing time : " + str(elapsedTime) + " With Accuracy : " + str(accuracy) + "\033[0;0m")
    print('\033[92m')
    fourthModel.summary()
    print(".count_params() = " + str(fourthModel.count_params()))
    print("\033[0;0m")
    return hist.history, accuracy
def main():
    theSeed = 50
    np.random.seed(theSeed)
    tf.random.set_seed(theSeed)
    buildTestAccuracyPlot()
def main2():
    theSeed = 50
    np.random.seed(theSeed)
    tf.random.set_seed(theSeed)
    '''
    runExp(extraV1["itemsToPlot"])
    for x in extraV1["itemsToPlot"]:
        hParams["experimentName"] = x
        processResults()
    buildValAccuracyPlot(extraV1["itemsToPlot"])
    buildTestAccuracyPlot(extraV1["itemsToPlot"])
    '''
    Counter["X"] = Counter["X"] + 1
    '''
    runExp(extraV2["itemsToPlot"])
    for x in extraV2["itemsToPlot"]:
        hParams["experimentName"] = x
        processResults()
    buildValAccuracyPlot(extraV2["itemsToPlot"])
    buildTestAccuracyPlot(extraV2["itemsToPlot"])
    '''
    Counter["X"] = Counter["X"] + 1

    #runExp(extraV3["itemsToPlot"])
    for x in extraV3["itemsToPlot"]:
        hParams["experimentName"] = x
        processResults()
    buildValAccuracyPlot(extraV3["itemsToPlot"])
    buildTestAccuracyPlot(extraV3["itemsToPlot"])
    Counter["X"] = Counter["X"] + 1

    runExp(extraV4["itemsToPlot"])
    for x in extraV4["itemsToPlot"]:
        hParams["experimentName"] = x
        processResults()
    buildValAccuracyPlot(extraV4["itemsToPlot"])
    buildTestAccuracyPlot(extraV4["itemsToPlot"])
    playsound("./sound.mp3")
main2()


import torch

# one thing thats quite confusing when working with torch is the idea of DIM or axis
# This article does a good job of explaining it:
# https://towardsdatascience.com/understanding-dimensions-in-pytorch-6edf9972d3be

#simple example
x = torch.tensor([1, 2, 3])
s = torch.sum(x)
assert(6 == s)

## Official
## torch.sum(input, dim, keepdim=False, dtype=None) → Tensor
## Returns the sum of each row of the input tensor in the given dimension dim.
## What the heck does this mean?
## example:

x = torch.tensor([
     [1, 2, 3],
     [4, 5, 6]
   ])

print(x.shape)  # torch.Size([2, 3])

# The way to understand the 'dim' of torch sum is that it collapses the specified axis.
# So when it collapses the axis/dim 0 (the row), it becomes just one row (it sums column-wise).
# When it collapses the axis/dim 1 (the column), it becomes just one column (it sums row-wise).

# e.g.
dim0 = torch.sum(x, dim=0)  # collapse rows
print(dim0) # tensor([5, 7, 9])

dim1 = torch.sum(x, dim=1)  # collapse columns
print(dim1) # tensor([ 6, 15])

# you can also pass two dims to the function to collapse BOTH rows and columns
dim0_1 = torch.sum(x, dim=[0,1])
print(dim0_1)  # tensor(21)

## 3d - 3d gets more confusing, the z axis takes over as dim=0.
y = torch.tensor([
     [
       [1, 2, 3],
       [4, 5, 6]
     ],
     [
       [7, 8, 9],
       [10, 11, 12]
     ],
     [
       [13, 14, 15],
       [16, 17, 18]
     ]
   ])
print(y.shape)  # torch.Size([3, 2, 3])

# sum and collapse along the z axis (matrix wise)
y0 = torch.sum(y, dim=0)
print(y0)    # tensor([[21, 24, 27],
             #         [30, 33, 36]])

# sum and collapse row wise
y1 = torch.sum(y, dim=1)
print(y1)  # tensor([[ 5,  7,  9],
            #       [17, 19, 21],
            #       [29, 31, 33]])

# sum and collapse column wise
y2 = torch.sum(y, dim=2)
print(y2)  # tensor([[ 6, 15],
            #        [24, 33],
            #        [42, 51]])

# point is, the last dimention is always column-wise
# second to last dimention is always row-wise ect.


import streamlit as st
import requests

st.title("Youtube Tag Genrator")
a=st.text_input("Enter You Querry","Type here here..")
if st.button("Search"):

    query = a
    parms = {"client": "firefox", "q": query, "hl": "en", }
    url = 'http://suggestqueries.google.com/complete/search'
    r = requests.get(url, params=parms)
    r.status_code
    # print(r.status)

    res = r.json()[1]
    for data in res:

        st.write(f"{data}",end='')

st.markdown("'
@kabir_arya001
",unsafe_allow_html=True)