# Adapted from
# https://github.com/davidsbatista/NER-Evaluation

import logging
from collections import namedtuple
from copy import deepcopy
from tqdm import tqdm
import pandas as pd
import json
from spacy.lang.en import English
#from spacy.gold import biluo_tags_from_offsets as get_biluo
from spacy.training import offsets_to_biluo_tags as get_biluo
from sklearn.metrics import f1_score, precision_score, recall_score
from ast import literal_eval
from typing import Dict, NewType
from utils.interval_merger import IntervalMerger

Entity = namedtuple("Entity", "e_type start_offset end_offset")
Metrics = NewType("Metrics", Dict[str,Dict[str,float]])

class EvaluatorNER:
    def __init__(self, df, pred_col_name, gold_col_name):
        """
            Parameters
            ----------
            - df: pd.DataFrame
            - pred_col_name: str
                name of the dataframe's column containing the model's predictions (list(tuple))
            - gold_col_name: str
                name of the dataframe's column containing the gold labels (list(tuple))
        """
        nlp = English()
        #df[gold_col_name] = df[gold_col_name].apply(lambda x: literal_eval(x))
        #df[pred_col_name] = df[pred_col_name].apply(lambda x: literal_eval(x))
        df["spacy_doc"] = df.apply(lambda row: nlp(row.text),axis=1)

        df["gold_fix"] = df.apply(lambda x: IntervalMerger().merge([self.fix_ent(x.spacy_doc,y) for y in x[gold_col_name]]), axis=1)
        df["pred_fix"] = df.apply(lambda x: IntervalMerger().merge([self.fix_ent(x.spacy_doc,y) for y in x[pred_col_name]]), axis=1)

        df["gold_tags"] = df.apply(lambda x: get_biluo(x.spacy_doc, x.gold_fix), axis=1)
        df["pred_tags"] = df.apply(lambda x: get_biluo(x.spacy_doc, x.pred_fix), axis=1)

        df["gold_tags"] = df.apply(lambda x: [tag.replace("U-","B-").replace("L-","I-") for tag in x.gold_tags], axis=1)
        df["pred_tags"] = df.apply(lambda x: [tag.replace("U-","B-").replace("L-","I-") for tag in x.pred_tags], axis=1)

        all_ent_types = set([y.split("-")[-1] for x in df.pred_tags.tolist() for y in x]) - set("O")

        self.evaluator = Evaluator(df.gold_tags.tolist(), df.pred_tags.tolist(), all_ent_types)
        self.evaluator.evaluate()

    def get(self) -> Metrics:
        return self.evaluator.get()

    def merge_discontinuous_overlaps(self, sdocs, annotation_type):
        for sdoc in ptqdm(sdocs):
            for span in sdoc.doc.spans:
                if span.contains_annotation(annotation_type):
                    for i in span.intervals:
                        for j in span.intervals:
                            if i != j and i.overlaps(j):
                                i.begin = min(i.begin, j.begin)
                                i.end = max(i.end, j.end)
                                span.intervals.remove(j)
        return sdocs

    def fix_ent(self, doc, ent):

        #start,end,ent_type = ent
        start,end = ent

        for tok in doc:
            tstart = tok.idx
            tend = tok.idx + len(tok)

            if tstart < start < tend:
                start = tstart
                break
            elif start == tstart-1:
                start += 1
                break

        for tok in doc:
            tstart = tok.idx
            tend = tok.idx + len(tok)

            if tstart < end < tend:
                end = tend
                break
            elif end == tend+1:
                end -= 1
                break

        #return (start, end, ent_type)
        return (start, end, "ADR")

class Evaluator():

    def __init__(self, true, pred, tags="ADR"):
        """
            Parameters
            ----------
            - true: list(tuple): gold labels
            - pred: list(tuple): predicted labels
        """

        if len(true) != len(pred):
            raise ValueError("Number of predicted documents does not equal true")

        self.true = true
        self.pred = pred
        self.tags = tags

        # Setup dict into which metrics will be stored.
        fields = ['correct','incorrect','partial','missed','spurious','possible','actual','precision','recall','f1']
        self.metrics_results = {field:0 for field in fields}

        # Copy results dict to cover the four schemes.

        self.results = {
            'strict': deepcopy(self.metrics_results),
            'ent_type': deepcopy(self.metrics_results),
            'partial':deepcopy(self.metrics_results),
            'exact':deepcopy(self.metrics_results),
            }

        # Create an accumulator to store results

        self.evaluation_agg_entities_type = {e: deepcopy(self.results) for e in tags}

    def evaluate(self):

        logging.info(
            "Imported %s predictions for %s true examples",
            len(self.pred), len(self.true)
        )

        for true_ents, pred_ents in tqdm(zip(self.true, self.pred), total=len(self.true)):

            # Check that the length of the true and predicted examples are the
            # same. This must be checked here, because another error may not
            # be thrown if the lengths do not match.

            if len(true_ents) != len(pred_ents):
                raise ValueError("Prediction length does not match true example length")

            # Compute results for one message

            tmp_results, tmp_agg_results = compute_metrics(
                collect_named_entities(true_ents),
                collect_named_entities(pred_ents),
                self.tags
            )

            # Cycle through each result and accumulate

            # TODO: Combine these loops below:

            for eval_schema in self.results:

                for metric in self.results[eval_schema]:

                    self.results[eval_schema][metric] += tmp_results[eval_schema][metric]

            # Calculate global precision and recall

            self.results = compute_precision_recall_wrapper(self.results)

            # Aggregate results by entity type

            for e_type in self.tags:

                for eval_schema in tmp_agg_results[e_type]:

                    for metric in tmp_agg_results[e_type][eval_schema]:

                        self.evaluation_agg_entities_type[e_type][eval_schema][metric] += tmp_agg_results[e_type][eval_schema][metric]

                # Calculate precision recall at the individual entity level

                self.evaluation_agg_entities_type[e_type] = compute_precision_recall_wrapper(self.evaluation_agg_entities_type[e_type])

        #self.print_metrics(self.results, "partial")
        #self.print_metrics(self.results, "strict")
        #return self.results, self.evaluation_agg_entities_type

    def get(self):
        return self.results

    def print_metrics(self, all_metrics, metrics_type):
        metrics = ["precision", "recall", "f1"]
        res = {m:-1 for m in metrics}

        print("-----------------------")
        print(f"NER {metrics_type} Metrics")
        for m in metrics:
            res[m] = all_metrics[metrics_type][m]
            print(f"{m:<10}| {round(all_metrics[metrics_type][m],4)}")

def collect_named_entities(tokens):
    """
    Creates a list of Entity named-tuples, storing the entity type and the start and end
    offsets of the entity.

    :param tokens: a list of tags
    :return: a list of Entity named-tuples
    """

    named_entities = []
    start_offset = None
    end_offset = None
    ent_type = None

    for offset, token_tag in enumerate(tokens):

        if token_tag == 'O':
            if ent_type is not None and start_offset is not None:
                end_offset = offset - 1
                named_entities.append(Entity(ent_type, start_offset, end_offset))
                start_offset = None
                end_offset = None
                ent_type = None

        elif ent_type is None:
            ent_type = token_tag[2:]
            start_offset = offset

        elif ent_type != token_tag[2:] or (ent_type == token_tag[2:] and token_tag[:1] == 'B'):

            end_offset = offset - 1
            named_entities.append(Entity(ent_type, start_offset, end_offset))

            # start of a new entity
            ent_type = token_tag[2:]
            start_offset = offset
            end_offset = None

    # catches an entity that goes up until the last token

    if (ent_type is not None) and (start_offset is not None) and end_offset is None:
        named_entities.append(Entity(ent_type, start_offset, len(tokens)-1))

    return named_entities

def compute_metrics(true_named_entities, pred_named_entities, tags):

    eval_metrics = {'correct': 0, 'incorrect': 0, 'partial': 0, 'missed': 0, 'spurious': 0, 'precision': 0, 'recall': 0, 'f1': 0}

    # overall results

    evaluation = {
        'strict': deepcopy(eval_metrics),
        'ent_type': deepcopy(eval_metrics),
        'partial': deepcopy(eval_metrics),
        'exact': deepcopy(eval_metrics)
    }

    # results by entity type

    evaluation_agg_entities_type = {e: deepcopy(evaluation) for e in tags}

    # keep track of entities that overlapped

    true_which_overlapped_with_pred = []

    # Subset into only the tags that we are interested in.
    # NOTE: we remove the tags we don't want from both the predicted and the
    # true entities. This covers the two cases where mismatches can occur:
    #
    # 1) Where the model predicts a tag that is not present in the true data
    # 2) Where there is a tag in the true data that the model is not capable of
    # predicting.

    true_named_entities = [ent for ent in true_named_entities if ent.e_type in tags]
    pred_named_entities = [ent for ent in pred_named_entities if ent.e_type in tags]

    # go through each predicted named-entity

    for pred in pred_named_entities:
        found_overlap = False

        # Check each of the potential scenarios in turn. See
        # http://www.davidsbatista.net/blog/2018/05/09/Named_Entity_Evaluation/
        # for scenario explanation.

        # Scenario I: Exact match between true and pred

        if pred in true_named_entities:
            true_which_overlapped_with_pred.append(pred)
            evaluation['strict']['correct'] += 1
            evaluation['ent_type']['correct'] += 1
            evaluation['exact']['correct'] += 1
            evaluation['partial']['correct'] += 1

            # for the agg. by e_type results
            evaluation_agg_entities_type[pred.e_type]['strict']['correct'] += 1
            evaluation_agg_entities_type[pred.e_type]['ent_type']['correct'] += 1
            evaluation_agg_entities_type[pred.e_type]['exact']['correct'] += 1
            evaluation_agg_entities_type[pred.e_type]['partial']['correct'] += 1

        else:

            # check for overlaps with any of the true entities

            for true in true_named_entities:

                pred_range = range(pred.start_offset, pred.end_offset)
                true_range = range(true.start_offset, true.end_offset)

                pred_range = range(pred.start_offset, pred.end_offset+1)
                true_range = range(true.start_offset, true.end_offset+1)

                # Scenario IV: Offsets match, but entity type is wrong

                if true.start_offset == pred.start_offset and pred.end_offset == true.end_offset \
                        and true.e_type != pred.e_type:

                    # overall results
                    evaluation['strict']['incorrect'] += 1
                    evaluation['ent_type']['incorrect'] += 1
                    evaluation['partial']['correct'] += 1
                    evaluation['exact']['correct'] += 1

                    # aggregated by entity type results
                    evaluation_agg_entities_type[true.e_type]['strict']['incorrect'] += 1
                    evaluation_agg_entities_type[true.e_type]['ent_type']['incorrect'] += 1
                    evaluation_agg_entities_type[true.e_type]['partial']['correct'] += 1
                    evaluation_agg_entities_type[true.e_type]['exact']['correct'] += 1

                    true_which_overlapped_with_pred.append(true)
                    found_overlap = True

                    break

                # check for an overlap i.e. not exact boundary match, with true entities

                elif find_overlap(true_range, pred_range):

                    true_which_overlapped_with_pred.append(true)

                    # Scenario V: There is an overlap (but offsets do not match
                    # exactly), and the entity type is the same.
                    # 2.1 overlaps with the same entity type

                    if pred.e_type == true.e_type:

                        # overall results
                        evaluation['strict']['incorrect'] += 1
                        evaluation['ent_type']['correct'] += 1
                        evaluation['partial']['partial'] += 1
                        evaluation['exact']['incorrect'] += 1

                        # aggregated by entity type results
                        evaluation_agg_entities_type[true.e_type]['strict']['incorrect'] += 1
                        evaluation_agg_entities_type[true.e_type]['ent_type']['correct'] += 1
                        evaluation_agg_entities_type[true.e_type]['partial']['partial'] += 1
                        evaluation_agg_entities_type[true.e_type]['exact']['incorrect'] += 1

                        found_overlap = True

                        break

                    # Scenario VI: Entities overlap, but the entity type is
                    # different.

                    else:
                        # overall results
                        evaluation['strict']['incorrect'] += 1
                        evaluation['ent_type']['incorrect'] += 1
                        evaluation['partial']['partial'] += 1
                        evaluation['exact']['incorrect'] += 1

                        # aggregated by entity type results
                        # Results against the true entity

                        evaluation_agg_entities_type[true.e_type]['strict']['incorrect'] += 1
                        evaluation_agg_entities_type[true.e_type]['partial']['partial'] += 1
                        evaluation_agg_entities_type[true.e_type]['ent_type']['incorrect'] += 1
                        evaluation_agg_entities_type[true.e_type]['exact']['incorrect'] += 1

                        # Results against the predicted entity

                        # evaluation_agg_entities_type[pred.e_type]['strict']['spurious'] += 1

                        found_overlap = True

                        break

            # Scenario II: Entities are spurious (i.e., over-generated).

            if not found_overlap:

                # Overall results

                evaluation['strict']['spurious'] += 1
                evaluation['ent_type']['spurious'] += 1
                evaluation['partial']['spurious'] += 1
                evaluation['exact']['spurious'] += 1

                # Aggregated by entity type results

                # NOTE: when pred.e_type is not found in tags
                # or when it simply does not appear in the test set, then it is
                # spurious, but it is not clear where to assign it at the tag
                # level. In this case, it is applied to all target_tags
                # found in this example. This will mean that the sum of the
                # evaluation_agg_entities will not equal evaluation.

                for true in tags:

                    evaluation_agg_entities_type[true]['strict']['spurious'] += 1
                    evaluation_agg_entities_type[true]['ent_type']['spurious'] += 1
                    evaluation_agg_entities_type[true]['partial']['spurious'] += 1
                    evaluation_agg_entities_type[true]['exact']['spurious'] += 1

    # Scenario III: Entity was missed entirely.

    for true in true_named_entities:
        if true in true_which_overlapped_with_pred:
            continue
        else:
            # overall results
            evaluation['strict']['missed'] += 1
            evaluation['ent_type']['missed'] += 1
            evaluation['partial']['missed'] += 1
            evaluation['exact']['missed'] += 1

            # for the agg. by e_type
            evaluation_agg_entities_type[true.e_type]['strict']['missed'] += 1
            evaluation_agg_entities_type[true.e_type]['ent_type']['missed'] += 1
            evaluation_agg_entities_type[true.e_type]['partial']['missed'] += 1
            evaluation_agg_entities_type[true.e_type]['exact']['missed'] += 1

    # Compute 'possible', 'actual' according to SemEval-2013 Task 9.1 on the
    # overall results, and use these to calculate precision and recall.

    for eval_type in evaluation:
        evaluation[eval_type] = compute_actual_possible(evaluation[eval_type])

    # Compute 'possible', 'actual', and precision and recall on entity level
    # results. Start by cycling through the accumulated results.

    for entity_type, entity_level in evaluation_agg_entities_type.items():

        # Cycle through the evaluation types for each dict containing entity
        # level results.

        for eval_type in entity_level:

            evaluation_agg_entities_type[entity_type][eval_type] = compute_actual_possible(
                entity_level[eval_type]
            )

    return evaluation, evaluation_agg_entities_type

def find_overlap(true_range, pred_range):
    """Find the overlap between two ranges

    Find the overlap between two ranges. Return the overlapping values if
    present, else return an empty set().

    Examples:

    >>> find_overlap((1, 2), (2, 3))
    2
    >>> find_overlap((1, 2), (3, 4))
    set()
    """

    true_set = set(true_range)
    pred_set = set(pred_range)

    overlaps = true_set.intersection(pred_set)

    return overlaps

def compute_actual_possible(results):
    """
    Takes a result dict that has been output by compute metrics.
    Returns the results dict with actual, possible populated.

    When the results dicts is from partial or ent_type metrics, then
    partial_or_type=True to ensure the right calculation is used for
    calculating precision and recall.
    """

    correct = results['correct']
    incorrect = results['incorrect']
    partial = results['partial']
    missed = results['missed']
    spurious = results['spurious']

    # Possible: number annotations in the gold-standard which contribute to the
    # final score

    possible = correct + incorrect + partial + missed

    # Actual: number of annotations produced by the NER system

    actual = correct + incorrect + partial + spurious

    results["actual"] = actual
    results["possible"] = possible

    return results

def compute_precision_recall(results, partial_or_type=False):
    """
    Takes a result dict that has been output by compute metrics.
    Returns the results dict with precison and recall populated.

    When the results dicts is from partial or ent_type metrics, then
    partial_or_type=True to ensure the right calculation is used for
    calculating precision and recall.
    """

    actual = results["actual"]
    possible = results["possible"]
    partial = results['partial']
    correct = results['correct']

    if partial_or_type:
        precision = (correct + 0.5 * partial) / actual if actual > 0 else 0
        recall = (correct + 0.5 * partial) / possible if possible > 0 else 0

    else:
        precision = correct / actual if actual > 0 else 0
        recall = correct / possible if possible > 0 else 0

    results["precision"] = precision
    results["recall"] = recall
    results["f1"] = 2 * precision * recall / (precision + recall) if precision + recall > 0 else 0

    return results

def compute_precision_recall_wrapper(results):
    """
    Wraps the compute_precision_recall function and runs on a dict of results
    """

    results_a = {key: compute_precision_recall(value, True) for key, value in results.items() if
                 key in ['partial', 'ent_type']}
    results_b = {key: compute_precision_recall(value) for key, value in results.items() if
                 key in ['strict', 'exact']}

    results = {**results_a, **results_b}

    return results


# %%
import os
from keras.preprocessing.sequence import pad_sequences
import tensorflow as tf
import pickle
# %%
path = os.path.join('NLP_Stuff', 'model.h5')
model = tf.keras.models.load_model(path)

path = os.path.join('NLP_Stuff', 'tokenizer.pkl')
with open(path, 'rb') as f:
    tokenizer = pickle.load(f)
# %%

def predict_sentiment(text):
    # preprocessing the given text
    text_seq = tokenizer.texts_to_sequences(text)
    text_pad = pad_sequences(text_seq, maxlen=200)

    # predicting the class
    predicted_sentiment = model.predict(text_pad).round()

    print(model.predict(text_pad))

    if predicted_sentiment == 1.0:
        return(print('It is a positive sentiment'))
    else:
        return(print('It is a negative sentiment'))

# # %%
# text = ['hahahahahaha funny cute']
# predict_sentiment(text)
# # %%%%
# x = ['no one cares about me. i will die alone']
# predict_sentiment(x)

# %%


# Imports
import Protests_Functions as pf
import seaborn as sns
import matplotlib.pyplot as plt
import pandas as pd
from collections import Counter

def count_genderfreq(text, dictionaryIndicators):
    indicators = []
    for word in text:
        if word in dictionaryIndicators:
            indicators.append(word)
    return len(indicators)

# Variables
farleft_file = 'C:/Users/khahn/PycharmProjects/NLP Protests/article database/Protests Links - Far Left.csv'
left_file = 'C:/Users/khahn/PycharmProjects/NLP Protests/article database/Protests Links - Left.csv'
center_file = 'C:/Users/khahn/PycharmProjects/NLP Protests/article database/Protests Links - Center.csv'
right_file = 'C:/Users/khahn/PycharmProjects/NLP Protests/article database/Protests Links - Right.csv'
farright_file = 'C:/Users/khahn/PycharmProjects/NLP Protests/article database/Protests Links - Far Right.csv'

genderDict = {'male indicators': ['male', 'man', 'men', 'he', 'his', 'him', 'boy', 'boys'], 'female indicators':
    ['female', 'woman', 'women', 'she', 'her', 'hers', 'girl', 'girls']}
dictionaryNames = ['male indicators', 'female indicators']

# Frequency of the words in the categories of article
# Get the text from all the articles per political ideology
farleft_text = pf.open_file(farleft_file)
left_text = pf.open_file(left_file)
center_text = pf.open_file(center_file)
right_text = pf.open_file(right_file)
farright_text = pf.open_file(farright_file)
all_text = farleft_text + left_text + farright_text
print('got all the text')

all_indicatorsm = count_genderfreq(all_text, genderDict['male indicators'])
farleft_indicatorsm = count_genderfreq(farleft_text, genderDict['male indicators'])
left_indicatorsm = count_genderfreq(left_text, genderDict['male indicators'])
center_indicatorsm = count_genderfreq(center_text, genderDict['male indicators'])
right_indicatorsm = count_genderfreq(right_text, genderDict['male indicators'])
farright_indicatorsm = count_genderfreq(farright_text, genderDict['male indicators'])

all_indicatorsf = count_genderfreq(all_text, genderDict['female indicators'])
farleft_indicatorsf = count_genderfreq(farleft_text, genderDict['female indicators'])
left_indicatorsf = count_genderfreq(left_text, genderDict['female indicators'])
center_indicatorsf = count_genderfreq(center_text, genderDict['female indicators'])
right_indicatorsf = count_genderfreq(right_text, genderDict['female indicators'])
farright_indicatorsf = count_genderfreq(farright_text, genderDict['female indicators'])
print('got all the indicators')

maleFreq = [farleft_indicatorsm, left_indicatorsm, center_indicatorsm, right_indicatorsm, farright_indicatorsm]
femaleFreq = [farleft_indicatorsf, left_indicatorsf, center_indicatorsf, right_indicatorsf, farright_indicatorsf]
frequency = [farleft_indicatorsm, left_indicatorsm, center_indicatorsm, right_indicatorsm, farright_indicatorsm, farleft_indicatorsf, left_indicatorsf, center_indicatorsf, right_indicatorsf, farright_indicatorsf]
words = ['Male gender', 'Male gender', 'Male gender', 'Male gender', 'Male gender', 'Female gender', 'Female gender',
         'Female gender', 'Female gender', 'Female gender']
leaning = ['Far left', 'Left', 'Center', 'Right', 'Far Right', 'Far left', 'Left', 'Center', 'Right', 'Far Right']

all_info = [words, frequency, leaning]
df = pd.DataFrame(all_info).transpose()
df.columns = ['Words', 'Frequency', 'Political Leaning']
print('made the dataframe')

plot = sns.catplot(x='Words',y='Frequency',hue='Political Leaning', data=df, kind='bar')
#plot.set_title('Frequency of Gender Words from Newspapers across the Political Spectrum Covering the Black Lives Matter Protests')
plt.show()


import argparse
import torch
import numpy as np
import TD3
from custom_env import PendulumEnv
from torch.utils.tensorboard import SummaryWriter
from noise import OrnsteinUhlenbeckActionNoise
import os
if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='TD3 Training on Nonlinear Pendulum System.')
    parser.add_argument("--save_dir", type=str)
    parser.add_argument("--run_name", type=str)
    parser.add_argument("--length", type=float, default=1.0, help="Length of pendulum.")
    parser.add_argument("--mass", type=float, default=1.0, help="Mass of pendulum.")
    parser.add_argument("--max_action", type=float, default=2.0, help="Maximum torque.")
    parser.add_argument("--noise", type=float, default=0.0, help="Noise applied to next state of environment")
    parser.add_argument("--use_linear", action="store_true", default=False)
    args = parser.parse_args()
    if not os.path.exists(os.path.join(os.getcwd(), f"data/{args.save_dir}")):
        os.makedirs(os.path.join(os.getcwd(), f"data/{args.save_dir}"), 0o777)
    if not os.path.exists(os.path.join(os.getcwd(), "tensorboard_runs")):
        os.mkdir("./tensorboard_runs")

    env = PendulumEnv(m=args.mass, l=args.length, noise=args.noise, max_action=args.max_action)
    writer = SummaryWriter(f"./tensorboard_runs/{args.run_name}")
    epochs = 40000
    tau = 0.01
    gamma = 0.99
    noise_stddev = 0.2
    batch_size = 64
    max_epoch_length = 200
    action_dim = env.action_space.shape[-1]
    ou_noise = OrnsteinUhlenbeckActionNoise(mu=np.zeros(action_dim),
                                            sigma=float(noise_stddev) * np.ones(action_dim))
    state_dim = env.observation_space.shape[-1]
    max_action = env.action_space.high[0]
    count = 0
    agent = TD3.TD3(state_dim=state_dim,
                    action_dim=action_dim,
                    max_action=max_action,
                    gamma=gamma,
                    tau=tau,
                    policy_freq=10,
                    use_linear=args.use_linear
                    )
    epoch_steps, returns, losses = [], [], []
    for epoch in range(epochs):
        state, done = env.reset(), False
        ou_noise.reset()
        ep_r = 0
        ep_step = 0
        ep_loss = 0
        while True:
            ep_step += 1
            count += 1
            action = agent.calc_action(state, ou_noise)
            next_state, reward, done, _ = env.step(action)
            done_float = float(done)
            ep_r += reward
            agent.replay_buffer.add(state, action, next_state, reward, done_float)
            state = next_state
            if count > batch_size:
                value_loss = agent.update()
                ep_loss += value_loss
            if done or ep_step >= max_epoch_length:
                break
        epoch_steps.append(ep_step)
        returns.append(ep_r)
        losses.append(ep_loss/ ep_step)
        writer.add_scalar("epoch/value_loss", ep_loss/ep_step, epoch)
        writer.add_scalar("epoch/return", ep_r, epoch)

        writer.add_scalar("epoch/ep_steps", ep_step, epoch)
        print(f"Epoch: {epoch}  Loss {ep_loss / ep_step}   Reward: {ep_r} Epoch Steps {ep_step} ")
        if epoch % 100 == 0:
            torch.save(agent.actor, f"./data/{args.save_dir}" + "/actor%s.pt" % epoch)
            torch.save(agent.critic, f"./data/{args.save_dir}" + "/critic%s.pt" % epoch)

    epoch_steps = np.array(epoch_steps)
    np.save(f"./data/{args.save_dir}/epoch_steps.npy", epoch_steps)
    returns = np.array(returns)
    np.save(f"./data/{args.save_dir}/returns.npy", returns)
    losses = np.array(losses)
    np.save(f"./data/{args.save_dir}/av_loss.npy", losses)


"""
可视化训练效果:
    程序运行结束，在cmd 运行tensorboard --logdir=path，然后在浏览器打开http://localhost:6006
"""
import tensorflow as tf
import numpy as np
from datetime import datetime
from sklearn.datasets import fetch_california_housing
from sklearn.preprocessing import StandardScaler

if __name__ == "__main__":
    now = datetime.utcnow().strftime("%Y%m%d%H%M%S")
    root_logdir = r"D://tf_logs"
    logdir = "{}/run-{}/".format(root_logdir, now)

    scaler = StandardScaler()
    housing = fetch_california_housing()
    scaler_housing_data = scaler.fit_transform(housing.data)
    m,n = scaler_housing_data.shape
    scaler_housing_data_bias = np.c_[np.ones((m,1)),scaler_housing_data]

    # 设置梯度下降超参
    learn_rate = 0.01
    n_epochs = 1000
    # 不指定大小，等到真正训练时再生成
    X = tf.placeholder(shape=(None,n+1),dtype=tf.float32,name='X')
    y = tf.placeholder(shape=(None,1), dtype=tf.float32, name='y')

    # 初始化theta 在-1 - 1 之间
    theta = tf.Variable(tf.random.uniform([n+1,1],-1.0,1.0),name='theta')
    # 计算预测值
    y_pred = tf.matmul(X,theta,name='predictions')

    # 限定变量作用域
    # mse 生成相关操作会被放在loss 框图内
    with tf.name_scope("loss") as scope:
        error = y_pred - y
        mse = tf.reduce_mean(tf.square(error), name="mse")

    # error = y_pred - y
    # 计算损失函数
    # mse = tf.reduce_mean(tf.square(error),name='mse')
    optimizer = tf.train.GradientDescentOptimizer(learning_rate=learn_rate)
    training_op = optimizer.minimize(mse)
    init = tf.global_variables_initializer()
    # 可视化
    mse_summary = tf.summary.scalar('MSE', mse)
    file_writer = tf.summary.FileWriter(logdir, tf.get_default_graph())

    # 设置批量的样本数
    batch_size = 1000
    n_batches = int(np.ceil(m/batch_size))

    with tf.Session() as sess:
        sess.run(init)
        for i in range(n_epochs):
            for batch_index in range (n_batches):
                # 随机批量，没有按顺序
                indices = np.random.randint(m, size=batch_size)
                X_batch = scaler_housing_data_bias[indices]
                y_batch = housing.target.reshape(-1,1)[indices]
                if batch_index%10 == 0:
                    # 每10次保存效果
                    summary_str = mse_summary.eval(feed_dict={X: X_batch, y: y_batch})
                    step = i * n_batches + batch_index
                    file_writer.add_summary(summary_str, step)
                # 此时才真正确定X，y 大小
                sess.run(training_op,feed_dict={X:X_batch,y:y_batch})
        best_theta = theta.eval()
    #  关闭
    file_writer.close()
    print("theta = ",best_theta)


import os
import random
import math
import pygame

from os import listdir
from os.path import isfile, join

pygame.init()
pygame.display.set_caption("Platformer")

WIDTH, HEIGHT = 1000, 800
FPS = 30
PLAYER_VEL = 4

window = pygame.display.set_mode((WIDTH, HEIGHT))

def flip(sprites):
    return [pygame.transform.flip(sprite, True, False) for sprite in sprites]

def load_sprite_sheets(dir1, dir2, width, height, direction=False):
    path = join("assets", dir1, dir2)
    images = [f for f in listdir(path) if isfile(join(path, f))]

    all_sprites = {}

    for image in images:
        sprite_sheet = pygame.image.load(join(path, image)).convert_alpha()

        sprites = []
        for i in range(sprite_sheet.get_width() // width):
            surface = pygame.Surface((width, height), pygame.SRCALPHA, 32)
            rect = pygame.Rect(i * width, 0, width, height)
            surface.blit(sprite_sheet, (0, 0), rect)
            sprites.append(pygame.transform.scale2x(surface))

        if direction:
            all_sprites[image.replace(".png", "") + "_right"] = sprites
            all_sprites[image.replace(".png", "") + "_left"] = flip(sprites)
        else:
            all_sprites[image.replace(".png", "")] = sprites

    return all_sprites

def load_block(size):
    path = join("assets", "Terrain", "Terrain.png")
    image = pygame.image.load(path).convert_alpha()
    surface = pygame.Surface((size, size), pygame.SRCALPHA, 32)
    rect = pygame.Rect(96, 0, size, size)
    surface.blit(image, (0, 0), rect)
    return pygame.transform.scale2x(surface)

class Player(pygame.sprite.Sprite):
    # Use Sprite for pixel collision
    COLOR = (255, 0, 0)
    GRAVITY = 8
    SPRITES = load_sprite_sheets("MainCharacters", "VirtualGuy", 32, 32, True)
    ANIMATION_DELAY = 2

    def __init__(self, x, y, width, height):
        super().__init__()
        self.rect = pygame.Rect(x, y, width, height)
        # Velocity here denotes the speed we move our player in every frame
        self.x_vel = 0
        self.y_vel = 0
        self.mask = None
        self.direction = "left"
        self.animation_count = 0
        self.fall_count = 0
        self.jump_count = 0
        self.hit = False
        self.hit_count = 0
        self.health = 5
        self.win = False

    def jump(self):
        self.y_vel = -self.GRAVITY * 2
        self.animation_count = 0
        self.jump_count += 1
        if self.jump_count == 1:
            self.fall_count = 0

    def move(self, dx, dy):
        self.rect.x += dx
        self.rect.y += dy

    def make_hit(self):
        if self.hit is False:
            self.hit = True
            self.health -= 1
        self.hit_count = 0

    def move_left(self, vel):
        self.x_vel = -vel
        if self.direction != "left":
            self.direction = "left"
            self.animation_count = 0

    def move_right(self, vel):
        self.x_vel = vel
        if self.direction != "right":
            self.direction = "right"
            self.animation_count = 0

    def loop(self, fps):
        self.y_vel += min(1, (self.fall_count / fps) * self.GRAVITY)
        self.move(self.x_vel, self.y_vel)

        if self.hit:
            self.hit_count += 1
        if self.hit_count > fps * 0.3:
            self.hit = False

        self.fall_count += 1
        self.update_sprite()

    def landed(self):
        self.fall_count = 0
        self.y_vel = 0
        self.jump_count = 0

    def hit_head(self):
        self.count = 0
        self.y_vel *= -1

    def update_sprite(self):
        sprite_sheet = "run"
        if self.hit:
            sprite_sheet = "hit"
        if self.x_vel == 0:
            sprite_sheet = "idle"
        elif self.y_vel != 0:
            if self.jump_count == 1:
                sprite_sheet = "jump"
            elif self.jump_count == 2:
                sprite_sheet = "double_jump"
        elif self.y_vel > self.GRAVITY * 2:
            sprite_sheet = "fall"

        sprite_sheet_name = sprite_sheet + "_" + self.direction
        sprites = self.SPRITES[sprite_sheet_name]
        sprite_index = (self.animation_count // self.ANIMATION_DELAY) % len(sprites)
        self.sprite = sprites[sprite_index]
        self.animation_count += 1
        self.update()

    def update(self):
        self.rect = self.sprite.get_rect(topleft = (self.rect.x, self.rect.y))
        self.mask = pygame.mask.from_surface(self.sprite)

    def draw_health(self, window, offset_x):
        pos = (self.rect.topleft[0] + 5 - offset_x, self.rect.top - 10)
        size = (50, 10)
        pygame.draw.rect(window, (0, 0, 0), (pos[0] - 2, pos[1] - 2, size[0] + 4, size[1] + 4), 1)
        pygame.draw.rect(window, (255, 0, 0), (pos[0], pos[1], size[0], size[1]))
        pygame.draw.rect(window, (0, 128, 0), (pos[0] + self.health * 10, pos[1], size[0] - self.health * 10, size[1]))

    def draw(self, window, offset_x):
        window.blit(self.sprite, (self.rect.x - offset_x, self.rect.y))
        self.draw_health(window, offset_x)

class Object(pygame.sprite.Sprite):
    def __init__(self, x, y, width, height, name = None):
        super().__init__()
        self.rect = pygame.Rect(x, y, width, height)
        self.image = pygame.Surface((width, height), pygame.SRCALPHA)
        self.width = width
        self.height = height
        self.name = name

    def draw(self, window, offset_x):
        window.blit(self.image, (self.rect.x - offset_x, self.rect.y))

class Block(Object):
    def __init__(self, x, y, size):
        super().__init__(x, y, size, size)
        block = load_block(size)
        self.image.blit(block, (0, 0))
        self.mask = pygame.mask.from_surface(self.image)

class Box(Object):
    def __init__(self, x, y, width, height):
        super().__init__(x, y, width, height)
        path = join("assets", "Items", "Boxes", "Box2", "Idle.png")
        image = pygame.image.load(path).convert_alpha()
        surface = pygame.Surface((width, height), pygame.SRCALPHA, 32)
        rect = pygame.Rect(0, 0, width, height)
        surface.blit(image, (0, 0), rect)
        box = pygame.transform.scale2x(pygame.transform.scale2x(surface))
        self.image.blit(box, (0, 0))
        self.mask = pygame.mask.from_surface(self.image)

class Flag(Object):
    ANIMATION_DELAY = 5

    def __init__(self, x, y, width, height):
        super().__init__(x, y, width, height, "flag")

        path = join("assets", "Items", "Checkpoints", "Checkpoint")
        images = [f for f in listdir(path) if isfile(join(path, f))]

        all_sprites = {}

        for image in images:
            sprite_sheet = pygame.image.load(join(path, image)).convert_alpha()

            sprites = []
            for i in range(sprite_sheet.get_width() // width):
                surface = pygame.Surface((width, height), pygame.SRCALPHA, 32)
                rect = pygame.Rect(i * width, 0, width, height)
                surface.blit(sprite_sheet, (0, 0), rect)
                sprites.append(pygame.transform.scale2x(surface))
                all_sprites[image.replace(".png", "")] = sprites

        self.flag = all_sprites
        self.image = self.flag["Checkpoint (Flag Idle)(64x64)"][0]
        self.mask = pygame.mask.from_surface(self.image)
        self.animation_count = 0

    def loop(self):
        sprites = self.flag["Checkpoint (Flag Idle)(64x64)"]
        sprite_index = (self.animation_count // self.ANIMATION_DELAY) % len(sprites)
        self.image = sprites[sprite_index]
        self.animation_count += 1

        self.rect = self.image.get_rect(topleft = (self.rect.x, self.rect.y))
        self.mask = pygame.mask.from_surface(self.image)

class RockHead(Object):
    ANIMATION_DELAY = 5
    spd = 5

    def __init__(self, x, y, width, height):
        super().__init__(x, y, width, height, "rock_head")
        self.rock_head = load_sprite_sheets("Traps", "Rock Head", width, height)
        self.image = self.rock_head["Blink (42x42)"][0]
        self.mask = pygame.mask.from_surface(self.image)
        self.animation_count = 0

    def loop(self):
        sprites = self.rock_head["Blink (42x42)"]
        sprite_index = (self.animation_count // self.ANIMATION_DELAY) % len(sprites)
        self.image = sprites[sprite_index]
        self.animation_count += 1
        self.rect.y += self.spd

        self.rect = self.image.get_rect(topleft = (self.rect.x, self.rect.y))
        self.mask = pygame.mask.from_surface(self.image)

        if self.animation_count // self.ANIMATION_DELAY > len(sprites):
            self.animation_count = 0

        if self.rect.y >= 600 or self.rect.y <= 150:
            self.spd = -self.spd

class Spikes(Object):
    ANIMATION_DELAY = 2

    def __init__(self, x, y, width, height):
        super().__init__(x, y, width, height, "spikes")
        self.spikes = load_sprite_sheets("Traps", "Spikes", width, height)
        self.image = self.spikes["Idle"][0]
        self.mask = pygame.mask.from_surface(self.image)
        self.animation_count = 0

    def loop(self):
        sprites = self.spikes["Idle"]
        sprite_index = (self.animation_count // self.ANIMATION_DELAY) % len(sprites)
        self.image = sprites[sprite_index]
        self.animation_count += 1

        self.rect = self.image.get_rect(topleft = (self.rect.x, self.rect.y))
        self.mask = pygame.mask.from_surface(self.image)

        if self.animation_count // self.ANIMATION_DELAY > len(sprites):
            self.animation_count = 0

class SpikeHead(Object):
    ANIMATION_DELAY = 5

    def __init__(self, x, y, width, height):
        super().__init__(x, y, width, height, "spike_head")
        self.spike_head = load_sprite_sheets("Traps", "Spike Head", width, height)
        self.image = self.spike_head["Blink (54x52)"][0]
        self.mask = pygame.mask.from_surface(self.image)
        self.animation_count = 0

    def loop(self):
        # sprite_sheet_name = sprite_sheet + "_" + self.direction
        sprites = self.spike_head["Blink (54x52)"]
        sprite_index = (self.animation_count // self.ANIMATION_DELAY) % len(sprites)
        self.image = sprites[sprite_index]
        self.animation_count += 1

        self.rect = self.image.get_rect(topleft = (self.rect.x, self.rect.y))
        self.mask = pygame.mask.from_surface(self.image)

        if self.animation_count // self.ANIMATION_DELAY > len(sprites):
            self.animation_count = 0

class Fire(Object):
    ANIMATION_DELAY = 2

    def __init__(self, x, y, width, height):
        super().__init__(x, y, width, height, "fire")
        self.fire = load_sprite_sheets("Traps", "Fire", width, height)
        self.image = self.fire["off"][0]
        self.mask = pygame.mask.from_surface(self.image)
        self.animation_count = 0
        self.animation_name = "off"

    def on(self):
        self.animation_name = "on"

    def off(self):
        self.animation_name = "off"

    def loop(self):
        # sprite_sheet_name = sprite_sheet + "_" + self.direction
        sprites = self.fire[self.animation_name]
        sprite_index = (self.animation_count // self.ANIMATION_DELAY) % len(sprites)
        self.image = sprites[sprite_index]
        self.animation_count += 1

        self.rect = self.image.get_rect(topleft = (self.rect.x, self.rect.y))
        self.mask = pygame.mask.from_surface(self.image)

        if self.animation_count // self.ANIMATION_DELAY > len(sprites):
            self.animation_count = 0

def get_background(name):
    image = pygame.image.load(join("assets", "Background", name))
    _, _, width, height = image.get_rect()
    tiles = []

    for i in range(WIDTH // width + 2):
        for j in range(HEIGHT // height + 1):
            # From top left, make tuple
            pos = (i*width, j*height)
            tiles.append(pos)

    return tiles, image

def draw(window, background, bg_image, player, objects, offset_x, scroll):
    for tile in background:
        window.blit(bg_image, (tile[0] + scroll, tile[1]))

    for obj in objects:
        obj.draw(window, offset_x)

    player.draw(window, offset_x)

    pygame.display.update()

def handle_vertical_collision(player, objects, dy):
    collided_objects = []
    for obj in objects:
        if pygame.sprite.collide_mask(player, obj):
            if dy > 0:
                player.rect.bottom = obj.rect.top
                player.landed()
            elif dy < 0:
                player.rect.top = obj.rect.bottom
                player.hit_head()

            collided_objects.append(obj)

    return collided_objects

def collide(player, objects, dx):
    player.move(dx, 0)
    player.update()
    collided_object = None
    for obj in objects:
        if pygame.sprite.collide_mask(player, obj):
            collided_object = obj
            break

    player.move(-dx, 0)
    player.update()
    return collided_object

def handle_move(player, objects):
    keys = pygame.key.get_pressed()

    player.x_vel = 0
    collide_left = collide(player, objects, -PLAYER_VEL * 2)
    collide_right = collide(player, objects, PLAYER_VEL * 2)

    if keys[pygame.K_LEFT] and not collide_left:
        player.move_left(PLAYER_VEL)
    if keys[pygame.K_RIGHT] and not collide_right:
        player.move_right(PLAYER_VEL)

    vertical_collide = handle_vertical_collision(player, objects, player.y_vel)
    to_check = [collide_left, collide_right, *vertical_collide]
    trap_name = ["fire", "spikes", "spike_head"]
    for obj in to_check:
        if obj and obj.name in trap_name:
            player.make_hit()
        if obj and obj.name == "flag":
            player.win = True

def draw_start_menu():
    window.fill((255, 255, 255))
    image = pygame.image.load(join("assets", "Sign", "ToBegin.png")).convert_alpha()
    window.blit(image, (WIDTH/2 - image.get_width()/2, HEIGHT/2 - image.get_height()/2))
    pygame.display.update()

def draw_game_over():
    window.fill((255, 255, 255))
    image = pygame.image.load(join("assets", "Sign", "GameOver.png")).convert_alpha()
    window.blit(image, (WIDTH/2 - image.get_width()/2, HEIGHT/2 - image.get_height()/2))
    pygame.display.update()

def draw_win():
    window.fill((255, 255, 255))
    image = pygame.image.load(join("assets", "Sign", "YouWin.jpg")).convert_alpha()
    window.blit(image, (WIDTH/2 - image.get_width()/2, HEIGHT/2 - image.get_height()/2))
    pygame.display.update()

def main(window):
    game_state = "start_menu"

    clock = pygame.time.Clock()
    block_size = 96

    background, bg_image = get_background("Cloud.jpg")
    player = Player(100, 100, 50, 50)

    wall = [Block(0, HEIGHT - block_size*(i+1), block_size) for i in range(HEIGHT // block_size + 1)]
    floor = [Block(i*block_size, HEIGHT - block_size, block_size) for i in range(0, (WIDTH*2) // block_size + 3)]

    boxes = []
    for i in range(7):
        for j in range(i):
            boxes.append(Box(block_size*14 + 80*j, HEIGHT - block_size - 75*(8-i-1) - 12, 112, 96))

    platform1 = [Block(block_size*2 + block_size*i, HEIGHT - block_size*3.6, block_size) for i in range(3)]
    platform2 = [Block(block_size*7 + block_size*i, HEIGHT - block_size*5.5, block_size) for i in range(3)]
    platform3 = [Block(block_size*7 + block_size*i, HEIGHT - block_size*3, block_size) for i in range(3)]

    rock_heads = [RockHead(block_size*10.5 + 64, 550, 42, 42), RockHead(block_size*10.5 + 128, 200, 42, 42), RockHead(block_size*10.5 + 192, 350, 42, 42)]

    fires = [Fire(block_size*7, HEIGHT - block_size*5.5 - 64, 16, 32), Fire(block_size*15.5 - 16, HEIGHT - block_size*4 - 64, 16, 32)]

    spikes1 = [Spikes(block_size + 16*i, HEIGHT - block_size - 32, 8, 16) for i in range(100)]
    traps = [*spikes1, SpikeHead(block_size*5 + 20, HEIGHT - block_size*4, 54, 52)]

    flag = Flag(block_size*14 + 640, HEIGHT - block_size - 128, 64, 64)

    for fire in fires:
        fire.on()

    objects = [*floor, *wall, *platform1, *platform2, *platform3, *fires, *traps, *rock_heads, *boxes, flag]

    run = True

    offset_x = 0
    scroll_area_width = 200
    scroll = 0

    while run:
        # To regulate the frame rate accross different devices
        clock.tick(FPS)

        # event handler
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                run = False
                break
            if event.type == pygame.KEYDOWN:
                if event.key == pygame.K_SPACE and player.jump_count < 2:
                    player.jump()

        if game_state == "start_menu":
            draw_start_menu()
            keys = pygame.key.get_pressed()
            if keys[pygame.K_SPACE]:
                game_state = "game"

        if game_state == "game_over":
            draw_game_over()
            keys = pygame.key.get_pressed()
            if keys[pygame.K_q]:
                pygame.quit()
                quit()

        if player.win:
            game_state = "win"
            draw_win()
            keys = pygame.key.get_pressed()
            if keys[pygame.K_q]:
                pygame.quit()
                quit()

        if game_state == "game":
            if player.health <= 0:
                game_state = "game_over"

            scroll -= 3
            # reset scroll
            if abs(scroll) > WIDTH:
                scroll = 0

            player.loop(FPS)
            flag.loop()
            for fire in fires:
                fire.loop()
            for trap in traps:
                trap.loop()
            for rock_head in rock_heads:
                rock_head.loop()

            if((player.rect.right - offset_x >= WIDTH - scroll_area_width) and player.x_vel > 0) or (
                (player.rect.left - offset_x <= scroll_area_width) and player.x_vel < 0):
                offset_x += player.x_vel

            handle_move(player, objects)
            draw(window, background, bg_image, player, objects, offset_x, scroll)

    pygame.quit()
    quit()

# Only call the function when the file is runned directly
# So importing something from the file won't run the game code
if __name__ == "__main__":
    main(window)


from http import HTTPStatus

from django.urls import reverse_lazy, reverse

from members.models import Member
from members.tests.test_member_base import MemberTestCase

class MemberListViewTest(MemberTestCase):

    def setUp(self):
        super().setUp()
        self.url = reverse_lazy('members:list')

    def main_page_redirect_here(self):
        response = self.client.get(reverse_lazy('main'))
        self.assertRedirects(response, self.url)

    def test_get_list_page(self):
        response = self.client.get(self.url)
        self.assertEqual(response.status_code, HTTPStatus.OK)
        self.assertContains(response, 'Team members')
        self.assertContains(response, 'You have 0 team members.')
        member = Member.objects.create(**self.member1_data)
        response = self.client.get(self.url)
        self.assertContains(response, 'You have 1 team members.')
        self.assertContains(response, reverse('members:edit', args=[member.id]))
        self.assertQuerysetEqual(response.context_data['members'],
                                 Member.objects.order_by('-date_joined').exclude(is_superuser=True))

        member = Member.objects.create(**self.member2_data)
        response = self.client.get(self.url)
        self.assertContains(response, 'You have 2 team members.')
        self.assertContains(response, reverse('members:edit', args=[member.id]))
        self.assertQuerysetEqual(response.context_data['members'],
                                 Member.objects.order_by('-date_joined').exclude(is_superuser=True))

        Member.objects.get(pk=member.id).delete()
        response = self.client.get(self.url)
        self.assertContains(response, 'You have 1 team members.')
        self.assertNotContains(response, reverse('members:edit', args=[member.id]))


from mrjob.job import MRJob
import re
import time
import datetime

#This line declares the class PartBJob1, that extends the MRJob format.
class GasPrice_Time(MRJob):

# this class will define two additional methods: the mapper method goes here:
    def mapper(self, _, line):
        try:
            fields = line.split(",")
            if len(fields) == 7:
                time_epoch = int(fields[6])
                month = time.strftime("%m",time.gmtime(time_epoch)) #returns each month
                year = time.strftime("%Y",time.gmtime(time_epoch)) #returns each year
                key = (year, month)
                gas_price = int(fields[5])
                yield (key, (gas_price, 1))
        except:
            pass

    def combiner(self, key, gas_price_sum):
        gas_price_total = 0
        count_total = 0
        for n in gas_price_sum:
            gas_price_total += int(n[0])
            count_total += int(n[1])
        yield (key, (gas_price_total, count_total))

#and the reducer method goes after this line:
    def reducer(self, key, gas_price_sum):
        gas_price_total = 0
        count_total = 0
        for n in gas_price_sum:
            gas_price_total += int(n[0])
            count_total += int(n[1])
        yield (key, gas_price_total / count_total)

#

if __name__ == '__main__':
    GasPrice_Time.run()


#!/usr/bin/env python
import pandas as pd
import rospy
import csv
import sys
from move_base_msgs.msg import MoveBaseActionGoal
from visualization_msgs.msg import Marker
from std_msgs.msg import Int32
from nav_msgs.msg import Path
#from visualization_msgs.msg import Marker1

predata=pd.read_csv('~/catkin_ws/src/kcctslam/config/waypointdata/wpdata.csv')
# get timestamp
from datetime import datetime as dt
tdatetime = dt.now()
tstr = tdatetime.strftime('%y%m%d_%H%M%S')
# output to excel file
predata.drop(predata.columns[[0]], axis=1,inplace=True)
predata.to_csv('~/catkin_ws/src/kcctslam/config/waypointdata/wpdata'+tstr+".csv")

df=pd.DataFrame(columns=['x', 'y','z','qx','qy','qz','qw','type','map'])
pub = rospy.Publisher("waypoint", Marker, queue_size = 10)
pub1 = rospy.Publisher("waypoint1", Marker, queue_size = 10)
i=0
map_num=0
path=Path

def mapcallback(map_num_row):
    global map_num
    map_num=map_num_row.data

def pathcallback(path_row):
    global path
    path=path_row

def listener():
    global path
    rospy.init_node('waypoint_manager', anonymous=True)

#    rospy.Subscriber("/move_base/goal", MoveBaseActionGoal, callback)
    rospy.Subscriber("/map_num", Int32, mapcallback)
    rospy.Subscriber("/Path",Path,pathcallback)
    rospy.spin()
    i=0
    for j in path.poses:
        df.loc[i] = [j.pose.position.x,j.pose.position.y,j.pose.position.z,j.pose.orientation.x,j.pose.orientation.y,j.pose.orientation.z,j.pose.orientation.w,1,map_num]
        df.to_csv('~/catkin_ws/src/kcctslam/config/waypointdata/wpdata.csv', header=True)
        i+=1
    print(" WayPoint SAVED!!")

if __name__ == '__main__':
    listener()


"""backend URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.11/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.conf.urls import url, include
    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
"""
from django.conf.urls import url, include
from django.contrib import admin
from rest_framework.authtoken import views as authviews
from model.views import casesFilter, getNeighborhoodsByCity, getCasesByCityNeighborhood, getNumberCasesByNeighborhoodCityYear,getNeighborhoodsToPredict,getPrediction, updateCSV

urlpatterns = [
    url(r'^admin/', admin.site.urls),
    url(r'^api/', include('model.urls')),
    url(r'^auth/', authviews.obtain_auth_token),
    url(r'^api/cases/', casesFilter),
    url(r'^api/neighborhoods/(?P<city_searched>\w+)/$', getNeighborhoodsByCity),
    url(r'^api/datesChart/', getCasesByCityNeighborhood),
    url(r'^api/casesCount/', getNumberCasesByNeighborhoodCityYear),
    url(r'^api/neighborhoods/predict/(?P<city_searched>\w+)/$', getNeighborhoodsToPredict),
    url(r'^api/prediction/', getPrediction),
    url(r'^api/updateCSV/', updateCSV)
]


"""
Numba-accelerated functions for filtering with `scipy.LowLevelallable`

Inspired by the beautifully simply example from Juan Nunez-Iglesias
https://ilovesymposia.com/2017/03/12/scipys-new-lowlevelcallable-is-a-game-changer/

>>> import numpy as np
>>> import scipy.ndimage as nd
>>> import grizli.nbutils
>>> arr = np.random.normal(size=1024)
>>> footprint = np.ones(51, dtype=int)
>>> # Slow with numpy overheads
>>> np_filtered = nd.generic_filter(arr, np.nanmedian, footprint=footprint))
>>> # Much faster!
>>> filtered = nd.generic_filter(arr, grizli.nbutils.nanmedian, footprint=footprint))

"""

import numpy as np

from numba import cfunc, carray
from numba.types import intc, intp, float64, voidptr
from numba.types import CPointer

from scipy import LowLevelCallable

@cfunc(intc(CPointer(float64), intp, CPointer(float64), voidptr))
def nb_nanmin(values_ptr, len_values, result, data):
    """
    Manual minimum
    """
    values = carray(values_ptr, (len_values,), dtype=float64)
    result[0] = np.inf
    for v in values:
        if (v < result[0]) & ~np.isnan(v):
            result[0] = v
    return 1

@cfunc(intc(CPointer(float64), intp, CPointer(float64), voidptr))
def nb_nanmax(values_ptr, len_values, result, data):
    """
    Manual minimum
    """
    values = carray(values_ptr, (len_values,), dtype=float64)
    result[0] = -np.inf
    for v in values:
        if (v > result[0]) & ~np.isnan(v):
            result[0] = v
    return 1

@cfunc(intc(CPointer(float64), intp, CPointer(float64), voidptr))
def nb_npnanmin(values_ptr, len_values, result, data):
    """
    numpy.nanmin
    """
    values = carray(values_ptr, (len_values,), dtype=float64)
    result[0] = np.nanmin(values)
    return 1

@cfunc(intc(CPointer(float64), intp, CPointer(float64), voidptr))
def nb_npnanmedian(values_ptr, len_values, result, data):
    """
    numpy.nanmedian
    """
    values = carray(values_ptr, (len_values,), dtype=float64)
    result[0] = np.nanmedian(values)
    return 1

@cfunc(intc(CPointer(float64), intp, CPointer(float64), voidptr))
def nb_npnanmean(values_ptr, len_values, result, data):
    """
    numpy.nanmean
    """
    values = carray(values_ptr, (len_values,), dtype=float64)
    result[0] = np.nanmean(values)

    return 1

@cfunc(intc(CPointer(float64), intp, CPointer(float64), voidptr))
def nb_npnansum(values_ptr, len_values, result, data):
    """
    numpy.nansum
    """
    values = carray(values_ptr, (len_values,), dtype=float64)
    result[0] = np.nansum(values)

    return 1

# Callables for scipy.ndimage.generic_filter
nanmin = LowLevelCallable(nb_nanmin.ctypes)
nanmax = LowLevelCallable(nb_nanmax.ctypes)
nbnanmin = LowLevelCallable(nb_npnanmin.ctypes)
nanmedian = LowLevelCallable(nb_npnanmedian.ctypes)
nanmean = LowLevelCallable(nb_npnanmean.ctypes)
nansum = LowLevelCallable(nb_npnansum.ctypes)


import os
import sys
from termcolor import colored
from getpass import getpass
from re import match
from platform import system
import shlex
import keyboard
import time

sys.path.append(os.path.abspath("../files"))
from internal import info as inf

def clear():
    os.system('cls' if os.name == 'nt' else 'clear')

clear()

def fprint(*values: object, sep: str | None = " ", end: str | None = "\n", config: dict = {"color": "blue", "start": "|"}) -> None:
    if len(config["start"]) <= 1:
        config["start"] += " "*(2-len(config["start"]))
    if type(values[0]) == list: text = sep.join(values[0])
    else: text = sep.join(values)
    values = colored(config["start"], config["color"]) + text
    print(values, end=end)
def fprintTitle(*values: object, sep: str | None = " ", end: str | None = "\n", config: dict = {"color": "blue", "start": "+- "}) -> None:
    if len(config["start"]) <= 1:
        config["start"] += " "*(2-len(config["start"]))
    if type(values[0]) == list: text = sep.join(values[0])
    else: text = sep.join(values)
    if str(config["start"]).endswith(" "):
        text += " "
    values = colored(config["start"], config["color"]) + text + colored("-"*(os.get_terminal_size().columns-len(text)-len(config["start"])), config["color"])
    print(values, end=end)
def fline(config: dict = {"color": "blue", "start": "+"}) -> None:
    text = colored(config["start"] + "-"*(os.get_terminal_size().columns-len(config["start"])), config["color"])
    print(text)
def finput(__prompt: object = "", config: dict = {"color": "blue", "start": "|"}) -> str:
    if len(config["start"]) <= 1:
        config["start"] += " "*(2-len(config["start"]))
    text = colored(config["start"], config["color"]) + str(__prompt)
    return input(text)
def fpause(value: object = colored("Press enter to continue... ", attrs=["dark"]), config: dict = {"color": "blue", "start": "|"}) -> None:
        if value == colored("Press enter to continue... ", attrs=["dark"]) and not os.name == "nt":
            value = colored("(Press enter to continue...) ")
            config["start"] = ">"
            config["color"] = "red"
        if len(config["start"]) <= 1:
            config["start"] += " "*(2-len(config["start"]))
        text = colored(config["start"], config["color"]) + str(value)
        return getpass(text)

# << MAIN CODE >>

fprintTitle("MegaShell Compiler")
fprint("Made by: @JHubi1 (GitHub)")
fprint("Version: 1.0.0")
fline()
appVersion = ""
while not match(r"[0-9]\.[0-9]\.[0-9](?:-)?[a-zA-Z]{0,10}", appVersion):
    keyboard.write(inf.VERSION)
    appVersion = finput(colored("Enter the version of the app: ", "yellow"))
path = os.path.abspath(os.getcwd() + "/output/" + appVersion + "/" + system().lower())
# file deepcode ignore UpdateAPI: input from user; user = dev
if os.path.exists(path):
    fline()
    fprint("The directory already exists!")
    if finput("Continue? [y/N]: ").lower() == "y":
        fprint("Continuing...")
    else:
        fprint("Aborting...")
        fpause()
        exit()
os.makedirs(path, exist_ok=True)
# file deepcode ignore PT: input from user; user = dev
os.chdir(path)

start_time = time.time()

commands = [
    'pyinstaller --noconfirm --onedir --console --icon "' + os.path.abspath("../../../../images/^logos/logo.ico") + '" --name "MegaShell" --add-data "' + os.path.abspath("../../../../commands") + ';commands" --add-data "' + os.path.abspath("../../../../config") + ';config" "' + os.path.abspath("../../../../main.py") + '";;;pyinstaller --noconfirm --onedir --console --name "MegaShell" --add-data "' +  shlex.quote(os.path.abspath("../../../../commands")) + ';commands" --add-data "' +  shlex.quote(os.path.abspath("../../../../config")) + ';config" "' +  shlex.quote(os.path.abspath("../../../../main.py")) + '"',

    'xcopy /E /I /Y "' + os.path.abspath("dist/MegaShell") + '" "' + os.path.abspath("MegaShell") + '";;;cp -R ' + os.path.abspath("dist/MegaShell") + ' ' + os.path.abspath("MegaShell"),
    'del /F "' + os.path.abspath("MegaShell.spec") + '";;;rm "' + os.path.abspath("MegaShell.spec") + '"',
    'rmdir /S /Q "' + os.path.abspath("dist") + '";;;rm -rf "' + os.path.abspath("dist") + '"',
    'rmdir /S /Q "' + os.path.abspath("build") + '";;;rm -rf "' + os.path.abspath("build") + '"',

    # IMPORTANT: ISCC must be in path!
    'ISCC /O+ /DMyAppVersion="' + appVersion + '" /DMainPath="' + os.path.abspath("../../../..") + '" /DExeVersion="' + appVersion.replace(" ", "").replace(".", "-") + '" "' + os.path.abspath("../../../installer/windows/MegaShellSetup.iss") + '";;;echo Installer creation not supported in Linux.'
]

fline()
for command in commands:
    command = command.split(";;;")
    if len(command) != 1:
        if system().lower() == "windows":
            command = command[0]
        elif system().lower() == "linux":
            command = command[1]
    else: command = command[0]
    print(colored("Executing: `" + command + "`", "green"))
    # deepcode ignore CommandInjection: commands from list
    os.system(command)
    fline()

end_time = time.time() - start_time

fprint("Compilation finished!")
fprint(colored("Total time: " + str(round(end_time, 2)) + "s", "green"))

fpause()
clear()


from telebot import types
from di_configuration import DIBot
from .open_menu import open_menu as menu

def about_dev(message):
    bot = DIBot.di_bot()
    inline_key = types.InlineKeyboardMarkup()
    url_button = types.InlineKeyboardButton(text="ПРИСОЕДИНЯЙСЯ!", url="https://vk.com/itclub_psuti")
    inline_key.add(url_button)
    about_msg = bot.send_message(
        message.chat.id,
        text='Чат-бот разработан студентами <b>ИТ-клуба ПГУТИ</b> для будующих студентов.'
             '\nБудем рады увидеть Вас в наших рядах!'
             '\nhttps://pp.vk.me/c629125/v629125487/21029/Qd9czt4U02E.jpg',
        parse_mode='HTML',
        reply_markup=inline_key)
    bot.register_next_step_handler(about_msg, open_menu)

def open_menu(message):
    menu(message)

handlers = {'about': about_dev}


import re
import os
import json
import scrapy
import datetime
from pathlib import Path
from dotenv import load_dotenv

try:
    load_dotenv()
except:
    env_path = Path(".env")
    load_dotenv(dotenv_path=env_path)
class SephoraUKPdpSpider(scrapy.Spider):
    name = 'sephora_uk_pdp_weekly'
    start_urls = ['https://www.sephora.co.uk/']
    CURRENT_DATETIME = datetime.datetime.now()
    CURRENT_DATE = CURRENT_DATETIME.strftime("%Y-%m-%d")
    DATE=CURRENT_DATE.replace("-","_")

    ROTATING_PROXY = os.getenv("ROTATING_PROXY")
    custom_settings={
        # 'FEEDS' : {f"s3://scraping-external-feeds-lapis-data/{CURRENT_DATE}/sephora_uk/%(name)s_{DATE}.json": {"format": "json",}},

    "CONCURRENT_REQUESTS":64,
    'ROTATING_PROXY_LIST' : [ROTATING_PROXY],
    "HTTPCACHE_ENABLED" : True,
    "HTTPCACHE_DIR" : 'httpcache',
    'HTTPCACHE_EXPIRATION_SECS':79200,
    "HTTPCACHE_IGNORE_HTTP_CODES":[502],
    "HTTPCACHE_STORAGE" : "scrapy.extensions.httpcache.FilesystemCacheStorage",
    }
    def parse(self, response):
        spider_name = SephoraUKPdpSpider.name.replace('pdp_weekly','categories')
        current_date = datetime.datetime.now().strftime("%Y-%m-%d")
        category_path = os.getcwd() + f'/exports/{spider_name}/{spider_name}_{current_date}.json'
        with open(category_path,'r',encoding='utf-8-sig') as f:
            json_file = f.read()
        category_names = json.loads(json_file)
        for main_cat in category_names:
            category_name = main_cat.get('name','').title()
            category_url = main_cat.get('url','')
            for main_sub_cat in main_cat['category_crumb']:
                if 'category_crumb' in (main_sub_cat).keys():
                    main_category = main_sub_cat.get('name','').title()
                    main_category_url = main_sub_cat.get('url','')
                    for sub_cat in main_sub_cat['category_crumb']:
                        sub_category = sub_cat.get('name','').title()
                        sub_cat_url = sub_cat.get('url','')
                        yield scrapy.Request(url=(sub_cat_url),callback=self.parse_list,dont_filter=True,cb_kwargs={"category_name":category_name,"category_url":category_url,"main_category":main_category,"main_category_url":main_category_url,"sub_category":sub_category,"sub_cat_url":sub_cat_url})
                else:
                    main_category = main_sub_cat.get('name','').title()
                    main_category_url = main_sub_cat.get('url','')
                    sub_category = ''
                    sub_cat_url = ""
                    yield scrapy.Request(url=(main_category_url),callback=self.parse_list,dont_filter=True,cb_kwargs={"category_name":category_name,"category_url":category_url,"main_category":main_category,"main_category_url":main_category_url,"sub_category":sub_category,"sub_cat_url":sub_cat_url})

    def parse_list(self,response,category_name,category_url,main_category,main_category_url,sub_category,sub_cat_url):
        if response.xpath('//div[@class="eba-component eba-product-listing"]/div[contains(@class,"Product")]/a/@href'):
            for product_list_xpath in response.xpath('//div[@class="eba-component eba-product-listing"]/div[contains(@class,"Product")]/a'):
                product_url =  product_list_xpath.xpath('./@href').get()
                yield scrapy.Request(url=response.urljoin(product_url),callback=self.product_detail,dont_filter=True,cb_kwargs={"category_name":category_name,"category_url":category_url,"main_category":main_category,"main_category_url":main_category_url,"sub_category":sub_category,"sub_cat_url":sub_cat_url})
        nexpage_url = response.xpath('//div[@class="loadMore loadMoreBottom"]/a/@href').get()
        if nexpage_url:
           yield scrapy.Request(url=response.urljoin(nexpage_url),callback=self.parse_list,dont_filter=True,cb_kwargs={"category_name":category_name,"category_url":category_url,"main_category":main_category,"main_category_url":main_category_url,"sub_category":sub_category,"sub_cat_url":sub_cat_url})
    async def product_detail(self,response,category_name,category_url,main_category,main_category_url,sub_category,sub_cat_url):
        item = {}
        item['url'] = response.url
        category_crumb = []
        main_cat = {}
        main_cat['name'] = category_name
        main_cat['url'] = category_url
        category_crumb.append(main_cat)
        sub_main_cat = {}
        sub_main_cat['name'] = main_category
        sub_main_cat['url'] = main_category_url
        category_crumb.append(sub_main_cat)
        if sub_category and sub_cat_url:
            sub_cat = {}
            sub_cat['name'] = sub_category
            sub_cat['url'] = sub_cat_url
            category_crumb.append(sub_cat)
        item['category_crumb'] = category_crumb
        if response.xpath('//h1/span[@class="pdp-product-brand-name"]/text()').get('').strip():
            item['name'] = response.xpath('//h1/span[@class="pdp-product-brand-name"]/text()').get('').strip()
            item['subtitle'] = None
            image_url_list = list(set(response.xpath("//div[contains(@class,'productpage-gallery')]/div/a/@href|//div[contains(@class,'productpage-image')]/img/@src").getall()))
            image_urls = [ url.replace(' ','%20') for url in image_url_list if not 'youtube' in url ]
            image_urls = [ url.replace(' ','%20') for url in image_urls if not 'player.' in url ]
            video_url = [ url for url in image_url_list if 'youtube' in url ]
            video_url_player = [ url for url in image_url_list if 'player.' in url ]
            if video_url_player:
                video_url.extend(video_url_player)
            item['image_url'] = image_urls
            item['has_video'] = False
            item["video"] = []
            if video_url:
                item['has_video'] = True
                video_urls = []
                for video_url_http in video_url:
                    if not 'http' in video_url_http:
                        video_url_http = f'https:{video_url_http}'
                        video_urls.append(video_url_http)
                    else:
                        video_urls.append(video_url_http)
                item["video"] = video_urls
            item['brand'] = None
            if re.search(r'<script\s*type\=\"application\/ld\+json\"[^>]*?>[\w\W]*?\"brand\"\:\s*\"([^>]*?)\"[^>]*?<\/script>',response.text):
                item['brand'] = re.findall(r'<script\s*type\=\"application\/ld\+json\"[^>]*?>[\w\W]*?\"brand\"\:\s*\"([^>]*?)\"[^>]*?<\/script>',response.text)[0]
            item['master_product_id'] = None
            item['gtin'] = None
            if  re.search('\"data\"\:\[\"([^>]*?)\"\]',response.text):
                item['master_product_id'] = re.findall('\"data\"\:\[\"([^>]*?)\"\]',response.text)[0]
            if re.search(r'data\-flix\-ean\=\"([^>]*?)\"',response.text):
                gtin = re.findall(r'data\-flix\-ean\=\"([^>]*?)\"',response.text)[0]
                item['gtin'] = gtin if gtin != "" else None

            in_stock = response.xpath('//span[@class="product-options-stock u-error"]/text()|//div[contains(@class,"stock-level")]/text()|//span[@class="product-options-stock"]/text()|//div[@class="sub-products"]/span[@class="option selected"]/span/span[contains(@class,"priceStock")]/span[2]/text()').get("")
            item['in-stock'] = True if 'in stock' in in_stock.lower().strip() else False

            item['price'] = None
            if response.xpath('//p[@class="price-info"]/span/span/text()'):
                item['price'] = ''.join(response.xpath('//p[@class="price-info"]/span/span/text()').getall()).strip()
            price_before = response.xpath('//p[@class="price-info"]/span/span[@class="Price-details"]/span[@class="rrp"]/text()').get('').strip()
            item['price_before'] = price_before if price_before else None
            promo_label = response.xpath('//div[@class="discountWrapper"]/span/text()').get('').strip()
            item['promo_label'] = promo_label if promo_label else None
            prices = []
            for variant_block in response.xpath('//div[@class="product-detail-information productpage-buy"]//div[@class="sub-products"]/span'):
                variant_dict = {}
                variant_dict['sku_id'] = variant_block.xpath('./@data-sku').get()
                variant_dict['gtin'] = item['gtin']
                in_stock = variant_block.xpath('./span/span[@class="priceStock "]/span[2]/text()').get('')
                variant_dict['in-stock'] = True if 'in stock' in in_stock.lower().strip() else False
                variant_dict['image_url'] = image_urls
                variant_dict['has_video'] = item['has_video']
                variant_dict['video'] = item["video"]
                variant_price = variant_block.xpath('./span/span[contains(@class,"priceStock")]/span[1]/text()').get()
                variant_dict['price'] = variant_price.strip() if variant_price else None
                variant_dict['price_before'] = item['price_before']
                variant_dict['promo_label'] = item['promo_label']
                variant_dict['variant_url'] = response.url
                variant_dict['data_color'] = variant_block.xpath('./span/span[@class="name"]/text()').get()
                variant_dict['data_size'] = None
                prices.append(variant_dict)
            item['prices'] = prices
            description = ' '.join(response.xpath('//section[@id="information"]/div//text()').getall())
            description = re.sub(r'\s+',' ',description)
            item['description'] = description.strip() if description else None

            pass_key_url = 'https://display.ugc.bazaarvoice.com/static/sephora-uk/main_site/en_GB/bvapi.js'
            pass_key_response = await self.requests_process(pass_key_url)
            pass_key,display_code='',''
            if re.search(r'passkey\:\"([^>]*?)\"',pass_key_response.text):
                pass_key = re.findall(r'passkey\:\"([^>]*?)\"',pass_key_response.text)[0]
            if re.search(r'displaycode\:\"([^>]*?)\"',pass_key_response.text):
                display_code = re.findall(r'displaycode\:\"([^>]*?)\"',pass_key_response.text)[0]

            review_url = f'https://api.bazaarvoice.com/data/batch.json?passkey={pass_key}&apiversion=5.5&displaycode={display_code}&resource.q0=products&filter.q0=id%3Aeq%3A{item["master_product_id"]}&stats.q0=reviews&filteredstats.q0=reviews&filter_reviews.q0=contentlocale%3Aeq%3Aen*%2Cfr*%2Cen_GB&filter_reviewcomments.q0=contentlocale%3Aeq%3Aen*%2Cfr*%2Cen_GB&resource.q1=reviews&filter.q1=isratingsonly%3Aeq%3Afalse&filter.q1=productid%3Aeq%3A{item["master_product_id"]}&filter.q1=contentlocale%3Aeq%3Aen*%2Cfr*%2Cen_GB&sort.q1=relevancy%3Aa1&stats.q1=reviews&filteredstats.q1=reviews&include.q1=authors%2Cproducts%2Ccomments&filter_reviews.q1=contentlocale%3Aeq%3Aen*%2Cfr*%2Cen_GB&filter_reviewcomments.q1=contentlocale%3Aeq%3Aen*%2Cfr*%2Cen_GB&filter_comments.q1=contentlocale%3Aeq%3Aen*%2Cfr*%2Cen_GB&limit.q1=100&offset.q1=0&limit_comments.q1=3&callback=BV._internal.dataHandler0'
            review_response = await self.requests_process(review_url)
            review_list = []
            if re.search(r'BV\._internal\.dataHandler0\(([^>]*?)\)$',review_response.text):
                review_block = re.findall(r'BV\._internal\.dataHandler0\(([^>]*?)\)$',review_response.text)[0]
                review_json = json.loads(review_block)
                review_list = self.review(review_json,review_list)
                count = review_json.get('BatchedResults').get('q1').get('TotalResults')
                count = count if count else 0
                for page_count in range(1,round(int(count)/100)+1):
                    review_next_page_url = f'https://api.bazaarvoice.com/data/batch.json?passkey={pass_key}&apiversion=5.5&displaycode={display_code}&resource.q0=products&filter.q0=id%3Aeq%3A{item["master_product_id"]}&stats.q0=reviews&filteredstats.q0=reviews&filter_reviews.q0=contentlocale%3Aeq%3Aen*%2Cfr*%2Cen_GB&filter_reviewcomments.q0=contentlocale%3Aeq%3Aen*%2Cfr*%2Cen_GB&resource.q1=reviews&filter.q1=isratingsonly%3Aeq%3Afalse&filter.q1=productid%3Aeq%3A{item["master_product_id"]}&filter.q1=contentlocale%3Aeq%3Aen*%2Cfr*%2Cen_GB&sort.q1=relevancy%3Aa1&stats.q1=reviews&filteredstats.q1=reviews&include.q1=authors%2Cproducts%2Ccomments&filter_reviews.q1=contentlocale%3Aeq%3Aen*%2Cfr*%2Cen_GB&filter_reviewcomments.q1=contentlocale%3Aeq%3Aen*%2Cfr*%2Cen_GB&filter_comments.q1=contentlocale%3Aeq%3Aen*%2Cfr*%2Cen_GB&limit.q1=100&offset.q1={100*page_count}&limit_comments.q1=3&callback=BV._internal.dataHandler0'
                    next_page_review_response = await self.requests_process(review_next_page_url)
                    if re.search(r'BV\._internal\.dataHandler0\(([^>]*?)\)$',next_page_review_response.text):
                        next_review_block = re.findall(r'BV\._internal\.dataHandler0\(([^>]*?)\)$',next_page_review_response.text)[0]
                        next_review_json = json.loads(next_review_block)
                        review_list = self.review(next_review_json,review_list)

            item["reviews"] = review_list
            yield item

    def review(self,response,review_list):
        for review_values in response.get('BatchedResults',{}).get('q1',{}).get('Results'):
            review_dict = {}
            review_dict['user'] = review_values.get('UserNickname')
            review_dict['date'] = review_values.get('SubmissionTime','').split('T')[0]
            review_dict['stars'] = review_values.get('Rating')
            review_title = review_values.get('Title')
            review_body = review_values.get('ReviewText')
            review_dict['review'] = f"[{review_title}] {review_body}"
            review_list.append(review_dict)
        return review_list

    async def requests_process(self,url):
        request = scrapy.Request(url)
        response = await self.crawler.engine.download(request, self)
        return response