import json
import typing

import audio_processing
import visual

import audioread.exceptions
import os
import pathlib
from matplotlib import pyplot as plt
import shutil
import random
from dataclasses import dataclass
import numpy as np
import pandas as pd
import datetime

GTZANLabels = ["blues", "classical", "country", "disco", "hiphop", "jazz", "metal", "pop", "reggae", "rock"]

@dataclass
class DatasetSplitDict:
    split_dict: typing.Dict[str, typing.Dict[str, typing.List[pathlib.Path]]]
    splits_names = ["train", "test", "val"]

    def __init__(self, original_genres_path: str = None):
        self.split_dict = {
            "train": {},
            "test": {},
            "val": {}
        }
        self.original_genres_path = original_genres_path

    def add_genre_record(self, genre_name: str, record: typing.Dict[str, typing.List[pathlib.Path]]) -> None:
        assert(sorted(DatasetSplitDict.splits_names) == sorted(list(record.keys())))

        for split in DatasetSplitDict.splits_names:
            self.split_dict[split].update(
                {genre_name: record[split]}
            )

    def to_pandas_dataframes(self) -> typing.Dict[str, pd.DataFrame]:
        splits_dataframes = dict()
        for split in self.split_dict.keys():
            split_dataframe = pd.DataFrame.from_dict(data=self.split_dict[split])
            splits_dataframes.update({split: split_dataframe})

        return splits_dataframes

def ratio_to_rounded_indices(ratio: typing.Tuple[float, float, float],
                             whole_dir_num: int
                             ) -> typing.Tuple[int, int, int]:

    # Lists of numbers of files inside split (train, test and val) calculated from ratio,
    # as floats and as integers after floor
    nums_f = [ratio[0] * whole_dir_num, ratio[1] * whole_dir_num, ratio[2] * whole_dir_num]
    nums = [int(num_f) for num_f in nums_f]

    # Check if flooring the floats made the sum lower than all the files to use
    unassigned_files = whole_dir_num - sum(nums)
    assert(unassigned_files >= 0)

    # Choose the splits with biggest decimals as the candidates for incrementation to assign the unassigned files
    if unassigned_files != 0:
        for idx in np.argsort(nums_f)[-unassigned_files:]:
            nums[idx] += 1

    # indices for slicing the list of all files inside directory to create the splits
    train_idx = nums[0]
    test_idx = nums[0] + nums[1]
    val_idx = nums[0] + nums[1] + nums[2]

    return train_idx, test_idx, val_idx

def shuffle_file_list(fl: typing.List[pathlib.Path], seed: typing.Any = None) -> typing.List[pathlib.Path]:
    if seed is not None:
        gen = random.Random(seed)
    else:
        gen = random.Random(datetime.datetime.now())
    shuffled = fl.copy()
    gen.shuffle(shuffled)
    return shuffled

def get_splits_for_genre(genre_dir: str,
                         shuffle: bool = True,
                         seed: int = None,
                         train_ratio: float = 0.70,
                         test_ratio: float = 0.20,
                         val_ratio: float = 0.10
                         ) -> typing.Dict[str, typing.List[pathlib.Path]]:

    assert(round(train_ratio + test_ratio + val_ratio) == 1)

    genres_dir_path = pathlib.Path(genre_dir)
    files_to_split = list(genres_dir_path.iterdir())
    if shuffle:
        files_to_split = shuffle_file_list(files_to_split, seed)

    train_idx, test_idx, val_idx = ratio_to_rounded_indices((train_ratio, test_ratio, val_ratio), len(files_to_split))

    files = {
        "train": files_to_split[:train_idx],
        "test": files_to_split[train_idx:test_idx],
        "val": files_to_split[test_idx:]
    }

    return files

def get_splits_for_dataset(genres_dir: str,
                           shuffle: bool = True,
                           seed: int = None,
                           train_ratio: float = 0.70,
                           test_ratio: float = 0.20,
                           val_ratio: float = 0.10
                           ) -> DatasetSplitDict:

    dataset_split_dict = DatasetSplitDict()
    for genre_dir in pathlib.Path(genres_dir).iterdir():
        genre_split = get_splits_for_genre(str(genre_dir), shuffle, seed, train_ratio, test_ratio, val_ratio)
        dataset_split_dict.add_genre_record(genre_dir.stem, genre_split)
    return dataset_split_dict

def validate_destination_dir(dest: str, clear_existing_dest: bool) -> bool:

    dest_path = pathlib.Path(dest)

    if dest_path.exists():
        if not dest_path.is_dir() and not clear_existing_dest:
            return False
        if len(os.listdir(dest_path)) != 0 and not clear_existing_dest:
            return False
        shutil.rmtree(dest_path)
    os.makedirs(dest_path)
    return True

def save_single_split_as_mels(split: typing.Dict["str", typing.List[pathlib.Path]],
                              split_name: str,
                              destination_dir: str,
                              mel_bands: int = 128,
                              sr_overwrite: int = None,
                              split_duration: float = None,
                              spec_log_scale: bool = True,
                              cut_fragment: bool = False,
                              overlap_ratio: float = 0.50,
                              dpi: int = 100
                              ) -> None:

    fig, ax = plt.subplots()
    plt.close(fig)

    for genre_name in split.keys():

        genre_destination_dir = pathlib.Path(destination_dir).joinpath(split_name).joinpath(genre_name)
        os.makedirs(genre_destination_dir, exist_ok=True)

        source_files = split[genre_name]
        for source_file in source_files:

            try:
                audio_data = audio_processing.load_to_mono(str(source_file), sr_overwrite)
            except (RuntimeError, audioread.exceptions.NoBackendError):
                print(f"\n(!) File {source_file.stem} was skipped because it couldnt be loaded and may be corrupted.")
                continue

            mels_to_save = list()
            if split_duration is not None:
                if split_duration > len(audio_data.timeseries)*audio_data.sr:
                    print(f"\n(!) File {source_file.stem} was skipped because it was shorter than the fragment length.")
                    continue
                if cut_fragment:
                    fragment = audio_processing.get_fragment_of_timeseries(audio_data,
                                                                           len(audio_data.timeseries)/audio_data.sr*0.5,
                                                                           split_duration
                                                                           )
                    mel = audio_processing.mel_from_timeseries(fragment, mel_bands,
                                                               sr_overwrite, log_scale=spec_log_scale)
                    mels_to_save.append(mel)
                else:
                    splits = audio_processing.split_timeseries(audio_data, split_duration, overlap_ratio)
                    if len(splits) == 0:
                        continue
                    mels_to_save = [
                        audio_processing.mel_from_timeseries(x, mel_bands, sr_overwrite, spec_log_scale)
                        for x in splits
                    ]
            else:
                mel = audio_processing.mel_from_timeseries(audio_data, mel_bands,
                                                           sr_overwrite, log_scale=spec_log_scale)
                mels_to_save.append(mel)

            for part_num, mel in enumerate(mels_to_save):
                visual.mel_only_on_ax(mel, ax)
                mel_save_filename = pathlib.Path(genre_destination_dir).joinpath(
                    f"{source_file.stem}_part{part_num}.png"
                )
                fig.savefig(fname=mel_save_filename, bbox_inches="tight", pad_inches=0.0, dpi=dpi)

            print(f"\rCurrent state: {split_name}: {genre_name}: {source_file.stem}", end="")

def save_splits_as_mels(dataset_splits: DatasetSplitDict,
                        destination_dir: str,
                        clear_dest: bool = True,
                        mel_bands: int = 128,
                        sr_overwrite: int = None,
                        split_duration: float = None,
                        cut_fragment: bool = False,
                        spec_log_scale: bool = True,
                        overlap_ratio: float = 0.50,
                        dpi: int = 100,
                        use_multiprocessing: bool = False
                        ) -> None:

    assert(len(dataset_splits.split_dict.keys()) > 0)

    if not validate_destination_dir(destination_dir, clear_dest):
        raise Exception

    if use_multiprocessing:
        raise NotImplementedError
    else:
        for split_name in dataset_splits.split_dict.keys():
            os.makedirs(pathlib.Path(destination_dir).joinpath(split_name))
            save_single_split_as_mels(dataset_splits.split_dict[split_name],
                                      split_name,
                                      destination_dir,
                                      mel_bands,
                                      sr_overwrite,
                                      split_duration,
                                      spec_log_scale,
                                      cut_fragment,
                                      overlap_ratio,
                                      dpi
                                      )

    print("Saved.")

def shuffle_mix_song_parts(splitted_dataset_dir: str,
                           destination_dir: str,
                           clear_dest: bool = True,
                           shuffle: bool = True,
                           seed: int = None,
                           train_ratio: float = 0.70,
                           test_ratio: float = 0.20,
                           val_ratio: float = 0.10
                           ) -> None:

    if not validate_destination_dir(destination_dir, clear_dest):
        raise Exception

    virtual_dir_dict = dict()
    for split in pathlib.Path(splitted_dataset_dir).iterdir():
        for genre_dir in split.iterdir():
            for file in genre_dir.iterdir():
                if genre_dir.stem in virtual_dir_dict.keys():
                    virtual_dir_dict[genre_dir.stem].append(file)
                else:
                    virtual_dir_dict.update({genre_dir.stem: [file]})

    virtual_dataset_split_dict = DatasetSplitDict()
    for genre in virtual_dir_dict.keys():
        files_to_split = virtual_dir_dict[genre]
        if shuffle:
            files_to_split = shuffle_file_list(files_to_split, seed)

        train_idx, test_idx, val_idx = ratio_to_rounded_indices((train_ratio, test_ratio, val_ratio), len(files_to_split))

        files = {
            "train": files_to_split[:train_idx],
            "test": files_to_split[train_idx:test_idx],
            "val": files_to_split[test_idx:]
        }

        virtual_dataset_split_dict.add_genre_record(genre, files)

    for split in virtual_dataset_split_dict.split_dict.keys():
        split_path = pathlib.Path(destination_dir).joinpath(split)
        os.makedirs(split_path)

        for genre in virtual_dataset_split_dict.split_dict[split].keys():
            genre_path = split_path.joinpath(genre)
            os.makedirs(genre_path)

            for file in virtual_dataset_split_dict.split_dict[split][genre]:
                shutil.copy(file, genre_path)

def labels_to_file(split_dir: str):
    split_path = pathlib.Path(split_dir)

    labels = [sorted(os.listdir(split_path))]

    label_int_mapping = dict()

    for l_idx, label in enumerate(labels):
        label_int_mapping.update({l_idx: label})

    with open(split_path.joinpath("labels.json"), "w+") as f:
        json.dump(obj=label_int_mapping, fp=f)

def dataset_to_mels(original_dir: str,
                    destination_dir: str,
                    clear_dest: bool = True,
                    mel_bands: int = 256,
                    sr_overwrite: int = None,
                    split_duration: float = None,
                    cut_fragment: bool = False,
                    spec_log_scale: bool = True,
                    overlap_ratio: float = 0.50,
                    dpi: int = 100
                    ):

    if not validate_destination_dir(destination_dir, clear_dest):
        raise Exception

    fig, ax = plt.subplots()
    plt.close(fig)

    genres = list(pathlib.Path(original_dir).iterdir())

    for genre in genres:

        genre_name = genre.name

        genre_destination_dir = pathlib.Path(destination_dir).joinpath(genre_name)
        os.makedirs(genre_destination_dir, exist_ok=True)

        source_files = list(pathlib.Path(original_dir).joinpath(genre_name).iterdir())
        for source_file in source_files:

            try:
                audio_data = audio_processing.load_to_mono(str(source_file), sr_overwrite)
            except (RuntimeError, audioread.exceptions.NoBackendError):
                print(f"\n(!) File {source_file.stem} was skipped because it couldnt be loaded and may be corrupted.")
                continue

            mels_to_save = list()
            if split_duration is not None:
                if split_duration > len(audio_data.timeseries)*audio_data.sr:
                    print(f"\n(!) File {source_file.stem} was skipped because it was shorter than the fragment length.")
                    continue
                if cut_fragment:
                    fragment = audio_processing.get_fragment_of_timeseries(audio_data,
                                                                           len(audio_data.timeseries)/audio_data.sr*0.5,
                                                                           split_duration
                                                                           )
                    mel = audio_processing.mel_from_timeseries(fragment, mel_bands,
                                                               sr_overwrite, log_scale=spec_log_scale)
                    mels_to_save.append(mel)
                else:
                    splits = audio_processing.split_timeseries(audio_data, split_duration, overlap_ratio)
                    if len(splits) == 0:
                        continue
                    mels_to_save = [
                        audio_processing.mel_from_timeseries(x, mel_bands, sr_overwrite, spec_log_scale)
                        for x in splits
                    ]
            else:
                mel = audio_processing.mel_from_timeseries(audio_data, mel_bands,
                                                           sr_overwrite, log_scale=spec_log_scale)
                mels_to_save.append(mel)

            for part_num, mel in enumerate(mels_to_save):
                visual.mel_only_on_ax(mel, ax)
                mel_save_filename = pathlib.Path(genre_destination_dir).joinpath(
                    f"{source_file.stem}_part{part_num}.png"
                )
                fig.savefig(fname=mel_save_filename, bbox_inches="tight", pad_inches=0.0, dpi=dpi)

            print(f"\rCurrent state: {genre_name}: {source_file.stem}", end="")

def dataset_to_specs(original_dir: str,
                    destination_dir: str,
                    clear_dest: bool = True,
                    sr_overwrite: int = None,
                    split_duration: float = None,
                    cut_fragment: bool = False,
                    spec_log_scale: bool = True,
                    log_frequency_axis: bool = False,
                    overlap_ratio: float = 0.50,
                    dpi: int = 100
                    ):

    if not validate_destination_dir(destination_dir, clear_dest):
        raise Exception

    fig, ax = plt.subplots()
    plt.close(fig)

    genres = list(pathlib.Path(original_dir).iterdir())

    for genre in genres:

        genre_name = genre.name

        genre_destination_dir = pathlib.Path(destination_dir).joinpath(genre_name)
        os.makedirs(genre_destination_dir, exist_ok=True)

        source_files = list(pathlib.Path(original_dir).joinpath(genre_name).iterdir())
        for source_file in source_files:

            try:
                audio_data = audio_processing.load_to_mono(str(source_file), sr_overwrite)
            except (RuntimeError, audioread.exceptions.NoBackendError):
                print(f"\n(!) File {source_file.stem} was skipped because it couldnt be loaded and may be corrupted.")
                continue

            specs_to_save = list()
            if split_duration is not None:
                if split_duration > len(audio_data.timeseries)*audio_data.sr:
                    print(f"\n(!) File {source_file.stem} was skipped because it was shorter than the fragment length.")
                    continue
                if cut_fragment:
                    fragment = audio_processing.get_fragment_of_timeseries(audio_data,
                                                                           len(audio_data.timeseries)/audio_data.sr*0.5,
                                                                           split_duration
                                                                           )
                    spec = audio_processing.spec_from_timeseries(fragment,
                                                               sr_overwrite, log_scale=spec_log_scale)
                    specs_to_save.append(spec)
                else:
                    splits = audio_processing.split_timeseries(audio_data, split_duration, overlap_ratio)
                    if len(splits) == 0:
                        continue
                    specs_to_save = [
                        audio_processing.spec_from_timeseries(x, sr_overwrite, spec_log_scale)
                        for x in splits
                    ]
            else:
                spec = audio_processing.spec_from_timeseries(audio_data,
                                                           sr_overwrite, log_scale=spec_log_scale)
                specs_to_save.append(spec)

            for part_num, spec in enumerate(specs_to_save):
                visual.spec_only_on_ax(spec, ax, log_frequency_axis)
                spec_save_filename = pathlib.Path(genre_destination_dir).joinpath(
                    f"{source_file.stem}_part{part_num}.png"
                )
                fig.savefig(fname=spec_save_filename, bbox_inches="tight", pad_inches=0.0, dpi=dpi)

            print(f"\rCurrent state: {genre_name}: {source_file.stem}", end="")

def split_mel_dataset(original_dir,
                      dest_dir=None,
                      shuffle: bool = True,
                      seed: int = None,
                      train_ratio: float = 0.70,
                      test_ratio: float = 0.20,
                      val_ratio: float = 0.10):

    if dest_dir is None:
        dest_dir=original_dir

    dataset_splits = get_splits_for_dataset(original_dir, shuffle, seed, train_ratio, test_ratio, val_ratio)

    for split_name in dataset_splits.split_dict.keys():
        os.makedirs(pathlib.Path(dest_dir).joinpath(split_name))
        for genre in dataset_splits.split_dict[split_name].keys():
            dest_dir_path = pathlib.Path(dest_dir).joinpath(split_name).joinpath(genre)
            os.makedirs(dest_dir_path, exist_ok=True)
            for source_file in dataset_splits.split_dict[split_name][genre]:
                shutil.copy(source_file, dest_dir_path)

if __name__ == "__main__":
    pass
    # DEFAULT_SPLIT_DEST = "/home/aleksy/gtzan_split/"
    # DEFAULT_MEL_DEST = "/home/aleksy/gtzan_mel/"
    # DEFAULT_SPLIT_DURATION = 5.0
    #
    # parser = argparse.ArgumentParser()
    # parser.add_argument()

    # main_split_dict = get_splits_for_dataset(
    #     genres_dir="/home/aleksy/dev/datasets/gtzan_fixed/Data/genres_original",
    #     seed=datetime.datetime.now()
    # )
    # #main_split_dict.to_pandas_dataframes()
    #
    # # save_splits_as_mels(main_split_dict, destination_dir="/home/aleksy/gtzan_versions/gtzan_spec_5_sec_50_512", split_duration=5.0, mel_bands=512, overlap_ratio=0.50)
    # # shuffle_mix_song_parts("/home/aleksy/gtzan_versions/gtzan_spec_5_sec_50_512", "/home/aleksy/gtzan_versions/gtzan_spec_5_sec_50_512_mixed")
    # #
    # # save_splits_as_mels(main_split_dict, destination_dir="/home/aleksy/gtzan_versions/gtzan_spec_5_sec_70_512", split_duration=5.0, mel_bands=512, overlap_ratio=0.70)
    # shuffle_mix_song_parts("/home/aleksy/gtzan_versions/5sec/gtzan_spec_5_sec_50", "/home/aleksy/gtzan_versions/gtzan_spec_5_sec_50_mixed")
    # dataset_to_specs("/home/aleksy/dev/datasets/gtzan_fixed/Data/genres_original", destination_dir="/home/aleksy/gtzan_spec_lin", split_duration=5.0, overlap_ratio=0.5, log_frequency_axis=False)
    # split_mel_dataset("/home/aleksy/gtzan_spec_lin")
    #
    # dataset_to_specs("/home/aleksy/dev/datasets/gtzan_fixed/Data/genres_original", destination_dir="/home/aleksy/gtzan_spec_log", split_duration=5.0, overlap_ratio=0.5, log_frequency_axis=True)
    # split_mel_dataset("/home/aleksy/gtzan_spec_log")
    # dataset_to_mels("/home/aleksy/dev/datasets/gtzan_fixed/Data/genres_original", destination_dir="/home/aleksy/gtzan_spec_5_sec_50_1024", split_duration=5.0, overlap_ratio=0.5, mel_bands=1024)
    # split_mel_dataset("/home/aleksy//home/aleksy/gtzan_spec_5_sec_50_1024")


from os import listdir
from io import StringIO

#==================================================================================================
# CONSTANTS

ABC        = 'abcdefghijklmnopqrstuvwxyz'

# Index of coincidence for Portuguese and English
# Source: https://en.wikipedia.org/wiki/Index_of_coincidence
PT_IOC     = 1.94
EN_IOC     = 1.73

# Etaoin refers to the sequence of characters ordered by frequency in a given language
# Source: https://pt.wikipedia.org/wiki/Frequ%C3%AAncia_de_letras
PT_ETAOIN  = 'aeosrindmutclpvghqbfzjxkwy'
EN_ETAOIN  = 'etaoinshrdlcumwfgypbvkjxqz'

# The index of coincidence for a key length tends to be similar for multiples of that key length.
# Example: if the key length for a vigenere encryption is 4, the calculated IoC for a key length of 8
# will be just as good or even better.
# Given that it is highly unlikely that subsequent multiples of the first best result are the correct
# ones, the first result whose difference to the best result is lower than DELTA is chosen as the
# correct key length.
DELTA      = 0.001

#==================================================================================================
#FUNCTIONS

# Computes index of coincidence for each key length in a range.
# Function arguments:
  # text: encrypted text.
  # max_key_len: Arbitrary value. The function won't compute index of coincidence for keys larger than this argument.
# Returns: list of dictionaries [{key_length, ioc, counts}]
  # key_length: key length.
  # ioc: corresponding index of coincidence.
  # counts: corresponding dictionary of letters and how many times they appear in a text section.
def ioc_by_key_length(text, max_key_len):

  results = []

  for i in range(1, max_key_len+1):
    # Divides the text in a number of sections equal to the assumed key length i
    print(f'Dividing text in {i} sections...')
    sections = divide_text_in_sections(text, i)

    # Computes average index of coincidence for all sections
    total_ioc = 0
    char_counts = []
    print('Calculating ioc...')
    for s in sections:
      (ioc, char_count) = index_of_coincidence(s)
      char_counts.append(char_count)
      total_ioc += ioc

    results.append({'key_length':i, 'ioc':total_ioc/i, 'counts':char_counts})
    print(f'Result for key length {i}: {total_ioc/i}')
    print('Done\n===========================================================\n\n')
  return results

def divide_text_in_sections(text, n):
  sections = [StringIO() for i in range(n)]
  for i, c in enumerate(text):
    sections[i%n].write(c)
  return [s.getvalue() for s in sections]

# Returns index of coincidence and character count for a given text
def index_of_coincidence(text):
  counts = {c:0 for c in ABC}

  for c in text:
    counts[c] += 1

  n = len(text)
  ioc = sum([v * (v-1) for v in counts.values()]) / ((n * (n-1)) / len(ABC))
  return (ioc, counts)

# Attempts text decryption by substituting each character in a section with it's
# corresponding character in the language based on frequency. Used for text analysis.
def decrypt_by_letter_frequency(text, char_counts, etaoin):
  res = StringIO()
  n_sections = len(char_counts)
  #print('char_counts:')
  #for cc in char_counts:
    #print(sorted(cc.items(), reverse=True, key=(lambda t: t[1])))
  substitution_dicts = []
  for cc in char_counts:
    cc_sorted = [k for (k,v) in rev_sorted_items(cc)]    #t = (char, count)
    substitution_dicts.append( dict(zip(cc_sorted, etaoin)) )
  # for sd in substitution_dicts:
    # print(sd)
  for i, c in enumerate(text):
    substitute_char = substitution_dicts[i%n_sections][c]
    res.write(substitute_char)

  return res.getvalue()

# Finds the vigenere encryption key given the character counts of each
# text section and the alphabet ordered by character frequency in the language
def get_vigenere_key(char_counts, etaoin):
  # Most frequent char in language
  dec = etaoin[0]
  key = StringIO()
  for cc in char_counts:
    sorted_chars = rev_sorted_items(cc)
    # Most frequent encrypted char for that section
    enc = sorted_chars[0][0]
    key.write( subtract_char(enc, dec) )
  return key.getvalue()

def decrypt_vigenere(text, key):
  res = StringIO()
  key_length = len(key)
  for i, c in enumerate(text):
    res.write(subtract_char(c, key[i%key_length]))
  return res.getvalue()

def encrypt_vigenere(text, key):
  res = StringIO()
  key_length = len(key)
  for i, c in enumerate(text):
    res.write(add_char(c, key[i%key_length]))
  return res.getvalue()

# Returns most likely language based on index of coincidence
def find_most_likely_language(ioc):
  language_stats = [
    {'language': 'English',    'ioc': EN_IOC, 'etaoin': EN_ETAOIN},
    {'language': 'Portuguese', 'ioc': PT_IOC, 'etaoin': PT_ETAOIN}
  ]

  best_score = 9999
  res = None
  for stats in language_stats:
    difference = abs(stats['ioc'] - ioc)
    if difference < best_score:
      res = stats
      best_score = difference

  return res

def subtract_char(c1, c2):
  res_ord = ord(c1)-ord(c2)
  res_ord = (res_ord%len(ABC)) + ord('a')
  return chr(res_ord)

def add_char(c1, c2):
  res_ord = ord(c1)+ord(c2) - 2*ord('a')
  res_ord = (res_ord%len(ABC)) + ord('a')
  return chr(res_ord)

# Takes a dictionary {k:v} and returns a list [(k,v)] reverse sorted by v
def rev_sorted_items(dictionary):
  return sorted(dictionary.items(), reverse=True, key=(lambda t: t[1]))

if __name__ == "__main__":
  main()


import csv
from collections import Counter
import pandas as pd

PATH = r'C:/Users/Yelyzaveta_Shapran/Desktop/New folder (2)/test_module/newsfeed.txt'

def get_words_letters(path_to_file):
    with open(path_to_file, 'r') as f:
        rows_from_file = [str(i) for i in f.readlines()]
        f.close()

    words = []
    letters = []
    for i in rows_from_file:
        every_line = i.replace('!', ',').replace('?', ',').replace(' ', ',').replace('.', ',').replace('\n', ',').replace(':', ',').replace('&', ',')
        new_rows = every_line.split(',')

        for word in new_rows:
            if word.isalpha():
                words.append(word.lower())
                for letter in word:
                    letters.append(letter)

    cnt_letters = dict(Counter(letters))
    cnt_words = dict(Counter(words))

    return words, letters, cnt_letters, cnt_words

words_list, letters_list, count_letters, count_words = get_words_letters(PATH)

def create_csv1():
    with open('task_1.csv', 'w', newline='') as f:
        w = csv.writer(f, delimiter='-')
        w.writerows(count_words.items())
        return 'File "task_1.csv" created successfully.'

def cnt_upper_l(dictionary):
    new_dict = {}
    for letter, count in dictionary.items():
        if letter.isupper():
            new_dict[letter] = count
        else:
            new_dict[letter] = 0

    return new_dict

count_of_upper_case = cnt_upper_l(count_letters)

def percent_of_total_cnt(dictionary):
    new_dict = {}
    total_cnt = sum(dictionary.values())
    for letter, count in dictionary.items():
        percent = count * 100.0 / total_cnt
        new_dict[letter] = percent

    return new_dict

percent = percent_of_total_cnt(count_letters)

# Second file creating with pandas for better performance
def create_csv2():
    letter = [*count_letters.keys()]
    count = [*count_letters.values()]
    cnt_up = [*count_of_upper_case.values()]
    pct = [*percent.values()]

    d = {'letter_name': letter, 'count_all': count, 'count_uppercase': cnt_up, 'percent': pct}

    df = pd.DataFrame(d)

    df.to_csv('task_2.csv', index=False)

    return 'File "task_2.csv" created successfully.'

print(create_csv1())
print(create_csv2())


import scrapy
from scrapy.spiders import CrawlSpider, Rule
from scrapy.http import Request
from scrapy.linkextractors import LinkExtractor

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

import json

import time

# Def custom CarItem class that contains various fields
class CarItem(scrapy.Item):
    cover_image = scrapy.Field()
    title = scrapy.Field()
    vin_number = scrapy.Field()

class CrawlingSpider(CrawlSpider):
    name = "mycrawler"
    allowed_domains = ["beavertontoyota.com"]
    url = 'https://www.beavertontoyota.com'
    start_urls = [url]

    rules = (
    #Rule(LinkExtractor(allow='/hours\.aspx'), callback='parse_dealer'),
    # Rule(LinkExtractor(allow='/searchnew\.aspx'), callback='parse_new'),
    Rule(LinkExtractor(allow='/searchnew\.aspx$'), callback='parse_test'),
    )

    def __init__(self, *args, **kwargs):
        super(CrawlingSpider, self).__init__(*args, **kwargs)
        self.driver = webdriver.Chrome()

    # sample table xpath
    def extract_hours_rows(self, tablePath, response):
        hours = []
        for i in range(1, 8):
            day = response.xpath(f'{tablePath}/tr[{i}]/td[1]/text()').get()
            opening_hour = response.xpath(f'{tablePath}/tr[{i}]/td[2]/text()').get()
            closing_hour = response.xpath(f'{tablePath}/tr[{i}]/td[3]/text()').get()
            hours.append({
                'day': day.strip() if day else None,
                'opening_hour': opening_hour.strip() if opening_hour else None,
                'closing_hour': closing_hour.strip() if closing_hour else None
            })
        return hours

    def parse_dealer(self, response):
        # parse the dealer information
        dealer_logo_url = response.xpath('/html/body/div[1]/header/div/div[1]/div[1]/ul/li[2]/a/img/@src').get()
        dealer_brand_url = response.xpath('/html/body/div[1]/header/div/div[1]/div[1]/ul/li[1]/a/img[1]/@src').get()
        dealer_contact = response.xpath('/html/body/div[1]/header/div/div[1]/div[2]/ul/li[1]/ul/li/span[2]/span/text()').get()

        dealer_address = ''
        dealer_hours = []

        # parse main dealer address
        dealer_address_list = response.xpath('/html/body/div[2]/section/div/div[2]/div[2]/div[1]/div[1]//text()').getall()
        dealer_address = '\n'.join([i.strip() for i in dealer_address_list if i.strip()])

        #parse main dealer opening hours
        # ** null value needs to be fixed **
        sales_hours_rows = '/html/body/div[2]/section/div/div[2]/div[3]/div[2]/div[2]/div/div[1]/table/tbody'
        sales_hours = self.extract_hours_rows(sales_hours_rows, response)
        dealer_hours.append({'type': 'SalesHours', 'hours': sales_hours})

        service_hours_rows = '/html/body/div[2]/section/div/div[2]/div[3]/div[2]/div[2]/div/div[2]/table/tbody'
        service_hours = self.extract_hours_rows(service_hours_rows, response)
        dealer_hours.append({'type': 'ServiceHours', 'hours': service_hours})

        parts_hours_rows = '/html/body/div[2]/section/div/div[2]/div[3]/div[2]/div[2]/div/div[3]/table/tbody'
        parts_hours = self.extract_hours_rows(parts_hours_rows, response)
        dealer_hours.append({'type': 'PartsHours', 'hours': parts_hours})

        data = {
            'website': self.url,
            'dealer_logo_url': self.url + dealer_logo_url,
            'dealer_brand_url' : self.url + dealer_brand_url,
            'dealer_address' : dealer_address,
            'dealer_contact' : dealer_contact,
            'dealer_hours' : dealer_hours
        }
        filename = 'dealer_info.json'

        with open(filename, 'w') as f:
            json.dump(data, f, indent=4)

    def parse_new(self, response):

        car_list = response.xpath('/html/body/div[2]/section/div[4]/div[2]/div[2]/div/div[6]/div/div/div[1]/div[2]').get()
        car_items = []
        for car in car_list:
            item = CarItem()
            # /div[1]/div[1]/div/div/div/div[1]/div[1]/a/div/div[2]/img[1]
            # /div[2]/div[1]/div/div/div/div[1]/div[1]/a/div/div[2]/img[1]
            item['cover_image'] = car.xpath('.//div[1]/div/div/div/div[1]/div[1]/a/div/div[2]/img[1]/@src').get()
            item['cover_image'] = car.xpath('.//div[1]/div[1]/div/div/div/div[1]/div[1]/a/div/div[2]/img[1]/@src').get()

            item['title'] = car.xpath('.//div[2]/div/div[1]/a/h3/span/text()').get()
            item['vin_number'] = car.xpath('.//div[2]/div/div[3]/div/div[1]/span[2]/text()').get()
            car_items.append(item)

        data = {
            'car_items': car_items
        }

        filename = 'new_car.json'

        with open(filename, 'w') as f:
            json.dump(data, f, indent=4)

    def parse_test(self, response):
        self.driver.get(response.url)
        # time.sleep(3)

        test1 = response.xpath('/html/body/div[2]/div[2]/a/div/div/div/h3/text()').get()
        test2 = self.driver.find_element('xpath', '/html/body/div[2]/div[2]/a/div/div/div/h3').text

        # Wait for the element to be visible
        wait = WebDriverWait(self.driver, 10)
        test3_element = wait.until(EC.visibility_of_element_located((By.XPATH, '/html/body/div[2]/section/div[4]/div[2]/div[1]/h3')))
        test3 = test3_element.text

        data = {
            'test1' : test1,
            'test2': test2,
            'test3' : test3
            # 'test1' : test1,
            # 'test5' : test5
        }

        filename = 'test.json'

        with open(filename, 'w') as f:
            json.dump(data, f, indent=4)


# Import Libraries
from bs4 import BeautifulSoup

from .helper import getHTML, insertDB

from ..models import Company

def blockchaincom():
    url = "https://www.blockchain.com/ventures"
    xpath = "/html/body/div[1]/div[1]/main/div[2]"

    html = getHTML(url, xpath)

    # get all divs with class "ventures__InvestorItem-sc-181vawo-8 wUbrE" under div
    divs = html.find_all("div", class_="ventures__InvestorItem-sc-181vawo-8 wUbrE")

    # get foreign key
    fk = Company.objects.get(name="Blockchain.com")

    for div in divs:
        # get a tag element
        aTag = div.find("a")

        # get href from a tag element
        url = aTag.get("href")

        # get domain name within url
        name = url.split("www.")[-1].split("//")[-1].split(".")[0]

        try: image = "https://www.blockchain.com" + aTag.find("img").get("src")
        except: image = str(aTag.find("svg"))

        insertDB(fk, name, url, image)


"""
Views related to the video upload feature
"""
import csv
import logging
from datetime import datetime, timedelta
from uuid import uuid4

import rfc6266
from boto import s3
from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.http import HttpResponse, HttpResponseNotFound
from django.utils.translation import ugettext as _
from django.utils.translation import ugettext_noop
from django.views.decorators.http import require_GET, require_http_methods
from edxval.api import (
    SortDirection,
    VideoSortField,
    create_video,
    get_videos_for_course,
    remove_video_for_course,
    update_video_status
)
from opaque_keys.edx.keys import CourseKey

from contentstore.models import VideoUploadConfig
from contentstore.utils import reverse_course_url
from edxmako.shortcuts import render_to_response
from util.json_request import JsonResponse, expect_json

from .course import get_course_and_check_access

__all__ = ["videos_handler", "video_encodings_download"]

LOGGER = logging.getLogger(__name__)

# Default expiration, in seconds, of one-time URLs used for uploading videos.
KEY_EXPIRATION_IN_SECONDS = 86400

VIDEO_SUPPORTED_FILE_FORMATS = {
    '.mp4': 'video/mp4',
    '.mov': 'video/quicktime',
}

VIDEO_UPLOAD_MAX_FILE_SIZE_GB = 5

# maximum time for video to remain in upload state
MAX_UPLOAD_HOURS = 24

class StatusDisplayStrings(object):
    """
    A class to map status strings as stored in VAL to display strings for the
    video upload page
    """

    # Translators: This is the status of an active video upload
    _UPLOADING = ugettext_noop("Uploading")
    # Translators: This is the status for a video that the servers are currently processing
    _IN_PROGRESS = ugettext_noop("In Progress")
    # Translators: This is the status for a video that the servers have successfully processed
    _COMPLETE = ugettext_noop("Ready")
    # Translators: This is the status for a video that is uploaded completely
    _UPLOAD_COMPLETED = ugettext_noop("Uploaded")
    # Translators: This is the status for a video that the servers have failed to process
    _FAILED = ugettext_noop("Failed")
    # Translators: This is the status for a video that is cancelled during upload by user
    _CANCELLED = ugettext_noop("Cancelled")
    # Translators: This is the status for a video which has failed
    # due to being flagged as a duplicate by an external or internal CMS
    _DUPLICATE = ugettext_noop("Failed Duplicate")
    # Translators: This is the status for a video which has duplicate token for youtube
    _YOUTUBE_DUPLICATE = ugettext_noop("YouTube Duplicate")
    # Translators: This is the status for a video for which an invalid
    # processing token was provided in the course settings
    _INVALID_TOKEN = ugettext_noop("Invalid Token")
    # Translators: This is the status for a video that was included in a course import
    _IMPORTED = ugettext_noop("Imported")
    # Translators: This is the status for a video that is in an unknown state
    _UNKNOWN = ugettext_noop("Unknown")

    _STATUS_MAP = {
        "upload": _UPLOADING,
        "ingest": _IN_PROGRESS,
        "transcode_queue": _IN_PROGRESS,
        "transcode_active": _IN_PROGRESS,
        "file_delivered": _COMPLETE,
        "file_complete": _COMPLETE,
        "upload_completed": _UPLOAD_COMPLETED,
        "file_corrupt": _FAILED,
        "pipeline_error": _FAILED,
        "upload_failed": _FAILED,
        "s3_upload_failed": _FAILED,
        "upload_cancelled": _CANCELLED,
        "duplicate": _DUPLICATE,
        "youtube_duplicate": _YOUTUBE_DUPLICATE,
        "invalid_token": _INVALID_TOKEN,
        "imported": _IMPORTED,
    }

    @staticmethod
    def get(val_status):
        """Map a VAL status string to a localized display string"""
        return _(StatusDisplayStrings._STATUS_MAP.get(val_status, StatusDisplayStrings._UNKNOWN))    # pylint: disable=translation-of-non-string

@expect_json
@login_required
@require_http_methods(("GET", "POST", "DELETE"))
def videos_handler(request, course_key_string, edx_video_id=None):
    """
    The restful handler for video uploads.

    GET
        html: return an HTML page to display previous video uploads and allow
            new ones
        json: return json representing the videos that have been uploaded and
            their statuses
    POST
        json: create a new video upload; the actual files should not be provided
            to this endpoint but rather PUT to the respective upload_url values
            contained in the response
    DELETE
        soft deletes a video for particular course
    """
    course = _get_and_validate_course(course_key_string, request.user)

    if not course:
        return HttpResponseNotFound()

    if request.method == "GET":
        if "application/json" in request.META.get("HTTP_ACCEPT", ""):
            return videos_index_json(course)
        else:
            return videos_index_html(course)
    elif request.method == "DELETE":
        remove_video_for_course(course_key_string, edx_video_id)
        return JsonResponse()
    else:
        if is_status_update_request(request.json):
            return send_video_status_update(request.json)

        return videos_post(course, request)

@login_required
@require_GET
def video_encodings_download(request, course_key_string):
    """
    Returns a CSV report containing the encoded video URLs for video uploads
    in the following format:

    Video ID,Name,Status,Profile1 URL,Profile2 URL
    aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa,video.mp4,Complete,http://example.com/prof1.mp4,http://example.com/prof2.mp4
    """
    course = _get_and_validate_course(course_key_string, request.user)

    if not course:
        return HttpResponseNotFound()

    def get_profile_header(profile):
        """Returns the column header string for the given profile's URLs"""
        # Translators: This is the header for a CSV file column
        # containing URLs for video encodings for the named profile
        # (e.g. desktop, mobile high quality, mobile low quality)
        return _("{profile_name} URL").format(profile_name=profile)

    profile_whitelist = VideoUploadConfig.get_profile_whitelist()

    videos = list(_get_videos(course))
    name_col = _("Name")
    duration_col = _("Duration")
    added_col = _("Date Added")
    video_id_col = _("Video ID")
    status_col = _("Status")
    profile_cols = [get_profile_header(profile) for profile in profile_whitelist]

    def make_csv_dict(video):
        """
        Makes a dictionary suitable for writing CSV output. This involves
        extracting the required items from the original video dict and
        converting all keys and values to UTF-8 encoded string objects,
        because the CSV module doesn't play well with unicode objects.
        """
        # Translators: This is listed as the duration for a video that has not
        # yet reached the point in its processing by the servers where its
        # duration is determined.
        duration_val = str(video["duration"]) if video["duration"] > 0 else _("Pending")
        ret = dict(
            [
                (name_col, video["client_video_id"]),
                (duration_col, duration_val),
                (added_col, video["created"].isoformat()),
                (video_id_col, video["edx_video_id"]),
                (status_col, video["status"]),
            ] +
            [
                (get_profile_header(encoded_video["profile"]), encoded_video["url"])
                for encoded_video in video["encoded_videos"]
                if encoded_video["profile"] in profile_whitelist
            ]
        )
        return {
            key.encode("utf-8"): value.encode("utf-8")
            for key, value in ret.items()
        }

    response = HttpResponse(content_type="text/csv")
    # Translators: This is the suggested filename when downloading the URL
    # listing for videos uploaded through Studio
    filename = _("{course}_video_urls").format(course=course.id.course)
    # See https://tools.ietf.org/html/rfc6266#appendix-D
    response["Content-Disposition"] = rfc6266.build_header(
        filename + ".csv",
        filename_compat="video_urls.csv"
    )
    writer = csv.DictWriter(
        response,
        [
            col_name.encode("utf-8")
            for col_name
            in [name_col, duration_col, added_col, video_id_col, status_col] + profile_cols
        ],
        dialect=csv.excel
    )
    writer.writeheader()
    for video in videos:
        writer.writerow(make_csv_dict(video))
    return response

def _get_and_validate_course(course_key_string, user):
    """
    Given a course key, return the course if it exists, the given user has
    access to it, and it is properly configured for video uploads
    """
    course_key = CourseKey.from_string(course_key_string)

    # For now, assume all studio users that have access to the course can upload videos.
    # In the future, we plan to add a new org-level role for video uploaders.
    course = get_course_and_check_access(course_key, user)

    if (
            settings.FEATURES["ENABLE_VIDEO_UPLOAD_PIPELINE"] and
            getattr(settings, "VIDEO_UPLOAD_PIPELINE", None) and
            course and
            course.video_pipeline_configured
    ):
        return course
    else:
        return None

def convert_video_status(video):
    """
    Convert status of a video. Status can be converted to one of the following:

        *   FAILED if video is in `upload` state for more than 24 hours
        *   `YouTube Duplicate` if status is `invalid_token`
        *   user-friendly video status
    """
    now = datetime.now(video['created'].tzinfo)
    if video['status'] == 'upload' and (now - video['created']) > timedelta(hours=MAX_UPLOAD_HOURS):
        new_status = 'upload_failed'
        status = StatusDisplayStrings.get(new_status)
        message = 'Video with id [%s] is still in upload after [%s] hours, setting status to [%s]' % (
            video['edx_video_id'], MAX_UPLOAD_HOURS, new_status
        )
        send_video_status_update([
            {
                'edxVideoId': video['edx_video_id'],
                'status': new_status,
                'message': message
            }
        ])
    elif video['status'] == 'invalid_token':
        status = StatusDisplayStrings.get('youtube_duplicate')
    else:
        status = StatusDisplayStrings.get(video['status'])

    return status

def _get_videos(course):
    """
    Retrieves the list of videos from VAL corresponding to this course.
    """
    videos = list(get_videos_for_course(course.id, VideoSortField.created, SortDirection.desc))

    # convert VAL's status to studio's Video Upload feature status.
    for video in videos:
        video["status"] = convert_video_status(video)

    return videos

def _get_index_videos(course):
    """
    Returns the information about each video upload required for the video list
    """
    return list(
        {
            attr: video[attr]
            for attr in ["edx_video_id", "client_video_id", "created", "duration", "status"]
        }
        for video in _get_videos(course)
    )

def videos_index_html(course):
    """
    Returns an HTML page to display previous video uploads and allow new ones
    """
    return render_to_response(
        "videos_index.html",
        {
            "context_course": course,
            "video_handler_url": reverse_course_url("videos_handler", unicode(course.id)),
            "encodings_download_url": reverse_course_url("video_encodings_download", unicode(course.id)),
            "previous_uploads": _get_index_videos(course),
            "concurrent_upload_limit": settings.VIDEO_UPLOAD_PIPELINE.get("CONCURRENT_UPLOAD_LIMIT", 0),
            "video_supported_file_formats": VIDEO_SUPPORTED_FILE_FORMATS.keys(),
            "video_upload_max_file_size": VIDEO_UPLOAD_MAX_FILE_SIZE_GB
        }
    )

def videos_index_json(course):
    """
    Returns JSON in the following format:
    {
        "videos": [{
            "edx_video_id": "aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa",
            "client_video_id": "video.mp4",
            "created": "1970-01-01T00:00:00Z",
            "duration": 42.5,
            "status": "upload"
        }]
    }
    """
    return JsonResponse({"videos": _get_index_videos(course)}, status=200)

def videos_post(course, request):
    """
    Input (JSON):
    {
        "files": [{
            "file_name": "video.mp4",
            "content_type": "video/mp4"
        }]
    }

    Returns (JSON):
    {
        "files": [{
            "file_name": "video.mp4",
            "upload_url": "http://example.com/put_video"
        }]
    }

    The returned array corresponds exactly to the input array.
    """
    error = None
    if "files" not in request.json:
        error = "Request object is not JSON or does not contain 'files'"
    elif any(
            "file_name" not in file or "content_type" not in file
            for file in request.json["files"]
    ):
        error = "Request 'files' entry does not contain 'file_name' and 'content_type'"
    elif any(
            file['content_type'] not in VIDEO_SUPPORTED_FILE_FORMATS.values()
            for file in request.json["files"]
    ):
        error = "Request 'files' entry contain unsupported content_type"

    if error:
        return JsonResponse({"error": error}, status=400)

    bucket = storage_service_bucket()
    course_video_upload_token = course.video_upload_pipeline["course_video_upload_token"]
    req_files = request.json["files"]
    resp_files = []

    for req_file in req_files:
        file_name = req_file["file_name"]

        try:
            file_name.encode('ascii')
        except UnicodeEncodeError:
            error_msg = 'The file name for %s must contain only ASCII characters.' % file_name
            return JsonResponse({'error': error_msg}, status=400)

        edx_video_id = unicode(uuid4())
        key = storage_service_key(bucket, file_name=edx_video_id)
        for metadata_name, value in [
                ("course_video_upload_token", course_video_upload_token),
                ("client_video_id", file_name),
                ("course_key", unicode(course.id)),
        ]:
            key.set_metadata(metadata_name, value)
        upload_url = key.generate_url(
            KEY_EXPIRATION_IN_SECONDS,
            "PUT",
            headers={"Content-Type": req_file["content_type"]}
        )

        # persist edx_video_id in VAL
        create_video({
            "edx_video_id": edx_video_id,
            "status": "upload",
            "client_video_id": file_name,
            "duration": 0,
            "encoded_videos": [],
            "courses": [course.id]
        })

        resp_files.append({"file_name": file_name, "upload_url": upload_url, "edx_video_id": edx_video_id})

    return JsonResponse({"files": resp_files}, status=200)

def storage_service_bucket():
    """
    Returns an S3 bucket for video uploads.
    """
    conn = s3.connection.S3Connection(
        settings.AWS_ACCESS_KEY_ID,
        settings.AWS_SECRET_ACCESS_KEY
    )
    # We don't need to validate our bucket, it requires a very permissive IAM permission
    # set since behind the scenes it fires a HEAD request that is equivalent to get_all_keys()
    # meaning it would need ListObjects on the whole bucket, not just the path used in each
    # environment (since we share a single bucket for multiple deployments in some configurations)
    return conn.get_bucket(settings.VIDEO_UPLOAD_PIPELINE["BUCKET"], validate=False)

def storage_service_key(bucket, file_name):
    """
    Returns an S3 key to the given file in the given bucket.
    """
    key_name = "{}/{}".format(
        settings.VIDEO_UPLOAD_PIPELINE.get("ROOT_PATH", ""),
        file_name
    )
    return s3.key.Key(bucket, key_name)

def send_video_status_update(updates):
    """
    Update video status in edx-val.
    """
    for update in updates:
        update_video_status(update.get('edxVideoId'), update.get('status'))
        LOGGER.info(
            'VIDEOS: Video status update with id [%s], status [%s] and message [%s]',
            update.get('edxVideoId'),
            update.get('status'),
            update.get('message')
        )

    return JsonResponse()

def is_status_update_request(request_data):
    """
    Returns True if `request_data` contains status update else False.
    """
    return any('status' in update for update in request_data)


from __future__ import annotations
from typing import Optional
from msrc.appconfig.schema import Element, SchemaSource, interpret_type
import dataclasses

def inspect(schema: type) -> Optional[SchemaSource]:
    if not dataclasses.is_dataclass(schema):
        return None
    fields = dataclasses.fields(schema)

    def make_element(field: dataclasses.Field[object]) -> Element:
        element_type = interpret_type(field.type)
        if element_type is not None:
            return Element(
                help=field.metadata.get("help", None),
                element_type=element_type,
                is_secret=not field.repr,
                has_default=not (
                    field.default is dataclasses.MISSING
                    and field.default_factory  # type: ignore
                    is dataclasses.MISSING),
                default_value=field.default)
        else:
            raise ValueError("%s: type %s unsupported."
                             % (field.name, field.type))
    return tuple((field.name, make_element(field))
                 for field in fields
                 if not field.name.startswith('_'))


import numpy as np
import pandas as pd
from tqdm.notebook import tqdm
import matplotlib.pyplot as plt
from IPython.display import display
from warnings import filterwarnings

filterwarnings('ignore')

# https://github.com/facebook/prophet/issues/223#issuecomment-326455744
class suppress_stdout_stderr(object):
    '''
    A context manager for doing a "deep suppression" of stdout and stderr in
    Python, i.e. will suppress all print, even if the print originates in a
    compiled C/Fortran sub-function.
       This will not suppress raised exceptions, since exceptions are printed
    to stderr just before a script exits, and after the context manager has
    exited (at least, I think that is why it lets exceptions through).

    '''
    def __init__(self):
        # Open a pair of null files
        self.null_fds = [os.open(os.devnull, os.O_RDWR) for x in range(2)]
        # Save the actual stdout (1) and stderr (2) file descriptors.
        self.save_fds = [os.dup(1), os.dup(2)]

    def __enter__(self):
        # Assign the null pointers to stdout and stderr.
        os.dup2(self.null_fds[0], 1)
        os.dup2(self.null_fds[1], 2)

    def __exit__(self, *_):
        # Re-assign the real stdout/stderr back to (1) and (2)
        os.dup2(self.save_fds[0], 1)
        os.dup2(self.save_fds[1], 2)
        # Close the null files
        for fd in self.null_fds + self.save_fds:
            os.close(fd)

def ro_framework(data, model,
                 target_col, feature_cols,
                 test_start, cv_window, ahead_offest,
                 test_predict=False, test_roll=False,
                 model_params=None,
                 metric='MAPE', debug=True, ahead_offest_freq='days',back_transform_func=None):

    # Checks
    _tfreq = ['years', 'months', 'weeks', 'days', 'hours', 'minutes',
              'seconds', 'microseconds', 'nanoseconds']
    if not (hasattr(model, 'fit') & hasattr(model, 'predict')):
        raise ValueError("Model Passed should be having '.fit' & '.predict' methods, i.e foloowing the sklearn API")
    if data.shape[0]<10:
        raise ValueError("Data used for modelling should be greater than 10 datapoints")
    if type(data) != pd.core.frame.DataFrame:
        raise ValueError("Data should be a pandas dataframe of type 'pd.DataFrame'")
    if type(data.index) != pd.core.indexes.datetimes.DatetimeIndex:
        raise ValueError("Data should be having index of type 'pd.DatetimeIndex'")
    if not (type(test_start) == str or type(test_start) == pd._libs.tslibs.timestamps.Timestamp):
        raise ValueError("'test_start' should be a 'pd.Timestamp' or 'str'")
    if not (pd.to_datetime(test_start)>data.index[0] and pd.to_datetime(test_start)<data.index[-1]):
        raise ValueError("'test_start' should be in between {0} and {1}".format(data.index.min(),
                                                                                data.index.max()))
    if not type(ahead_offest) == pd._libs.tslibs.offsets.DateOffset:
        raise ValueError("'ahead_offest' should be a 'pd.DateOffset'")
    if type(cv_window) != int:
        raise ValueError("'cv_window' should be an `int`")
    if not any([ahead_offest_freq==k for k in _tfreq]):
        raise ValueError(f"'ahead_offest_freq' should be one of {_tfreq}")
    if getattr(ahead_offest, ahead_offest_freq) <= 0:
        raise ValueError(f"'ahead_offest' should be greater than `0 {ahead_offest_freq}`")
    if target_col not in data.columns:
        raise ValueError(f"{target_col} not found in `data`")
    if feature_cols:
        missing_feats = [k for k in feature_cols if k not in data.columns]
        if missing_feats:
            raise ValueError(f"{missing_feats} columns not present in the dataset")
    if data.index.freq == None:
        raise ValueError(f"data.index.freq should not be `None`")

    # Initialisations
    cvDF = pd.DataFrame(columns = ['Actual', 'Forecast', metric])
    testDF = pd.DataFrame(columns=['Actual', 'Forecast', metric])

    model = copy.deepcopy(model)

    modelling_data = data.copy()
    modelling_data.sort_index(inplace=True)

    training_data = data[data.index<test_start].copy()
    testing_data = data[data.index>=test_start].copy()

    train_start = training_data.index[0]

    test_start = pd.to_datetime(test_start)
    test_end = modelling_data.index[-1]

    if cv_window > training_data.shape[0]*0.5:
        raise ValueError("`cv_window` should be less than {}".format(int(training_data.shape[0]*0.5)))

    # Fore debugging purpose
    if debug:
        debugDF = pd.DataFrame(columns=['Train Start', 'Train End', 'CV Point', 'Diff'])
        for cv_date in training_data.index[-cv_window:]:
            train_end = cv_date-ahead_offest
            _packet = {'Train Start': train_start, 'Train End': train_end,
                       'CV Point': cv_date, 'Diff': str(getattr(ahead_offest, ahead_offest_freq))+' '+ahead_offest_freq}
            debugDF = debugDF.append(_packet, ignore_index=True)
        display(debugDF)
        return None

    # Picking the Metrics
    if metric == 'MAPE':
        metric_func = lambda y,yhat: np.round(abs((np.array(y)-np.array(yhat))/np.array(y))*100,2)
    elif metric == 'MSE':
        metric_func = lambda y,yhat: np.round((np.array(y)-np.array(yhat))**2,2)

    # Cross Validation Loop
    cvDF, fitted_model = _roll_loop_modelling(pred_indices = training_data.index[-cv_window:], data = training_data.copy(),
                                             train_start = train_start, feature_cols = feature_cols,
                                             target_col = target_col, ahead_offest = ahead_offest,
                                             metric_func = metric_func, modedf = cvDF, metric = metric,
                                             model = model, model_params = model_params, back_transform_func = back_transform_func)

    # Testing - Using the last fitted_model
    if test_predict:
        if test_roll:
            testDF, _ = _roll_loop_modelling(pred_indices = testing_data.index, data = modelling_data.copy(),
                                                        train_start = train_start, feature_cols = feature_cols,
                                                        target_col = target_col, ahead_offest = ahead_offest,
                                                        metric_func = metric_func, modedf = testDF, metric = metric,
                                                        model = model, model_params = model_params, _desc = 'Running Test Roll', back_transform_func = back_transform_func)
        else:
            testDF = _simple_modelling(data = testing_data, fitted_model=fitted_model,
                                      feature_cols=feature_cols, target_col=target_col,
                                      metric_func=metric_func, modedf=testDF, metric=metric,
                                      back_transform_func=back_transform_func, test_start=test_start, test_end=test_end)

    # Prepare Overall Metric
    overallDF = pd.DataFrame([cvDF[metric].mean(), testDF[metric].mean()],
                             columns=['Overall '+metric],
                             index=['CV', 'Test'])

    cvDF = cvDF.replace({np.inf:np.nan})
    testDF = testDF.replace({np.inf:np.nan})
    return cvDF, testDF, overallDF, fitted_model

def _roll_loop_modelling(pred_indices, data,
                        train_start, feature_cols,
                        target_col, ahead_offest,
                        metric_func, modedf, metric,
                        model, model_params, _desc = 'Running CV Roll', back_transform_func=None):

    for pred_date in tqdm(pred_indices, desc=_desc, leave=True):
        train_end = pred_date-ahead_offest
        # Filter the data
        _train_data = data.truncate(before=train_start, after=train_end)
        _pred_data = data.loc[pred_date].to_frame().T

        if feature_cols:
            # Multivariate
            if 'Prophet' in str(model):
                _prophet_model = model.get_pmodelinstance()
                _traindf =  _train_data.copy()
                _traindf = _traindf[[target_col]+feature_cols]
                _traindf.index.name = 'ds'
                _traindf.reset_index(inplace=True)
                _traindf.rename(columns={target_col:'y'}, inplace=True)

                # Make the forecasting dataframe
                _forecastdf =  _pred_data.copy()
                _forecastdf = _forecastdf[feature_cols]
                _forecastdf.index.name = 'ds'
                _forecastdf.reset_index(inplace=True)
                _forecastdf.rename(columns={target_col:'y'}, inplace=True)

                with suppress_stdout_stderr():
                    _fitted_model = _prophet_model.fit(_traindf)
                    _forecast = _prophet_model.predict(_forecastdf).yhat.values[0]
                _actual = _pred_data[target_col].values[0]
        else:
            # Univariate
            if 'statsmodels' in str(model):
                # Update Model Params Based on library
                model_params['endog'] = _train_data[target_col]
                modeldef = model(**model_params)

                _fitted_model = modeldef.fit()
                _forecast = _fitted_model.predict(start=pred_date, end=pred_date).values[0]
                _actual = _pred_data[target_col].values[0]
            elif 'Prophet' in str(model):
                # Make the training dataframe
                _prophet_model = model.get_pmodelinstance()
                _traindf =  _train_data.copy()
                _traindf.index.name = 'ds'
                _traindf.reset_index(inplace=True)
                _traindf.rename(columns={target_col:'y'}, inplace=True)

                # Make the forecasting dataframe
                _forecastdf = pd.DataFrame(columns=['ds'])
                _forecastdf['ds'] = [pred_date]
                with suppress_stdout_stderr():
                    _fitted_model = _prophet_model.fit(_traindf)
                    _forecast = _prophet_model.predict(_forecastdf).yhat.values[0]
                _actual = _pred_data[target_col].values[0]

        # Update Metric Sheets
        if back_transform_func:
            _actual = back_transform_func(_actual)
            _forecast = back_transform_func(_forecast)

        modedf.loc[pred_date, 'Actual'] = _actual
        modedf.loc[pred_date, 'Forecast'] = _forecast
        modedf.loc[pred_date,  metric] = metric_func(_actual, _forecast)

    return modedf, _fitted_model

def _simple_modelling(data, fitted_model, feature_cols,
                        target_col, metric_func, modedf, metric,
                        back_transform_func, test_start=None, test_end=None):

    if feature_cols:
        # Multivariate
        if 'Prophet' in str(fitted_model):
            # Make the forecasting dataframe
            _forecastdf =  data.copy()
            modedf['Actual'] = _forecastdf[target_col]
            _forecastdf = _forecastdf[feature_cols]
            _forecastdf.index.name = 'ds'
            _forecastdf.reset_index(inplace=True)
            with suppress_stdout_stderr():
                modedf['Forecast'] = fitted_model.predict(_forecastdf).yhat.values
    else:
        # Univariate
        if 'statsmodels' in str(fitted_model):
            modedf['Actual'] = data[target_col]
            modedf['Forecast'] = fitted_model.predict(start=test_start, end=test_end)
            modedf[metric] = modedf.apply(lambda x : metric_func(x.Actual, x.Forecast), axis=1)
        elif 'Prophet' in str(fitted_model):
            # Make the training dataframe
            _forecastdf =  data.copy()
            modedf['Actual'] = _forecastdf[target_col]
            _forecastdf.index.name = 'ds'
            _forecastdf.reset_index(inplace=True)
            _forecastdf = _forecastdf[['ds']].copy()
            modedf['Forecast'] = fitted_model.predict(_forecastdf).yhat.values

    # Update Metric Sheets
    if back_transform_func:
        modedf['Actual'] = back_transform_func(modedf['Actual'])
        modedf['Forecast'] = back_transform_func(modedf['Forecast'])
    modedf[metric] = modedf.apply(lambda x : metric_func(x.Actual, x.Forecast), axis=1)

    return modedf


from flask import Flask, render_template, request, make_response, abort
from flask_socketio import SocketIO, emit
import os
import sqlite3
import json
from auth import authenticate
import datetime
from routes import app

socketio = SocketIO(app)

clients = []

@socketio.on('send_message')
def send_message_handler(data):
    data["by"] = request.cookies.get('USER_ID')
    conn = sqlite3.connect('data.db')
    c = conn.cursor()
    now = datetime.datetime.now()
    currentTime = now.strftime("%Y-%m-%d %H:%M:%S")
    c.execute(f"""INSERT INTO Messages(by_user,body,time,room)
    VALUES
    ({data["by"]},'{data["message"]}','{currentTime}',{data["room"]})
    """)
    conn.commit()
    c.execute(f"""SELECT username FROM Users WHERE id={data["by"]}""")
    messageBY = c.fetchone()[0]
    for i in clients:
        if i["room"] == data["room"]:
            socketio.emit('message_received', {
                          "by": messageBY, "message": data["message"], "time": currentTime}, room=i["id"])

@socketio.on('connected')
def connect(data):
    clients.append({'id': request.sid, 'room': data['room']})

@socketio.on('disconnect')
def disconnect():
    for i in range(len(clients)):
        if clients[i]['id'] == request.sid:
            del clients[i]
            break

# if __name__ == '__main__':
#     socketio.run(app, debug=True)

class ApplicationInterface:
    def __init__(self,socketio,app):
        self.socket = socketio
        self.app = app

    def run(self,host='0.0.0.0',port=5000):
        self.socket.run(self.app,host=host,port=port)

application = ApplicationInterface(socketio,app)


'''
Service Two Convert The Given Timezone in IST

'''

from flask import Flask,request,jsonify
from datetime import datetime
import pytz
ist = pytz.timezone("Asia/Kolkata")

#Init App
app=Flask(__name__)

@app.route('/',methods=['GET'])
def getStatus():
    return jsonify({'message':'Service One Running'})

@app.route('/convert/<timestamp>',methods=['GET'])
def getTimestampIst(timestamp):
    #date = datetime.fromtimestamp(int(timestamp), ist).replace(tzinfo=None)

    try:
       date = datetime.fromtimestamp(int(timestamp), ist).replace(tzinfo=None)
       return jsonify({'date':str(date),'timezone':"IST"})
    except:
      print("An exception occurred")
      return jsonify({'error':'Please Insert Correct timezone'})

#Run server

if __name__=='__main__':
    app.run(host='0.0.0.0', port=5000,debug=True)


from fabric.api import run, settings, env, cd, lcd, prompt, local
from fabric.contrib.console import confirm

root_path = '/Users/lballard/'
env.hosts = ['pds-rings-tools.seti.org']

prod_deploy_dir = 'dictionary'
git_branch = 'master'

def push():
    """
    pushes code to repo and pushes repo to staging
    """

    # then checkout code from repo in another directory, and transfer that copy to server
    with lcd('/Users/lballard/'):
        # checks out git repo into local dir /users/lballard/.
        # then rsyncs that copy to production

        # clean up old deploys
        local('rm -rf dictionary')

        # grab the local repo (this is all because couldn't grab remote from server)
        local('git clone -b ' + git_branch + ' file:////Users/lballard/projects/dictionary')

        if prod_deploy_dir != 'dictionary':
            local('rm -rf %s' % prod_deploy_dir)
            local('mv dictionary %s' % prod_deploy_dir) # rename it down here before rsyncing it

        # rsync that code to dev directory on production
        local('rsync -r -vc -e ssh --exclude .git --exclude static_media %s lballard@pds-rings-tools.seti.org:~/.' % prod_deploy_dir)

def deploy():
    """
    take a backup of the currently deployed source on the server
    """
    with cd('/home/lballard/'):
        # first take a backup:
        run('sudo rsync -r -vc --exclude logs /home/django/djcode/' + prod_deploy_dir + ' backups/.')

        # go
        run('sudo rsync -r -vc --exclude logs ' + prod_deploy_dir + ' /home/django/djcode/.')
        run('sudo touch /home/django/djcode/' + prod_deploy_dir + '/*.wsgi')
        run('sudo touch /home/django/djcode/' + prod_deploy_dir + '/dictionary/*.wsgi')


import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.neural_network import MLPClassifier
from sklearn.metrics import confusion_matrix
import matplotlib.pyplot as plt

def convertDiagnosis(value):
	if value == 'M':
		return 1
	if value == 'B':
		return 0
	return 3;

def accuracy(cm):
	return (cm[0][0]+cm[1][1])/(cm[0][0] + cm[0][1] + cm[1][0] + cm[1][1])

data = pd.read_csv("../data/data.csv")
data = data.drop('id',axis=1)
data['diagnosis'] = data['diagnosis'].apply(convertDiagnosis)
ndata=(data-data.mean())/data.std()
ndata['diagnosis'] = data['diagnosis']
data = ndata

training_set, validation_set = train_test_split(data, test_size = 0.2)

# X_train = training_set.iloc[:,0:-1].values
# Y_train = training_set.iloc[:,-1].values
# X_val = validation_set.iloc[:,0:-1].values
# y_val = validation_set.iloc[:,-1].values

X_train = training_set.loc[:, training_set.columns != 'diagnosis']
Y_train = training_set.loc[:, training_set.columns == 'diagnosis']
X_val = validation_set.loc[:, validation_set.columns != 'diagnosis']
Y_val = validation_set.loc[:, validation_set.columns == 'diagnosis']

classifier = MLPClassifier(hidden_layer_sizes=(50,50,50), max_iter=100,activation = 'logistic',solver='sgd',random_state=1)
classifier.fit(X_train, Y_train.values.ravel())
y_pred = classifier.predict(X_val)
cm = confusion_matrix(y_pred, Y_val.values.ravel())
print("Accuracy of MLPClassifier : {}".format(accuracy(cm)))

loss_values = classifier.loss_curve_
plt.plot(loss_values)
plt.show()


from Scenes.SceneManager import *
import pyglet

class MenuScene(Scene):
    """Menu scene in progress."""

    def __init__(self):
        super().__init__()
        # Commenting the lines that depend on images.
        # We don't have access to them.

        # self.bg = pyglet.image.load("menu_bg.jpg")
        # self.snake = pyglet.sprite.Sprite ( pyglet.image.load("snake.png") )
        # self.snake.position = 320, 0
        self._status = "menu"
        self.user_chip = TypeShip.FIGHTER

        self._idx_choose = 1

        self._menu = {
            1: TypeShip.FIGHTER,
            2: TypeShip.BUG,
            3: TypeShip.SAUCER,
        }

    def init(self, master, width, height):
        self.key_handler = key.KeyStateHandler()
        # background = pyglet.graphics.OrderedGroup(0)
        self.width = width
        self.height = height

        self.batch = pyglet.graphics.Batch()
        self.menu = master.make_menu(self.batch)
        self._label = pyglet.text.Label(
            text='<< ASTROIDE >>',
            x=400, y=550,
            anchor_x='center',
            batch=self.batch,
            group=None)

        self.menu.select_choose(self.user_chip)

    def on_key_press(self, app, symbol, modifiers):
        if symbol == pyglet.window.key.UP:
            if self._idx_choose > 1:
                self._idx_choose -= 1
        elif symbol == pyglet.window.key.DOWN:
            if self._idx_choose < 3:
                self._idx_choose += 1

        if symbol == pyglet.window.key._1:
            self._idx_choose = 1
        elif symbol == pyglet.window.key._2:
            self._idx_choose = 2
        elif symbol == pyglet.window.key._3:
            self._idx_choose = 3

        self.user_chip = self._menu[self._idx_choose]
        self.menu.select_choose(self.user_chip)

    def on_key_release(self, app, symbol, modifiers):
        if symbol == pyglet.window.key.ENTER:
            self._status = "play"
            app.set_curr_ship(self.user_chip)
            app.changeStatus(self._status)

    def on_step(self, app, dt):
        # print("Step: {}".format(dt))
         pass

    def on_draw(self, manager):
        super().on_draw(manager)
        manager.window.clear()
        self.batch.draw()

        # self.bg.blit(0, 0)
        # self.snake.draw()


# -*- coding: utf-8 -*-
from django.conf import settings
from oauth.oauth import OAuthToken

from account.backends import DenyException
from django.utils.datastructures import MultiValueDictKeyError

import json
import urllib

class VkBackend():
    key       = settings.ACCOUNT_VK_KEY
    secret    = settings.ACCOUNT_VK_SECRET
    name      = 'vk'
    title     = 'Vkontakte'

    def get_auth_url(self, return_to=None):
        args = { 'client_id' : self.key,
                 'redirect_uri'  : return_to,
                 'scope' : 'wall',
                 'response_type' : 'code',
                 }
        return (None,
                "http://api.vk.com/oauth/authorize?" + urllib.urlencode(args)
                )

    def get_token(self, request, redirect_url=None):
        try:
            code = request.GET['code']
        except MultiValueDictKeyError:
            if request.GET['error']:
                raise DenyException('Access dennied by user')
        #
        db_request_token = request.session.get('account_token','')
        args = { 'client_id'  : self.key,
                 'redirect_uri' : redirect_url,
                 'client_secret' : self.secret,
                 'code' : code,
                 'format' : 'json',
                 }

        url = "https://api.vk.com/oauth/access_token?" + urllib.urlencode(args)
        urllib._urlopener = urllib.FancyURLopener()
        urllib._urlopener.addheader('Connection', 'close')
        data = urllib.urlopen(url)
        data = json.loads(data.read())
        return data['access_token']

    def serialize(self, token):
        return token.to_string()

    def deserialize(self, db_token):
        return OAuthToken.from_string(db_token)

    def get_user_data(self, access_token):
        param={'access_token': access_token,
               'fields': 'nickname, screen_name, uid, photo',
               }
        url='https://api.vk.com/method/users.get?'+urllib.urlencode(param)
        user_info = json.loads(urllib.urlopen(url).read())['response'][0]
        return { 'id': user_info.get('uid',''),
                 'name': '%s %s' % (user_info.get('first_name',''),
                                    user_info.get('last_name',''),
                                    ),
                 'screen_name': user_info.get('screen_name',''),
                 'picture': user_info.get('photo',None),
                 'link': '//vk.com/%s' % user_info.get('vuid',''),
                 'backend': 'vk',
                 }


from django.shortcuts import render, get_object_or_404, redirect
from django.contrib.auth.models import User
from lottery.models import Profile
from lottery.models import Ticket, WinningPick
from .forms import WithdrawalForm, UserUpdateForm, ProfileUpdateForm
from django.core.mail import send_mail
from django.contrib import messages
from django.core.paginator import Paginator
from datetime import datetime

# Create your views here.

def profile(request, username):
    user = get_object_or_404(User, username=username)
    p = Paginator(Ticket.objects.filter(user_id=user.id).order_by("-date"), 10)
    page = request.GET.get("page")
    tickets = p.get_page(page)
    today = datetime.now()

    if request.method == "POST":
        w_form = WithdrawalForm(request.POST)
        if w_form.is_valid():
            select_bank = w_form.cleaned_data["select_bank"]
            account_number = w_form.cleaned_data["account_number"]
            withdraw_amount = w_form.cleaned_data["withdraw_amount"]
            email = user.email
            subject = f"Withdrawal Request by - {user.username}"
            message = f"User: {user.username}\nBank: {select_bank}\nAccount Number:{account_number}\nRequested amount: #{withdraw_amount}\n"
            print(message)
            send_mail(
                subject,
                message,
                email,
                ["Lttrglbl@gmail.com"],
            )
            messages.success(request, "Withdrawal Request Has been successfully sent.")
            return redirect("profile", user.username)

    else:
        w_form = WithdrawalForm()
    context = {
        "user": user,
        "tickets": tickets,
        "w_form": w_form,
        "today": today,
    }
    return render(request, "account/profile.html", context)

def update_profile(request, username):
    if request.method == "POST":
        u_form = UserUpdateForm(request.POST, instance=request.user)
        p_form = ProfileUpdateForm(
            request.POST, request.FILES, instance=request.user.profile
        )
        if u_form.is_valid() and p_form.is_valid():
            u_form.save()
            p_form.save()
            messages.success(request, f"account successfully updated")
            return redirect(
                "profile", username=request.user.username
            )  # i solved that by passing the requesting user username
    else:
        u_form = UserUpdateForm(instance=request.user)
        p_form = ProfileUpdateForm(instance=request.user.profile)

    context = {"u_form": u_form, "p_form": p_form}
    return render(request, "account/update_profile.html", context)


import requests
from lxml import etree
from retrying import retry
import json
'''
循环爬取百度贴吧 手机 极速版
'''

class TiebaSpider(object):
    def __init__(self,name):
        self.url = 'http://tieba.baidu.com/mo/q---FA77541F561680C166FF4B9C2A85CA24%3AFG%3D1--1-3-0--2--wapp_1507619411320_819/m?kw={}'.format(name)
        self.headers = {"User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1"}
        self.base_url = 'http://tieba.baidu.com/mo/q---FA77541F561680C166FF4B9C2A85CA24%3AFG%3D1--1-3-0--2--wapp_1507619411320_819/'

    @retry(stop_max_attempt_number=3)
    def parse_url(self,url):
        try:
            response = requests.get(url=url,headers=self.headers,timeout=2)
            html_str = response.content
            return etree.HTML(html_str)
        except:
            return None

    def get_data_list(self,element):
        if element is None:
            return
        div_list = element.xpath('//div[@class="i"]')
        data_list = []
        for div in div_list:
            data={
                'title':div.xpath('./a/text()')[0].replace('\xa0','') if div.xpath('./a/text()') else None,
                'href': self.base_url + div.xpath('./a/@href')[0] if div.xpath('./a/@href') else None
            }
            data_list.append(data)
        next_url = self.base_url + element.xpath('//a[text()="下一页"]/@href')[0] if element.xpath('//a[text()="下一页"]/@href') else None
        print(next_url)
        return data_list,next_url

    def get_content_and_pictures(self,data_list):
        for data in data_list:
            element = self.parse_url(data['href'])
            div = element.xpath('//div[@class="d"]')[0]
            data['content'] = div.xpath('./div[@class="i"]/text()')[0]
            data['pic_list']= [ requests.utils.unquote(i.split('src=')[1]) for i in div.xpath("./div[@class='i']/a/@href") if i.startswith('http') ]

    def save(self,data_list):
        with open('./tieba.txt','a') as f:
            for d in data_list:
                f.write(json.dumps(d,ensure_ascii=False,indent=2))
                f.write('\n')

    def run(self):
        next_url = self.url
        while next_url is not None:
            # 1 获取 首页的url地址
            # 2发送请求获取响应
            element = self.parse_url(next_url)
            # 3 提取title href 下一页的urli地址
            data_list,next_url = self.get_data_list(element)
            # 4 发送帖子href的请求，获取帖子里面的图片
            self.get_content_and_pictures(data_list)
            # 5 保存到本地
            self.save(data_list)
            # 6 请求下一页url  循环

if __name__ == '__main__':
    spider = TiebaSpider( '哈尔滨工业大学' )
    spider.run()

01 循环爬取百度贴吧.py


__date__ = "2017-04-30T20:03:36+0300"
__author__ = "John Saturday <john.saturday.73@gmail.com>"
__credits__ = "Alan McIntyre <https://github.com/alanmcintyre>"

from http.client import OK as _HTTP_OK
from zlib import MAX_WBITS as _MAX_WBITS

#from Cookie import CookieError
from http.client import HTTPException, BadStatusLine
from socket import error as SocketError

#from Cookie import SimpleCookie
from decimal import Decimal
from hashlib import sha512 as _sha512
from hmac import new as newhash
from http.client import HTTPSConnection
from re import search
from urllib.parse import urlparse
from zlib import decompress as _zdecompress

try:
    from simplejson import loads as jsonloads
except ImportError:
    from json import loads as jsonloads

API_REFRESH = 2            # data refresh time
BTCE_HOST = 'wex.nz'    # BTC-e host (HTTP/SSL)
CF_COOKIE = '__cfduid'     # CloudFlare security cookie
HTTP_TIMEOUT = 10      # connection timeout (max: 60 sec)

class APIError(Exception):
    "Raise exception when the BTC-e API returned an error."
    pass

class BTCEConnection(object):
    """BTC-e Trade/Public API persistent HTTPS connection.
    @cvar conn: shared httplib.HTTPSConnection between instances"""
    _headers = {         # common HTTPS headers
        'Accept': 'application/json',
        'Accept-Charset': 'utf-8',
        'Accept-Encoding': 'identity',
        'Cache-Control': 'no-cache',
        'Connection': 'keep-alive',
        }
    _post_headers = {    # common and POST headers
        'Content-Type': 'application/x-www-form-urlencoded',
        }
    conn = None    # type 'httplib.HTTPSConnection'
    resp = None    # type 'httplib.HTTPResponse'

    @classmethod
    def __init__(cls, compr=True, timeout=HTTP_TIMEOUT):
        """Initialization of shared HTTPS connection.
        @param compr: HTTPS compression (default: identity)
        @param timeout: HTTPS timeout (default/max: 30/60 sec)"""
        if compr is False:
            compr = 'identity'
        elif compr is True:
            compr = 'gzip, deflate'

        if not cls.conn:
            # Create a new connection.
            cls.conn = HTTPSConnection(BTCE_HOST, strict=True, timeout=timeout)
            cls._post_headers.update(cls._headers)
        elif timeout != cls.conn.timeout:
            # Update the connection timeout.
            cls.conn.timeout = timeout
            cls.conn.close()
        if compr and compr != cls._headers['Accept-Encoding']:
            # Update the connection compression.
            cls._headers['Accept-Encoding'] = compr
            cls._post_headers.update(cls._headers)
            cls.conn.close()

    @classmethod
    def _signature(cls, apikey, msg):
        """Calculation of the SHA-512 authentication signature.
        @param apikey: API-key dict {'Key': '...', 'Secret': '...'}
        @param msg: method and parameters (Trade API)"""
        sign = newhash(apikey['Secret'], msg=msg, digestmod=_sha512)
        cls._post_headers['Key'] = apikey['Key']
        cls._post_headers['Sign'] = sign.hexdigest()

    @classmethod
    def _setcookie(cls):
        "Get the CloudFlare cookie and update security."
        cookie_header = cls.resp.getheader('Set-Cookie')
        try:
            cf_cookie = SimpleCookie(cookie_header)[CF_COOKIE]
        except (CookieError, KeyError):
            pass    # with/out previous cookie
        else:
            cf_value = cf_cookie.OutputString('value')
            cls._headers['Cookie'] = cls._post_headers['Cookie'] = cf_value

    @classmethod
    def _decompress(cls, data):
        """Decompress connection response.
        @return: decompressed data <type 'str'>"""
        encoding = cls.resp.getheader('Content-Encoding')
        if encoding == 'gzip':
            data = _zdecompress(data, _MAX_WBITS+16)
        elif encoding == 'deflate':
            data = _zdecompress(data, -_MAX_WBITS)
        # else: failback to 'identity' encoding
        return data

    @classmethod
    def jsonrequest(cls, url, apikey=None, **params):
        """Create query to the BTC-e API (JSON response).
        @raise httplib.HTTPException, socket.error: connection errors
        @param url: plain URL without parameters (Trade/Public API)
        @param apikey: API-key dict {'Key': '...', 'Secret': '...'}
        @param **params: method and/or parameters (Trade/Public API)
        @return: API response (JSON data) <type 'str'>"""
        if apikey:    # args: Trade API
            method = 'POST'
            body = urlparse(params)
            cls._signature(apikey, body)
            headers = cls._post_headers
        else:         # args: Public API
            method = 'GET'
            if params:
                url = '{}?{}'.format(url, urlparse(params))
            body = None
            headers = cls._headers
        while True:
            # Make a HTTPS request.
            try:
                cls.conn.request(method, url, body=body, headers=headers)
                cls.resp = cls.conn.getresponse()
            except BadStatusLine:
                cls.conn.close()
                continue
            except (HTTPException, SocketError):
                cls.conn.close()
                raise
            cls._setcookie()
            break
        return cls._decompress(cls.resp.read())

    @classmethod
    def apirequest(cls, url, apikey=None, **params):
        """Create query to the BTC-e API (decoded response).
        @raise APIError, httplib.HTTPException: BTC-e and CloudFlare errors
        @param url: plain URL without parameters (Public/Trade API)
        @param apikey: API-key dict {'Key': '...', 'Secret': '...'}
        @param **params: method and/or parameters (Public/Trade API)
        @return: API response (decoded data) <type 'dict'>"""
        data = cls.jsonrequest(url, apikey, **params)
        try:
            data = jsonloads(data, parse_float=Decimal, parse_int=Decimal)
        except ValueError:
            if cls.resp.status == _HTTP_OK:
                # BTC-e API unknown errors.
                raise APIError(str(data) or "Unknown Error")
            else:
                # HTTP or CloudFlare errors.
                raise HTTPException("{} {}".format(
                    cls.resp.status, cls.resp.reason))
        else:
            if 'error' in data:
                # BTC-e API standard errors.
                raise APIError(str(data['error']))
        return data

class TradeAPIv1(BTCEConnection):
    "BTC-e Trade API v1 <https://btc-e.com/tapi/docs>."
    def __init__(self, apikey, **connkw):
        """Initialization of the BTC-e Trade API v1.
        @raise APIError: where no return an invalid nonce error
        @param apikey: API-key dict {'Key': '...', 'Secret': '...'}
        @param **connkw: compr, timeout (see: BTCEConnection class)"""
        super(TradeAPIv1, self).__init__(**connkw)
        self._apikey = apikey            # type 'dict' (keys: 'Key', 'Secret')
        self.nonce = self._getnonce()    # type 'long' (min/max: 1-4294967294)

    def _getnonce(self):
        """Get nonce value from BTC-e API error.
        @return: nonce parameter <type 'long'>"""
        try:
            self.apirequest('/tapi', self._apikey, nonce=None)
        except APIError as error:
            if 'invalid nonce' not in error.message:
                raise
            nonce = search(r'\d+', error.message).group()
        return int(nonce)

    def _nextnonce(self):
        """Increase and return next nonce parameter.
        @return: nonce parameter <type 'long'>"""
        self.nonce += 1
        return self.nonce

    def call(self, method, **params):
        """Create query to the BTC-e Trade API v1.
        @param method: getInfo | Trade | ActiveOrders | OrderInfo |
            CancelOrder | TradeHistory (max: 2000) | TransHistory (max: 2000)
        @param method*: WithdrawCoin | CreateCoupon | RedeemCoupon
        @param **params: param1=value1, param2=value2, ..., paramN=valueN
        @return: API response (see: online documentation) <type 'dict'>"""
        params['method'] = method
        params['nonce'] = self._nextnonce()
        return self.apirequest('/tapi', self._apikey, **params)['return']

class PublicAPIv3(BTCEConnection):
    "BTC-e Public API v3 <https://btc-e.com/api/3/docs>."
    def __init__(self, *pairs, **connkw):
        """Initialization of the BTC-e Public API v3.
        @param *pairs: [btc_usd[-btc_rur[-...]]] or arguments
        @param **connkw: compr, timeout (see: BTCEConnection class)"""
        super(PublicAPIv3, self).__init__(**connkw)
        self.pairs = pairs    # type 'str' (delimiter: '-')

        # Get and/or join all pairs.
        if not self.pairs:
            self.pairs = self.call('info')['pairs'].keys()
        if not isinstance(self.pairs, str):
            self.pairs = '-'.join(self.pairs)

    def call(self, method, pairs=None, **params):
        """Create query to the BTC-e Public API v3.
        @param method: info | ticker | depth | trades
        @param pairs: [btc_usd[-btc_rur[-...]]] <type 'str'>
        @param **params: limit=150 (max: 5000), ignore_invalid=1
        @return: API response (see: online documentation) <type 'dict'>"""
        if method == 'info':
            url = '/api/3/{}'.format(method)
        else:    # method: ticker, depth, trades
            pairs = pairs or self.pairs
            url = '/api/3/{}/{}'.format(method, pairs)
        return self.apirequest(url, **params)


"""miniappservice URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.9/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.conf.urls import url, include
    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
"""
from django.conf.urls import url
from django.contrib.auth.decorators import login_required
from django.views.generic.list import ListView
import django.contrib.auth.views as auth_views

from .models import Trabajo

urlpatterns = [
    url(
        r'^$',
        login_required(ListView.as_view(
                template_name='inicio.html',
                queryset=Trabajo.objects.all(),
                context_object_name='trabajos_list'
            )
        ),
        name='home'
    ),
]


from fastapi import APIRouter, status
import requests

router = APIRouter(
    tags=['GET']
)

@router.get('/tasks', status_code=status.HTTP_202_ACCEPTED)
def get_all():
    r = requests.get('http://localhost:4001/tasks/all')
    return r.json()
#ByField
@router.get('/tasks/byField', status_code=status.HTTP_202_ACCEPTED)
def get_by_method(field:str, payload:str ):
    r = requests.get('http://localhost:4001/tasks/byField',json={"field":field,"payload":payload})
    return r.json()

@router.get('/tasks/{pk}', status_code=status.HTTP_202_ACCEPTED)
def get_by_pk(pk:str):
    r = requests.get('http://localhost:4001/tasks/byPK',json={"pk_id_task":pk})
    return r.json()

@router.get('/tasks/allByFk/{payload}', status_code=status.HTTP_202_ACCEPTED)
def get_by_method( payload:str ):
    r = requests.get('http://localhost:4001/tasks/allByFk',json={"field":"fk_id_user","payload":payload})
    return r.json()


'''
ADMINFINDER.PY : Used to find possible admin panels of a site by concentating suffixes to the
base URL one by one. Uses asyncio.
This file is a part of the Witch Project
By Cryptonian007 [24|11|21::08:43]
'''

from core.formatting import *
from urllib.parse import urlparse
import asyncio,aiohttp,sys
#import progressbar

ok_sites = []
failed = 0
total = 0

def check_suffixes(lst):
    # yeah ik it's stupid
    lol = []
    for i in lst:
        if i[0] == '/':lol.append(i)
        else:lol.append('/'+i)
    return lol

async def check(url,suffix,client,boolean):
    global failed
    # checks the HTTP response (if site is good or bad)
    if boolean:
        if url.endswith('/'):
            url = url[0:-1]
            nurl = url + suffix
        else:
            nurl = url + suffix
    else:nurl = urlparse(url).scheme + '://' + urlparse(url).netloc + suffix
    try:
        resp = await client.get(nurl)
        code = resp.status
        if code == 200:
            ok(nurl)
            ok_sites.append(nurl)
        else:failed += 1;minus(f'{nurl}')
    except:failed += 1;minus(f'{nurl}')

async def check_urllist(url,suffixes,boolean):
    # intuitive
    global total
    async with aiohttp.ClientSession() as session:
        tasks = []
        for suffix in suffixes:
            total += 1
            task = asyncio.create_task(check(url,suffix,session,boolean))
            tasks.append(task)
        await asyncio.gather(*tasks)

def adminFinder(url,suffixfile,boolean):
    # main function
    try:
        f = open(suffixfile)
        lst = f.read().splitlines()
        new_lst = check_suffixes(lst)
        asyncio.new_event_loop().run_until_complete(check_urllist(url,new_lst,boolean))
    except Exception as e:
        error(f'Exception! [{e}]')
    return ok_sites,total,failed


import urllib2
import sys
import os.path

import lxml.html as html
import pandas as pd

reddit_url = 'https://reddit.com'
start_url = 'https://reddit.com/r/jokes'
headers = { 'User-Agent': '/u/Broscious' }
upvote_threshold = 100

def url_open(url):
    req = urllib2.Request(url, headers=headers)
    return urllib2.urlopen(req)

def retrieve_urls():
    root = html.parse(url_open(start_url)).getroot()
    scores = root.xpath('//div[@class="score unvoted"]/text()')
    relative_paths = root.xpath('//a[@class="title may-blank "]/@href')
    absolute_paths = [reddit_url + relative for relative in relative_paths]
    filtered_paths = []
    for url, upvotes in zip(absolute_paths, scores):
        try:
            value = int(upvotes)
        except:
            value = -1
        if value >= upvote_threshold:
            filtered_paths.append(url)
    return filtered_paths

def retrieve_joke(url):
    root = html.parse(url_open(url)).getroot()
    setup = root.xpath('//a[@class="title may-blank "]/text()')[0]
    content = root.xpath('//div[@class="expando expando-uninitialized"]//p/text()')
    return setup, content

def one_line_filter(setup, content):
    for paragraph in content[1:]:
        low_para = paragraph.lower()
        if low_para.startswith('edit:'):
            continue
        else:
            return None

    punchline = content[0]
    if punchline.count('.') > 1:
        return None

    return setup, punchline

def filter_results(urls, results):
    returns = []
    for url, result in zip(urls, results):
        title, content = result
        joke = one_line_filter(title, content)
        if joke is not None:
            returns.append((url, joke[0], joke[1]))
    return returns

def save_results(one_liners, joke_file, url_file):
    used_urls = pd.read_csv(url_file)
    added_urls = []
    added_jokes = []
    for url, setup, punchline in one_liners:
        if not (used_urls['url'] == url).any():
            added_urls.append(url)
            added_jokes.append((setup, punchline))
    url_df = pd.DataFrame(added_urls, columns=['url'])
    jokes_df = pd.DataFrame(added_jokes, columns=['Joke', 'Punchline'])

    with open(url_file, 'a') as f:
        url_df.to_csv(f, index=False, header=False, encoding='utf-8')

    with open(joke_file, 'a') as f:
        jokes_df.to_csv(f, index=False, header=False, encoding='utf-8')

def ensure_existance(joke_file, url_file):
    if not os.path.isfile(url_file):
        with open(url_file, 'w') as f:
            f.write(u'url\n')

    if not os.path.isfile(joke_file):
        with open(joke_file, 'w') as f:
            f.write(u'Joke, Punchline\n')

def main():
    joke_file = sys.argv[1]
    url_file = sys.argv[2]
    ensure_existance(joke_file, url_file)
    urls = retrieve_urls()
    results = [retrieve_joke(url) for url in urls]
    one_liners = filter_results(urls, results)
    save_results(one_liners, joke_file, url_file)

if __name__ == '__main__':
    main()