#!/usr/bin/env python
# encoding: utf-8

# The MIT License (MIT)

# Copyright (c) 2016 CNRS

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# AUTHORS
# Hervé BREDIN - http://herve.niderb.fr

import warnings
import itertools
import numpy as np
from scipy.spatial.distance import pdist, squareform
from pyannote.generators.batch import BaseBatchGenerator
from pyannote.audio.generators.labels import FixedDurationSequences
from pyannote.audio.embedding.callbacks import UpdateGeneratorEmbedding

class TripletGenerator(object):
    """Triplet generator for triplet loss sequence embedding

    Generates ([Xa, Xp, Xn], 1) tuples where
      * Xa is the anchor sequence (e.g. by speaker S)
      * Xp is the positive sequence (also uttered by speaker S)
      * Xn is the negative sequence (uttered by a different speaker)

    and such that d(f(Xa), f(Xn)) < d(f(Xa), f(Xp)) + margin where
      * f is the current state of the embedding network (being optimized)
      * d is the (euclidean or cosine) distance
      * margin is the triplet loss margin (e.g. 0.2, typically)

    Parameters
    ----------
    extractor: YaafeFeatureExtractor
        Yaafe feature extraction (e.g. YaafeMFCC instance)
    file_generator: iterable
        File generator (the training set, typically)
    distance: {'sqeuclidean', 'cosine'}
        Distance for which the embedding is optimized. Defaults to 'sqeuclidean'.
    margin : float, optional
        Defaults to 0.2.
    duration: float, optional
    step: float, optional
        Duration and step of sliding window (in seconds).
        Default to 3s and half duration.
    min_duration: float, optional
        Sequence minimum duration. When provided, generates sequences with
        random duration in range [min_duration, duration]. Defaults to
        fixed-duration sequences.
    per_label: int, optional
        Number of samples per label. Defaults to 40.
    per_fold: int, optional
        When provided, randomly split the training set into
        fold of `per_fold` labels (e.g. 40) after each epoch.
        Defaults to using the whole training set.
    batch_size: int, optional
        Batch size. Defaults to 32.
    """

    def __init__(self, extractor, file_generator,
                 distance='sqeuclidean', margin=0.2,
                 duration=3.0, min_duration=None, step=None,
                 per_fold=0, per_label=40, batch_size=32):

        super(TripletGenerator, self).__init__()

        self.extractor = extractor
        self.file_generator = file_generator
        self.distance = distance
        self.margin = margin
        self.duration = duration
        self.min_duration = min_duration
        self.step = step
        self.per_fold = per_fold
        self.per_label = per_label
        self.batch_size = batch_size

        self.generator_ = FixedDurationSequences(
            self.extractor,
            duration=self.duration,
            min_duration=self.min_duration,
            step=self.step,
            batch_size=-1)

        self.triplet_generator_ = self.iter_triplets()

        # consume first element of generator
        # this is meant to pre-generate all labeled sequences once and for all
        # and get the number of unique labels into self.n_labels
        next(self.triplet_generator_)

    def iter_triplets(self):

        # pre-generate all labeled sequences (from the whole training set)
        # this might be huge in memory
        X, y = [], []
        for batch_sequences, batch_labels in self.generator_(self.file_generator):
            X.append(batch_sequences)
            y.append(batch_labels)
        X = np.vstack(X)
        y = np.hstack(y)

        # unique labels
        unique, y, counts = np.unique(y, return_inverse=True, return_counts=True)
        self.n_labels = len(unique)

        # warn that some labels have very few training samples
        too_few_samples = np.sum(counts < self.per_label)
        if too_few_samples > 0:
            msg = '{n} labels (out of {N}) have less than {per_label} training samples.'
            warnings.warn(msg.format(n=too_few_samples,
                                     N=self.n_labels,
                                     per_label=self.per_label))

        # HACK (see __init__ for details on why this is done)
        yield

        # infinite loop
        while True:

            # shuffle labels
            shuffled_labels = np.random.choice(self.n_labels,
                                               size=self.n_labels,
                                               replace=False)

            if self.per_fold < 1:
                self.per_fold = self.n_labels

            # take them per_fold per per_fold
            for k in range(self.n_labels / self.per_fold):
                from_label = k * self.per_fold
                to_label = (k+1) * self.per_fold
                labels = shuffled_labels[from_label: to_label]

                # select min(per_label, count) sequences
                # at random for each label

                # per_label[i] contains the actual number of examples
                # available for ith label -- as it may actually be smaller
                # than the requested self.per_label for small classes (e.g.
                # for speakers without only a few seconds of speech)
                per_label = []

                # indices contains the list of indices of all sequences
                # to be used for later triplet selection
                indices = []

                for label in labels:

                    # number of available sequences for current label
                    # per_label.append(min(self.per_label, counts[label]))
                    per_label.append(self.per_label)

                    # NOTE the impact of choosing per_label instead of
                    # min(per_label, counts[label]) should be evaluated.
                    # indeed, for labels with a counts[label] smaller than
                    # per_label, the following 'choice' will repeat sequences.
                    # is this really an issue?

                    # randomly choose this many sequences
                    # from the set of available sequences
                    i = np.random.choice(
                        np.where(y == label)[0],
                        size=per_label[-1],
                        replace=True)

                    # append indices of selected sequences
                    indices.append(i)

                # after this line, per_label[i] will contain the position of
                # the first sequence of ith label so that the range
                # per_label[i]: per_label[i+1] points to the indices
                # corresponding to all sequences from ith label
                per_label = np.hstack([[0], np.cumsum(per_label)])

                # turn indices into a 1-dimensional numpy array.
                # combined with (above) per_label, it can be used
                # to get all indices of sequences from a given label
                indices = np.hstack(indices)

                # pre-compute pairwise distances d(f(X), f(X')) between every
                # pair (X, X') of selected sequences, where f is the current
                # state of the embedding being optimized, and d is the
                # euclidean distance

                # selected sequences
                sequences = X[indices]
                # their embeddings (using current state of embedding network)
                embeddings = self.embedding.transform(
                    sequences, batch_size=self.batch_size)

                # pairwise squared euclidean distances
                distances = squareform(pdist(embeddings, metric=self.distance))

                for i in range(self.per_fold):

                    positives = list(range(per_label[i], per_label[i+1]))
                    negatives = list(range(per_label[i])) + list(range(per_label[i+1], per_label[-1]))

                    # loop over all (anchor, positive) pairs for current label
                    for anchor, positive in itertools.combinations(positives, 2):

                        # find all negatives within the margin
                        d = distances[anchor, positive]
                        within_margin = np.where(
                            distances[anchor, negatives] < d + self.margin)[0]

                        # choose one at random (if at least one exists)
                        if len(within_margin) < 1:
                            continue
                        # TODO / add an option to choose the most difficult one
                        negative = negatives[np.random.choice(within_margin)]

                        yield [sequences[anchor], sequences[positive], sequences[negative]], 1

                        # FIXME -- exit this loop when an epoch has ended
                    # FIXME -- exit this loop when an epoch has ended
                # FIXME -- exit this loop when an epoch has ended

    def __iter__(self):
        return self

    def next(self):
        return self.__next__()

    def __next__(self):
        return next(self.triplet_generator_)

    @property
    def shape(self):
        return self.generator_.shape

    def signature(self):
        shape = self.shape
        return (
            [
                {'type': 'ndarray', 'shape': shape},
                {'type': 'ndarray', 'shape': shape},
                {'type': 'ndarray', 'shape': shape}
            ],
            {'type': 'scalar'}
        )

    def callbacks(self, extract_embedding=None):
        callback = UpdateGeneratorEmbedding(
            self, extract_embedding=extract_embedding, name='embedding')
        return [callback]

class TripletBatchGenerator(BaseBatchGenerator):
    """Triplet generator for triplet loss sequence embedding

    Generates ([Xa, Xp, Xn], 1) batch tuples where
      * Xa are anchor sequences (e.g. by speaker S)
      * Xp are positive sequences (also uttered by speaker S)
      * Xn are negative sequences (uttered by a different speaker)

    and such that d(f(Xa), f(Xn)) < d(f(Xa), f(Xp)) + margin where
      * f is the current state of the embedding network (being optimized)
      * d is the euclidean distance
      * margin is the triplet loss margin (e.g. 0.2, typically)

    Parameters
    ----------
    feature_extractor: YaafeFeatureExtractor
        Yaafe feature extraction (e.g. YaafeMFCC instance)
    file_generator: iterable
        File generator (the training set, typically)
    distance: {'sqeuclidean', 'cosine'}
        Distance for which the embedding is optimized. Defaults to 'sqeuclidean'.
    margin : float, optional
        Defaults to 0.2.
    duration: float, optional
    step: float, optional
        Duration and step of sliding window (in seconds).
        Default to 3s and half step.
    min_duration: float, optional
        Sequence minimum duration. When provided, generates sequences with
        random duration in range [min_duration, duration]. Defaults to
        fixed-duration sequences.
    per_label: int, optional
        Number of samples per label. Defaults to 40.
    per_fold: int, optional
        Randomly split the training set into disjoint folds of `per_fold`
        labels. Defaults to using one big fold.
    batch_size: int, optional
        Batch size. Defaults to 32.
    """
    def __init__(self, feature_extractor, file_generator,
                 distance='sqeuclidean', margin=0.2,
                 duration=3.0, min_duration=None, step=None,
                 per_fold=0, per_label=40, batch_size=32):

        self.triplet_generator_ = TripletGenerator(
            feature_extractor, file_generator,
            margin=margin, distance=distance,
            duration=duration, min_duration=min_duration, step=step,
            per_fold=per_fold, per_label=per_label, batch_size=batch_size)

        super(TripletBatchGenerator, self).__init__(
            self.triplet_generator_, batch_size=batch_size)

    def signature(self):
        return self.triplet_generator_.signature()

    @property
    def shape(self):
        return self.triplet_generator_.shape

    def get_steps_per_epoch(self, protocol, subset='train'):
        """
        Parameters
        ----------
        protocol : pyannote.database.protocol.protocol.Protocol
        subset : {'train', 'development', 'test'}, optional

        Returns
        -------
        steps_per_epoch : int
            Number of batches per epoch.
        """
        n_labels = len(protocol.stats(subset)['labels'])
        per_label = self.triplet_generator_.per_label
        samples_per_epoch = per_label * (per_label - 1) * n_labels
        return samples_per_epoch // self.batch_size

    def callbacks(self, extract_embedding=None):
        return self.triplet_generator_.callbacks(
            extract_embedding=extract_embedding)


#!/usr/bin/env python3
import json
import os
import sys
from timeit import default_timer

from dataReadWrite.NativeReadWrite import NativeReadWrite
from dataReadWrite.ReadWriteAll import ALGORITHMS
from graphHDS.prune_cluster import ALGORITHMS as SHAVING_ALGORITHMS
from graphHDS.pseudo_stability import get_algo_cluster_stabilities
from labeling.LabelingManager import LabelingManager
from util import convert_connections_to_edges
from util import shave_clustering, SHAVING_METHOD

def main():
    from argparse import ArgumentParser

    parser = ArgumentParser(description="generates labeling file")

    parser.add_argument("--staging-dir", "-s", required=True,
                        help="root staging dir of auto-hds g")
    parser.add_argument("--experiment-name", "-e", required=True,
                        help="Experiment name usually yyyy-mm-dd")
    parser.add_argument("--algo", "-a", required=True,
                        choices=ALGORITHMS.keys(),
                        help="clustering algorithm to use")
    parser.add_argument("--top-k", "-k", type=int, required=True,
                        help="top k clusters from HMETIS")
    parser.add_argument("--k-dithered", "-kd", type=int, required=True,
                        help="k dithered")
    parser.add_argument("--shaving-algo", "-sa", choices=SHAVING_ALGORITHMS,
                        help="shaving algorithm to use (1=sorted by flow to "
                             "node, 2=incremental shaving by flow to node)")
    parser.add_argument("--max-sample", "-ms", type=int, required=True,
                        help="Max sample per cluster")
    parser.add_argument("--seed", "-sd", type=int, required=True, help="seed")
    parser.add_argument("--shave-type", "-t", required=True,
                        choices={"rand_shave", "flow_shave"},
                        help="shave type to do on non autohds-g algos")
    parser.add_argument("--params-suffix", "-p", required=False, default=None, help="the params file location. the"
                                                                                " program will look for a file with"
                                                                                " judge_params.[suffix].json.")
    parser.add_argument('--debug', action="store_true", default=False,
                        help="Debug allows individual loaders and algo to sample for testing",)

    args = parser.parse_args()

    staging_dir = os.path.expanduser(args.staging_dir)
    experiment_name = args.experiment_name
    algo = args.algo
    top_k = args.top_k
    k_dithered = args.k_dithered
    shaving_algo = args.shaving_algo
    max_sample = args.max_sample
    seed = args.seed
    shave_type = args.shave_type
    params_suffix = args.params_suffix
    debug = args.debug

    shaving_method = SHAVING_METHOD.RAND if shave_type == "rand_shave" else SHAVING_METHOD.FLOW

    if shaving_method == SHAVING_METHOD.FLOW and shaving_algo is None:
        parser.error("--shaving-algo/-sa is required if shaving method is flow_shave")

    if not os.path.isdir(staging_dir):
        print("Could not find staging dir: {}".format(staging_dir))
        sys.exit(1)

    if params_suffix is None:
        judge_params_file = os.path.join(staging_dir, "judge_params.json")
    else:
        judge_params_file = os.path.join(staging_dir, "judge_params.{}.json".format(params_suffix))

    if not os.path.exists(judge_params_file):
        print("You need to run generate_genediver_judging_file.py before running this!")
        sys.exit(1)
    with open(judge_params_file, "r") as f:
        judge_params = json.load(f)
        for _var in ["k_deduped", "fraction_kept", "num_points_in_graph", "jaccard_thresh"]:
            if _var not in judge_params:
                print("Required param {} not found in judge_param file at: {}".format(_var, judge_params_file))
    k_deduped = judge_params["k_deduped"]
    fraction_kept = judge_params["fraction_kept"]
    num_points_in_graph = judge_params["num_points_in_graph"]
    jaccard_thresh = judge_params["jaccard_thresh"]
    keep_disconnected_nodes = judge_params["keep_disconnected_nodes"]

    if shaving_algo not in SHAVING_ALGORITHMS:
        raise ValueError("shaving algorithm '{}' does not exist".format(shaving_algo))
    if k_deduped < top_k:
        raise Exception("k_deduped is {} but the number of algo clusters is only {}".format(top_k, k_deduped))

    native_read_writer = NativeReadWrite(staging_dir, experiment_name)

    # ==LOAD INPUT GRAPH to get the number of points for clustering_edges==
    print("Loading graph from file...", end="", flush=True)
    start_time = default_timer()
    connection_graph, num_points, num_edges = native_read_writer.read_graph(jaccard_threshold=jaccard_thresh)

    input_graph = convert_connections_to_edges(connection_graph)
    del connection_graph

    if num_points != num_points_in_graph:
        print("Graph has changed since last genediver judge file generation.")
        print("You need to run generate_genediver_judging_file.py before running this!")
        sys.exit(1)
    print(" done. (time={:.3f} s)".format(default_timer() - start_time))
    print()

    if params_suffix is None:
        algo_staging_dir = os.path.join(staging_dir, algo)
    else:
        algo_staging_dir = os.path.join(staging_dir, algo + "." + params_suffix)

    if not os.path.isdir(algo_staging_dir):
        print("Could not find algo staging dir: {}, please run graph algo first!".format(algo_staging_dir))
        sys.exit(1)

    hdsg_to_external_id_mapping = native_read_writer.read_mapping()

    # ==READ CLUSTERING FILE OUTPUT==
    print("Loading clustering from file...", end="", flush=True)
    start_time = default_timer()
    # TODO: warn when num_algo_clusters does not match the filename
    reader = ALGORITHMS[algo](algo_staging_dir)
    cluster_points = reader.read_clustering()
    if not keep_disconnected_nodes:

        reverse_node_id_mapping = dict()
        with open(os.path.join(algo_staging_dir, "id_mapping.tsv")) as f:
            for line in map(str.strip, f):
                old_node_id, new_node_id = line.split("\t")
                reverse_node_id_mapping[int(new_node_id)] = int(old_node_id)

        for cluster_id, cluster in cluster_points.items():
            cluster_points[cluster_id] = set(map(reverse_node_id_mapping.get, cluster))

    print(" done. (time={:.3f} s)".format(default_timer() - start_time))

    # ==SHAVE CLUSTERS (if filtering didn't already shave enough)==
    print("Sample proportion: {}".format(fraction_kept))

    print("Shaving clusters using algorithm {}...".format(shaving_algo))
    start_time = default_timer()
    # shave clusters down to the same number of nodes clustered as autoHDS-G
    shaved_cluster_points = shave_clustering(
        num_points_in_graph=num_points_in_graph,
        clustering=cluster_points,
        method=shaving_method,
        fraction_kept=fraction_kept,
        seed=seed,
        input_graph=input_graph,
        shaving_algo=shaving_algo,
        debug=debug
    )

    num_shaved_cluster_points = LabelingManager.count_unique_points_in_clusters(shaved_cluster_points)
    print("num shaved cluster points: {}".format(num_shaved_cluster_points))
    print("Finished shaving clusters in (time={:.3f} s)".format(default_timer() - start_time))
    print()

    # ==FILTER CLUSTERS==
    # This will work even if top_k = num_algo_clusters
    print("Getting the top {} clusters from the output of algo...".format(top_k), end="", flush=True)
    start_time = default_timer()
    # get cluster_stabilities
    cluster_stabilities = get_algo_cluster_stabilities(
        graph=input_graph,
        cluster_points=shaved_cluster_points,
    )

    filtered_cluster_points, top_k_cluster_ids, full_dithered_cluster_ids = LabelingManager.filter_clustering(
        clustering=shaved_cluster_points,
        cluster_stabilities=cluster_stabilities,
        top_k=top_k,
        k_dithered=k_dithered
    )
    num_filtered_points = LabelingManager.count_unique_points_in_clusters(filtered_cluster_points)
    print(" done. (time={:.3f} s)".format(default_timer() - start_time))
    print("Number of nodes in judged clusters before shaving: {}".format(num_filtered_points))

    if params_suffix:
        algo += "." + params_suffix

    LabelingManager.generate_judging_files(
        clustering=shaved_cluster_points,
        filtered_clustering=filtered_cluster_points,
        top_k_cluster_ids=top_k_cluster_ids,
        k_dithered_cluster_ids=full_dithered_cluster_ids,
        algorithm_name=algo,
        max_sample=max_sample,
        seed=seed,
        cluster_stabilities=cluster_stabilities,
        staging_dir=staging_dir,
        hdsg_to_external_id_mapping=hdsg_to_external_id_mapping
    )

if __name__ == "__main__":
    main()


import requests
import bs4
from bs4 import BeautifulSoup
import re
import os
import traceback

def getCurrInfo(fpath):
    try:
        bocurl = 'http://www.usd-cny.com/bankofchina.htm'
        html = getHTML(bocurl)
        infoDict = {}
        soup = BeautifulSoup(html, 'html.parser')
        currInfo = soup.find('div', attrs={'class': 'pp'})
        name = currInfo.find_all('a')[0]
        infoDict.update({'»õ±ÒÃû³Æ': name.text.split()[0]})

        keyList = currInfo.find_all('th')
        valueList = currInfo.find_all('td')
        keyList1 = currInfo.find_all('a')
        print('count country', len(keyList1))
        for i in range(len(keyList1)):
            key = keyList1[i].text
            val = valueList[i].text
            infoDict[key] = val

        with open(fpath, 'w', encoding='utf-8') as f:
            f.write(str(infoDict) + '\n')
    except:
        traceback.print_exc()


from django.contrib import admin
from .models import *
from django.utils.translation import ugettext_lazy as _

@admin.register(Student)
class StudentAdmin(admin.ModelAdmin):
    fields = ('s_number', 's_name', 's_surname', 's_education', 's_is_dgs', 'dgs_file')
    list_display = (
        's_number', 's_name', 's_surname', 's_education', 's_is_dgs'
    )
    list_filter = ('s_number', 's_education')
    search_fields = ('s_number', 's_name ', 'company_name')

@admin.register(Internship)
class InternshipAdmin(admin.ModelAdmin):
    fields = (
        'i_student', 'company_name', 'city', 's_date', 'f_date',
        's_day', 'i_subject', 'is_arge'
    )
    list_display = (
        'i_student', 'company_name', 'city', 's_date', 'f_date',
        's_day', 'i_subject', 'is_arge'
    )
    list_filter = (
        'i_student', 'company_name', 'city', 's_date', 'f_date',
        's_day', 'i_subject', 'is_arge'
    )
    search_fields = ('i_student__s_number',)

@admin.register(Commission)
class CommissionAdmin(admin.ModelAdmin):
    fields = ('c_name', 'c_surname', 'c_internship')
    list_display = ('get_commission', 'c_surname')
    list_filter = ('c_name', 'c_surname')
    search_fields = ('c_name', 'c_surname')

    def get_commission(self, obj):
        commission = '-'.join(
            [commission.c_name for commission in obj.commission.all()]
        )
        return commission

@admin.register(Interview)
class InterviewAdmin(admin.ModelAdmin):
    fields = ('internship', 'commission', 'i_datetime')
    list_display = ('internship', 'i_datetime', 'get_commission')
    list_filter = ('internship', 'commission', 'i_datetime')
    search_fields = ('internship', 'commission', 'i_datetime')

    def get_commission(self, obj):
        commission = '-'.join(
            [commission.c_name for commission in obj.commission.all()]
        )
        return commission

@admin.register(NoteScale)
class NoteScaleAdmin(admin.ModelAdmin):
    fields = (
        'n_internship', 'continuation', 'd_w_on_time', 'b_a_the_officer',
        'b_a_co_workers', 'project', 'regularity', 'presentations',
        'content', 'interview'
    )
    list_display = (
        'n_internship', 'continuation', 'd_w_on_time', 'b_a_the_officer',
        'b_a_co_workers', 'project', 'regularity', 'presentations', 'content',
        'interview'
    )
    list_filter = (
        'n_internship', 'continuation', 'd_w_on_time', 'b_a_the_officer',
        'b_a_co_workers', 'project', 'regularity', 'presentations', 'content',
        'interview'
    )
    search_fields = (
        'n_internship', 'continuation', 'd_w_on_time', 'b_a_the_officer',
        'b_a_co_workers', 'project', 'regularity', 'presentations', 'content',
        'interview'
    )

    def get_n_internship(self, obj):
        n_internship = '-'.join(
            [n_internship.interview for n_internship in obj.n_internship.all()]
        )
        return n_internship


import sys
sys.path.append('../config')
from config.Config import Config
from subprocess import Popen
from subprocess import PIPE

class HandBrakeAdapter:

    @staticmethod
    def run(inputFile, outputFile):
        command = HandBrakeAdapter.__buildCommand(inputFile, outputFile)
        p = Popen(command, stdout=PIPE, stderr=PIPE)
        stdout, stderr = p.communicate()
        if p.returncode != 0:
            raise Exception(stderr)

    @staticmethod
    def __buildCommand(inputFile, outputFile):
        command = []
        command.append('HandBrakeCLI')
        command.append('-i')
        command.append(inputFile)
        command.append('-o')
        command.append(outputFile)
        command.append('-x')
        command.append(Config.get('handbrake.profile'))
        return command


import os
import numpy as np
import multiprocessing
from functools import partial
from keras.preprocessing.image import Iterator, _count_valid_files_in_directory, _list_valid_filenames_in_directory, \
    load_img, img_to_array
import keras.backend as K

class DirectoryIteratorWithFname(Iterator):
    """Iterator capable of reading images from a directory on disk.
    # Arguments
        directory: Path to the directory to read images from.
            Each subdirectory in this directory will be
            considered to contain images from one class,
            or alternatively you could specify class subdirectories
            via the `classes` argument.
        image_data_generator: Instance of `ImageDataGenerator`
            to use for random transformations and normalization.
        target_size: tuple of integers, dimensions to resize input images to.
        color_mode: One of `"rgb"`, `"grayscale"`. Color mode to read images.
        classes: Optional list of strings, names of sudirectories
            containing images from each class (e.g. `["dogs", "cats"]`).
            It will be computed automatically if not set.
        class_mode: Mode for yielding the targets:
            `"binary"`: binary targets (if there are only two classes),
            `"categorical"`: categorical targets,
            `"sparse"`: integer targets,
            `"input"`: targets are images identical to input images (mainly
                used to work with autoencoders),
            `None`: no targets get yielded (only input images are yielded).
        batch_size: Integer, size of a batch.
        shuffle: Boolean, whether to shuffle the data between epochs.
        seed: Random seed for data shuffling.
        data_format: String, one of `channels_first`, `channels_last`.
        save_to_dir: Optional directory where to save the pictures
            being yielded, in a viewable format. This is useful
            for visualizing the random transformations being
            applied, for debugging purposes.
        save_prefix: String prefix to use for saving sample
            images (if `save_to_dir` is set).
        save_format: Format to use for saving sample images
            (if `save_to_dir` is set).
    """

    def __init__(self, directory, image_data_generator,
                 target_size=(256, 256), color_mode='rgb',
                 classes=None, class_mode='categorical',
                 batch_size=32, shuffle=True, seed=None,
                 data_format=None,
                 save_to_dir=None, save_prefix='', save_format='png',
                 follow_links=False, subset=None, interpolation='nearest'):
        if data_format is None:
            data_format = K.image_data_format()
        self.directory = directory
        self.image_data_generator = image_data_generator
        self.target_size = tuple(target_size)
        if color_mode not in {'rgb', 'grayscale'}:
            raise ValueError('Invalid color mode:', color_mode,
                             '; expected "rgb" or "grayscale".')
        self.color_mode = color_mode
        self.data_format = data_format
        if self.color_mode == 'rgb':
            if self.data_format == 'channels_last':
                self.image_shape = self.target_size + (3,)
            else:
                self.image_shape = (3,) + self.target_size
        else:
            if self.data_format == 'channels_last':
                self.image_shape = self.target_size + (1,)
            else:
                self.image_shape = (1,) + self.target_size
        self.classes = classes
        if class_mode not in {'categorical', 'binary', 'sparse',
                              'input', None}:
            raise ValueError('Invalid class_mode:', class_mode,
                             '; expected one of "categorical", '
                             '"binary", "sparse", "input"'
                             ' or None.')
        self.class_mode = class_mode
        self.save_to_dir = save_to_dir
        self.save_prefix = save_prefix
        self.save_format = save_format
        self.interpolation = interpolation

        white_list_formats = {'png', 'jpg', 'jpeg', 'bmp'}

        # first, count the number of samples and classes
        self.samples = 0

        if not classes:
            classes = []
            for subdir in sorted(os.listdir(directory)):
                if os.path.isdir(os.path.join(directory, subdir)):
                    classes.append(subdir)
        self.num_class = len(classes)
        self.class_indices = dict(zip(classes, range(len(classes))))

        def _recursive_list(subpath):
            return sorted(os.walk(subpath, followlinks=follow_links), key=lambda tpl: tpl[0])

        pool = multiprocessing.pool.ThreadPool()
        function_partial = partial(_count_valid_files_in_directory,
                                   white_list_formats=white_list_formats,
                                   split=None,
                                   follow_links=follow_links)
        self.samples = sum(pool.map(function_partial,
                                    (os.path.join(directory, subdir)
                                     for subdir in classes)))

        print('Found %d images belonging to %d classes.' % (self.samples, self.num_class))

        # second, build an index of the images in the different class subfolders
        results = []

        self.filenames = []
        self.classes = np.zeros((self.samples,), dtype='int32')
        i = 0
        for dirpath in (os.path.join(directory, subdir) for subdir in classes):
            results.append(pool.apply_async(_list_valid_filenames_in_directory,
                                            (dirpath, white_list_formats, None,
                                             self.class_indices, follow_links)))
        for res in results:
            classes, filenames = res.get()
            self.classes[i:i + len(classes)] = classes
            self.filenames += filenames
            i += len(classes)
        pool.close()
        pool.join()
        super(DirectoryIteratorWithFname, self).__init__(self.samples, batch_size, shuffle, seed)

    def next(self):
        """For python 2.x.
        # Returns
            The next batch.
        """
        with self.lock:
            index_array = next(self.index_generator)
            current_batch_size = index_array.size

        # The transformation of images is not under thread lock
        # so it can be done in parallel
        batch_x = np.zeros((current_batch_size,) + self.image_shape, dtype=K.floatx())
        batch_fn = ['' for _ in range(current_batch_size)]
        grayscale = self.color_mode == 'grayscale'
        # build batch of image data
        for i, j in enumerate(index_array):
            fname = self.filenames[j]
            img = load_img(os.path.join(self.directory, fname),
                           grayscale=grayscale,
                           target_size=self.target_size,
                           interpolation=self.interpolation)
            x = img_to_array(img, data_format=self.data_format)
            x = self.image_data_generator.random_transform(x)
            x = self.image_data_generator.standardize(x)
            batch_x[i] = x
            batch_fn[i] = fname.split(os.path.sep)[-1]
        # optionally save augmented images to disk for debugging purposes
        if self.save_to_dir:
            for i, j in enumerate(index_array):
                img = array_to_img(batch_x[i], self.data_format, scale=True)
                fname = '{prefix}_{index}_{hash}.{format}'.format(prefix=self.save_prefix,
                                                                  index=j,
                                                                  hash=np.random.randint(1e7),
                                                                  format=self.save_format)
                img.save(os.path.join(self.save_to_dir, fname))
        # build batch of labels
        if self.class_mode == 'input':
            batch_y = batch_x.copy()
        elif self.class_mode == 'sparse':
            batch_y = self.classes[index_array]
        elif self.class_mode == 'binary':
            batch_y = self.classes[index_array].astype(K.floatx())
        elif self.class_mode == 'categorical':
            batch_y = np.zeros((len(batch_x), self.num_class), dtype=K.floatx())
            for i, label in enumerate(self.classes[index_array]):
                batch_y[i, label] = 1.
        else:
            return batch_x, batch_fn
        return batch_x, batch_y, batch_fn


"""Test module for the `map_columns` processor."""

from copy import deepcopy
from pytest import fixture
from common.processors.map_columns import build_lookup_table, apply_mapping

@fixture(scope='function')
def lookup_table():
    """The lookup table is a generator so we need a new one for each test.
    """
    datapackage = {
        'resources': [
            {
                'schema': {
                    'fields': [
                        {
                            'name': 'foo',
                            'maps_to': 'project_id'
                        },
                        {
                            'name': 'bar',
                            'maps_to': 'invalid_fiscal_field'
                        },
                        {
                            'name': 'spam',
                            'maps_to': None
                        },
                        {
                            'name': 'eggs',
                        },

                    ]

                }
            }
        ]
    }
    return build_lookup_table(datapackage)

@fixture(scope='function')
def input_row():
    """Row objects are mutable so we need a new one for each test.
    """
    return {
        'foo': 1,
        'bar': 2,
        'spam': 3,
        'eggs': 4
    }

# noinspection PyShadowingNames
def test_build_lookup_table_returns_correct_mapping(lookup_table):
    assert dict(lookup_table) == {'foo': 'project_id'}

# noinspection PyShadowingNames
def test_apply_mapping_toggles_the_correct_fields(input_row, lookup_table):
    output_row = apply_mapping(deepcopy(input_row), mapping=lookup_table)
    assert set(output_row) ^ set(input_row) == {'project_id', 'foo'}

# noinspection PyShadowingNames
def test_apply_mapping_returns_correct_row(input_row, lookup_table):
    expected_output = {
        'project_id': 1,
        'bar': 2,
        'spam': 3,
        'eggs': 4
    }
    assert apply_mapping(input_row, mapping=lookup_table) == expected_output

# noinspection PyShadowingNames
def test_apply_mapping_with_empty_lookup_returns_identical_row(input_row):
    assert apply_mapping(input_row, mapping=None) == input_row


from django.core.urlresolvers import reverse
from rest_framework.status import HTTP_200_OK, HTTP_204_NO_CONTENT, \
    HTTP_401_UNAUTHORIZED
from rest_framework.test import APITestCase

from users.tests.factories import UserFactory

class UserViewTestCase(APITestCase):
    """Tests for UserView."""

    def setUp(self):
        self.api_url = reverse('api:users:user')

    def test_self_view_for_not_authenticated_user(self):
        response = self.client.get(self.api_url, format='json')

        self.assertEqual(response.status_code, HTTP_401_UNAUTHORIZED)
        self.assertEqual(response.data['detail'],
                         'Authentication credentials were not provided.')

    def test_self_view_for_authenticated_user(self):
        user = UserFactory()
        self.client.force_authenticate(user=user)

        response = self.client.get(self.api_url, format='json')

        self.assertEqual(response.status_code, HTTP_200_OK)
        self.assertEqual(response.data['email'],
                         user.email)
        self.assertEqual(response.data['full_name'],
                         user.full_name)

class PublicUserViewTestCase(APITestCase):
    """Tests for PublicUserView."""

    def test_for_not_public_user_profile(self):
        private_user = UserFactory()
        api_url = reverse('api:users:public-user',
                          kwargs={'user_id': private_user.pk})

        response = self.client.get(api_url, format='json')
        self.assertEqual(response.status_code, HTTP_204_NO_CONTENT)

    def test_for_public_user_profile(self):
        public_user = UserFactory(is_public=True)
        api_url = reverse('api:users:public-user',
                          kwargs={'user_id': public_user.pk})

        response = self.client.get(api_url, format='json')
        self.assertEqual(response.status_code, HTTP_200_OK)
        self.assertEqual(response.data['email'],
                         public_user.email)
        self.assertEqual(response.data['full_name'],
                         public_user.full_name)


import kombu
import time
import json
import sys

from villas.controller.command import Command
from villas.controller.commands.simulator import SimulatorCommand

class MonitorCommand(Command):

    @staticmethod
    def add_parser(subparsers):
        parser = subparsers.add_parser('monitor', help='Listen to events')
        parser.set_defaults(func=MonitorCommand.run)

        filt = parser.add_argument_group('Filter simulators')
        filt.add_argument('-r', '--realm')
        filt.add_argument('-c', '--category')
        filt.add_argument('-t', '--type')
        filt.add_argument('-u', '--uuid')

    @staticmethod
    def run(connection, args):
        exchange = kombu.Exchange(name='villas', type='headers')

        headers = SimulatorCommand.get_headers(args)
        headers['x-match'] = 'any' if len(headers) > 0 else 'all'

        queue = kombu.Queue(exchange=exchange,
                            binding_arguments=headers,
                            durable=False)

        consumer = kombu.Consumer(connection,
                                  queues=queue,
                                  on_message=MonitorCommand.on_message)

        try:
            with consumer:
                while True:
                    connection.drain_events()
        except KeyboardInterrupt:
            pass

    @staticmethod
    def on_message(message):
        entry = {
            'time': time.time(),
            'payload': message.payload,
        }

        entry.update(message.properties)

        sys.stdout.write('%s\n' % json.dumps(entry))
        sys.stdout.flush()


import keyboard
while True:
    print("_____________________________Tamrin #4 _________________________________\n\n")
    MySentence = "farshad,nematpour,25,Ahvaz,Mashhad,Azad-University,Data-Minning"
    print("Our words are: ",MySentence)
    input("enter for Spliting: ")

    MySentence_Splited = MySentence.split(",")
    print("Word after packaging : ", MySentence_Splited)

    print("\n___________________________Farshad NematPour ___________________________\n\n")
    keyboard.wait('q')
    keyboard.send('ctrl+6')


##Narcissit bot
import tweepy
import time

auth=tweepy.OAuthHandler(consumer_key,consumer_secret)
auth.set_access_token(access_token,access_token_secret)
api=tweepy.API(auth)
user=api.me()

def limit_handler(cursor):
	try:
		while True:
			yield cursor.next()
	except tweepy.RateLimitError:
		time.sleep(1000)

search_string='Rojan'
numberOfTweets=2

for tweet in tweepy.Cursor(api.search,search_string).items(numberOfTweets):
	try:
		#tweet.retweet()
		tweet.favorite()
		print('I liked this tweet')

	except tweepy.TweepError as e:
		print(e.reason)

	except StopIteration:
		break


#-*- coding:utf-8 -*-
#import csv
import os, shutil
import time
#import threading

#from kivy.uix.label import Label
#from kivy.uix.button import Button

#import pyaudio
#import wave
import playsound as pls

from gtts import gTTS
import speech_recognition as sr

import songBox_playerPyaudio as sbp

nowDate =  time.strftime("%Y_%m_%d")

baseDir = os.getcwd()#os.path.abspath('py_song')#workging 폴더
mp3Dir = os.path.join(baseDir, "songBox_list")#노래 폴더
fontDir = os.path.join(baseDir, "songBox_font")#글자 폴더
imgDir = os.path.join(baseDir, "songBox_img")#그림 폴더
dataDir = os.path.join(mp3Dir, "0_file_data")#데이터목록 csv 저장폴더
userListDir = os.path.join(mp3Dir, "1_dir_userlist")#userlist 폴더
singerListDir = os.path.join(mp3Dir, "2_dir_singerlist")#userlist 폴더
ttsDir = os.path.join(mp3Dir, "3_tts")#tts 폴더
ignoreFile = ['ffmpeg','youtube-dl.exe','.DS_Store','list_data','0_file_data','1_dir_userlist','2_dir_singerlist','3_tts']#dir 안에 존재해야만하는 파일 but mp3 아닌 파일 목록

winColor =[211/255, 211/255, 211/255, 1]#[189/255,183/255,110/255,1]#[211/255,211/255,211/255,1]#
boxColor = [188/255, 143/255, 143/255, 1]#[110/255,0/255,0/255,1]#[105/255,105/255,105/255,1]#
textColor = [255/255, 255/255, 255/255, 1]
stopColor = [105/255,105/255,105/255,1]#gold[189/255,183/255,110/255,1]#red[110/255,0/255,0/255,1]

bartextColor =[189/255,183/255,110/255,1]#deepblue+green[30/255,160/255,180/255,1]#grey[105/255,105/255,105/255,1]#red[110/255,0/255,0/255,1]

soundbar_fontsize="22sp"
menu_fontsize="20sp"
text_fontsize="17sp"

#tempObj = Label(text='1')
#tempLabel=Label(text='ready')
#tempBtn=Button(text='temp')

#==============first msg========================================================
def play_msg(msg):
    print(f"play_msg:{play_msg}")
    get_msg = msg
    flag = False

    if msg == 'Tell me the song title':
        pls.playsound(f'{ttsDir}\\startmsg.mp3', False)
        duration = sbp.cal_duration(f"{ttsDir}\\startmsg.mp3")
        time.sleep(duration)
    else:
        tts = gTTS(text=f"{get_msg}",lang="en",tld="com")
        tts.save(f"{ttsDir}\\msg.mp3")

        pls.playsound(f'{ttsDir}\\msg.mp3', False)
        duration = sbp.cal_duration(f"{ttsDir}\\msg.mp3")
        time.sleep(duration)
        os.remove(f"{ttsDir}\\msg.mp3")

    if msg != 'Tell me the song title':
        flag = True

    return flag

#==============yes or no========================================================
def yesno_voice():
    print(f"start_yesno_voice")
    start = time.time()
    r = sr.Recognizer()
    m = sr.Microphone()
    audio=False

    while audio == False:
        try:
            with m as source:
                print(f'yes or no:')
                audio = r.listen(source)

        except Exception as errmsg:
                yesno_msg = f"{errmsg}retry."
                print(f"{errmsg}retry.")

                return yesno_msg

    try:
        yesno_msg=r.recognize_google(audio)
        print(f'answer_yes or no:{yesno_msg}')

        return yesno_msg.lower()

    except Exception as errmsg:
        yesno_msg=errmsg
        return yesno_msg

#==============start voice conversation=========================================
def searchsong_voice():
    print(f"start_searchsong_voice")
    msg = 'Tell me the song title'
    yesno_msg = ""
    yesno_msg_sec = ""
    flag = play_msg(msg)
    flag = False
    audio = False

    start = time.time()
    r = sr.Recognizer()
    m = sr.Microphone()

    #==============소리 감지할 때까지 ===============================================
    while audio == False:

        try:
            with m as source:
                print(f'speak:')
                audio = r.listen(source)
                flag=True

            #==============time check===========================================
            if time.time()-start > 10:
                print(f"I waited {time.time()-start} seconds")
                msg = f"I waited {int(time.time()-start)} seconds. Will I wait for you to speak?"
                flag = play_msg(msg)

                while yesno_msg !="no" and yesno_msg !="yes":
                    yesno_msg = yesno_voice()

                    if yesno_msg == "no":
                        msg = f"Ok. Bye"
                        flag = play_msg(msg)
                        print(f"wait_no{yesno_msg}")
                        return False, msg

                    elif yesno_msg == "yes":
                        print(f"wait_yes{yesno_msg}")
                        return searchsong_voice()

                    else:
                        msg = f"I can't understand your answer. Tell me one more time."
                        flag = play_msg(msg)

        #==============exception check==========================================
        except Exception as errmsg:
                print(f"{errmsg}")
                flag = play_msg(msg)

                while yesno_msg !="no" and yesno_msg !="yes":
                    yesno_msg = yesno_voice()

                    if yesno_msg == "no":
                        msg = f"Ok. Bye"
                        flag = play_msg(msg)
                        print(f"first_except_no{yesno_msg}")
                        return False, msg

                    elif yesno_msg == "yes":
                        print(f"first_except_yes{yesno_msg}")
                        return searchsong_voice()
                    else:
                        msg = f"I can't understand your answer. Tell me one more time."
                        flag = play_msg(msg)

    #==============소리 감지 & flag == True=======================================
    if flag == True and audio != False:
        try:
            speechSong=r.recognize_google(audio)
            print(f'You said:{speechSong}')
            msg = f'You said:{speechSong}. Did you say this song title?'
            play_msg(msg)

            while yesno_msg !="no" and yesno_msg !="yes":

                yesno_msg = yesno_voice()

                if yesno_msg == "no":
                    msg = f'Would you like to try again?'
                    print(f"second_no{yesno_msg}")
                    play_msg(msg)
                    yesno_msg_sec

                elif yesno_msg == "yes":
                    print(f"second_yes{yesno_msg}")
                    return search_song(speechSong)

                else:
                    msg = f"I can't understand your answer. Tell me one more time."
                    play_msg(msg)

            if yesno_msg == "no":
                while yesno_msg_sec !="no" or yesno_msg_sec !="yes":
                    yesno_msg_sec = yesno_voice()

                    if yesno_msg_sec == "no":
                        msg = f"Ok. Bye"
                        flag = play_msg(msg)
                        print(f"second_no_no{yesno_msg_sec}")
                        return False, msg

                    elif yesno_msg_sec == "yes":
                        print(f"second_no_yes{yesno_msg_sec}")
                        return searchsong_voice()
                    else:
                        msg = f"I can't understand your answer. Tell me one more time."
                        flag = play_msg(msg)

        except Exception as errmsg:
                print(f"I can't recognize the voice.{errmsg}")
                msg = f"I can't recognize the voice. Would you like to try again?"
                flag = play_msg(msg)

                while yesno_msg !="no" and yesno_msg !="yes":
                    yesno_msg = yesno_voice()

                    if yesno_msg == "no":
                        msg = f"Ok. Bye"
                        flag = play_msg(msg)
                        print(f"second_except_no{yesno_msg}")
                        return False, msg

                    elif yesno_msg == "yes":
                        print(f"second_except_yes{yesno_msg}")
                        return searchsong_voice()

                    else:
                        msg = f"I can't understand your answer. Tell me one more time."
                        flag = play_msg(msg)

#==============search song > return result =====================================
def search_song(speechSong):
    global GET_PLAY_BTN

    yesno_msg=''
    print(f'start search_song:{speechSong}')

    try:
        songList=os.listdir(f'{mp3Dir}')

        for i in songList:
            if i not in ignoreFile:

                title=i.split('_')

                if speechSong.lower() == title[0].lower():
                    print(f"find {i}")

                    return True, i

        msg = f"This song title {speechSong} could not be found. Would you like to try again?"
        res = play_msg(msg)

        while yesno_msg !="no" and yesno_msg !="yes":
            yesno_msg = yesno_voice()

            if yesno_msg == "no":
                msg = f"Ok. Bye"
                flag = play_msg(msg)
                print(f'search_song__no:{yesno_msg}')
                return False, msg

            elif yesno_msg == "yes":
                print(f'search_song__yes:{yesno_msg}')
                return searchsong_voice()

            else:
                msg = f"I can't understand your answer. Tell me one more time."
                flag = play_msg(msg)

    except Exception as errmsg:
            print(f"I can't find this song {speechSong}. {errmsg}")
            msg = f"I can't find this song {speechSong}. Would you like to try again?"
            flag = play_msg(msg)

            while yesno_msg !="no" and yesno_msg !="yes":
                yesno_msg = yesno_voice()

                if yesno_msg == "no":
                    msg = f"Ok. Bye"
                    flag = play_msg(msg)
                    print(f'search_song_except_no:{yesno_msg}')
                    return False, msg

                elif yesno_msg == "yes":
                    print(f'search_song_except_yes:{yesno_msg}')
                    return searchsong_voice()

                else:
                    msg = f"I can't understand your answer. Tell me one more time."
                    flag = play_msg(msg)

"""
def recognize_test(play_btn):
    #print(play_btn,pause_btn)
    get_PLAY_BTN = play_btn
    #get_PAUSE_BTN = pause_btn
    #os.system(f"open {baseDir}/searhsong.mp3")
    #searchsong = SoundLoader.load(f"{baseDir}/searchsong.mp3")
    #searchsong.play()
    #searchsong.volume = 1.0
    #searchsong.loop = False
    audio = False
    start = time.time()
    r = sr.Recognizer()
    m = sr.Microphone()

    while audio == False:
        try:
            with m as source:
                print(f'speak:')
                audio = r.listen(source)
            if time.time()-start > 10:
                sbp.get_restart()
                print("Waited 10 seconds")
                return 0
        except Exception as msg:
                print(f"{msg}retry.")
                return 0

    try:
        speechText=r.recognize_google(audio)
        print(f'You said:{speechText}')

    except Exception as msg:
            sbp.get_restart()
            print(f"can't recognize the voice. {msg}")
            return 0

    if audio != False:

        songList=os.listdir(f'{mp3Dir}')
        for i in songList:
            if i not in ignoreFile:
                title=i.split('_')
                if speechText.lower() == title[0].lower():
                    print(f"find {i}")

                    #get_PAUSE_BTN.text = "|||"
                    #get_PAUSE_BTN.color = boxColor
                    playTitle = [i]
                    playing = [tempObj,]
                    replay=True
                    playnum=0

                    sbp.get_restart()
                    thread_one = threading.Thread(target = sbp.get_playThread, args = (playTitle,replay,playing,get_PLAY_BTN,tempBtn,tempLabel,playnum), daemon=True)
                    thread_one.start()
                    return 0

        #nosong = SoundLoader.load(f"{baseDir}/nosong.mp3")
        #nosong.play()
        #nosong.loop = False
        #os.system(f"open {baseDir}/nosong.mp3")
        sbp.get_restart()
        print(f"can't find {speechText}")
        return 0
"""


'''
Define crossover operation over two equal-length strings A and B as follows:

the result of that operation is a string of the same length as the input strings
result[i] is either A[i] or B[i], chosen at random
Given array of strings inputArray and a string result, find for how many pairs of strings from inputArray the result of the crossover operation over them may be equal to result.

Note that (A, B) and (B, A) are the same pair. Also note that the pair cannot include the same element of the array twice (however, if there are two equal elements in the array, they can form a pair).

Example

For inputArray = ["abc", "aaa", "aba", "bab"] and result = "bbb", the output should be
stringsCrossover(inputArray, result) = 2.

Input/Output

[execution time limit] 4 seconds (py3)

[input] array.string inputArray

A non-empty array of equal-length strings.

Guaranteed constraints:
2 ≤ inputArray.length ≤ 10,
1 ≤ inputArray[i].length ≤ 10.

[input] string result

A string of the same length as each of the inputArray elements.

Guaranteed constraints:
result.length = inputArray[i].length.

[output] integer
'''

from itertools import combinations

def stringsCrossover(inputArray, result):
    combs = list(combinations(inputArray, 2))
    count = 0
    for combination in combs:
        found = True
        for i, char in enumerate(result):
            if char != combination[0][i] and char != combination[1][i]:
                found = False
                break
        if found:
            count += 1
    return count

inputArray = ["aa", "ab", "ba"]
result = "bb"

print(stringsCrossover(inputArray, result))


# coding=utf-8
# Copyright 2018 The TensorFlow Datasets Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Download manager interface.

"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import collections
import contextlib
import os

import concurrent.futures
import six
from tensorflow import gfile

from tensorflow_datasets.core import utils
from tensorflow_datasets.core.download import local_backend
from tensorflow_datasets.core.download import util
from tensorflow_datasets.core.proto import download_generated_pb2 as download_pb2

# Number of thread to use to parallelize the extractions
_NUM_PARALLEL_DOWNLOADS = 50
_NUM_PARALLEL_EXTRACTS = 1

TAR_EXT = ['.tar', '.tar.gz', '.tgz', '.tar.bz2', '.tbz2', '.tbz', '.tb2']
# Tuple to match extensions with extraction function.
ExtractFormat = collections.namedtuple('ExtractFormat', 'ext, fn')

class DownloadManager(object):
  """Class which manages the download and extraction of data.

  This has the following advantage:
   * Remove the boilerplate code common to each dataset as now the user only
     has to provide urls.
   * Allow multiple download backends (local and cloud). Otherwise the user
     may be tempted to use standard Python function (ex: tarfile) which may
     not be compatible with one of the backend.
   * Allow to launch multiple download in parallel.
   * Better ressources management (ex: a same url used by multiple
     datasets is downloaded/extracted only once).

  The function members accept either plain value, or values wrapped into list
  or dict. Giving a data structure will parallelize the downloads.

  Example:

    # Sequential download: str -> str
    train_dir = dl_manager.download_and_extract('https://abc.org/train.tar.gz')
    test_dir = dl_manager.download_and_extract('https://abc.org/test.tar.gz')

    # Parallel download: list -> list
    image_files = dl_manager.download(
        ['https://a.org/1.jpg', 'https://a.org/2.jpg', ...])

    # Parallel download: dict -> dict
    data_dirs = dl_manager.download_and_extract({
       'train': 'https://abc.org/train.zip',
       'test': 'https://abc.org/test.zip',
    })
    data_dirs['train']
    data_dirs['test']

  For more customization on the download/extraction (ex: passwords, output_name,
  ...), you can give UrlInfo() or ExtractInfo() instead of plain string.

  It is also possible to use the download manager to process a custom function
  with the same guaranties/features than download and extraction.

    # The "/vocab/en-fr" path is cached and can be reused accros run. If
    # the path already exists, the cached value is re-used
    vocab_dir = dl_manager.execute_and_cache(generate_vocab_fn, '/vocab/en-fr')

  """

  def __init__(self, cache_dir, manual_dir=None, mode=None):
    """Download manager constructor.

    Args:
      cache_dir: `str`, Cache directory where all downloads, extractions and
        other artifacts are stored.
      manual_dir: `str`, Directory containing manually downloaded data. Default
        to cache_dir.
      mode (GenerateMode): Mode to FORCE_REDOWNLOAD, REUSE_CACHE_IF_EXISTS or
        REUSE_DATASET_IF_EXISTS. Default to REUSE_DATASET_IF_EXISTS.
    """
    self._cache_dir = os.path.expanduser(cache_dir)
    self._manual_dir = os.path.expanduser(manual_dir or cache_dir)
    self._backend = local_backend.LocalBackend()

    # The generation mode to indicates if we re-use the cached download or
    # force re-downloading data.
    mode = mode or util.GenerateMode.REUSE_DATASET_IF_EXISTS
    self._mode = util.GenerateMode(mode)  # str to Enum

    # Create the root directory if not exists yet
    gfile.MakeDirs(self._cache_dir)

  # Public API

  def download(self, urls_info):
    """Downloads the given urls.

    If one of the download already exists, the cached value will be reused.

    Args:
      urls_info (UrlInfo): The url to download. If a string is passed, it will
        automatically be converted into UrlInfo object

    Returns:
      downloaded_filepaths (str): The downloaded files paths
    """

    # We may want to add a pack kwarg such to keep all downloads in the same
    # directory

    def _download(url_info):
      """Function applied for each individual download."""
      # Convert string to UrlInfo object
      url_info = to_url_info(url_info)

      # Download the url
      return self._process_and_cache_uri(
          uri=url_info.url,
          uri_info=url_info,
          process_trial_fn=self._download,
      )

    # Run the download function on each of the urls
    return _parallel_run(
        _download,
        urls_info,
        max_workers=_NUM_PARALLEL_DOWNLOADS,
    )

  def extract(self, extracts_info):
    """Extract the given path.

    If one of the extraction already exists, the cached value will be reused.

    Args:
      extracts_info (ExtractInfo): The path to extract. If a string is passed,
        it will automatically be converted into ExtractInfo object

    Returns:
      extracted_dirs (str): The downloaded files
    """

    def _extract(extract_info):
      """Function applied for each individual extraction."""
      # Convert string to UrlInfo object
      extract_info = to_extract_info(extract_info)

      # Create the extract uri from the path
      if not extract_info.path.startswith(self._cache_dir):
        raise ValueError(
            'Trying to extract a file ({}) which is not in the cache of the '
            'download manager ({})'.format(extract_info.path, self._cache_dir))
      extract_uri = util.lchop(extract_info.path, self._cache_dir)
      # TODO(epot): There is a risk to reach the max length of filename (256
      # characters in linux and 1k total on CNS)
      extract_uri = os.path.join('extract://', extract_uri.strip('/'))

      # Process an extraction
      return self._process_and_cache_uri(
          uri=extract_uri,
          uri_info=extract_info,
          process_trial_fn=self._extract,
      )

    # Run the extract function on each of the paths
    return _parallel_run(
        _extract,
        extracts_info,
        max_workers=_NUM_PARALLEL_EXTRACTS,
    )

  def download_and_extract(self, urls_info):
    """Convinience method to perform download+extract in a single command.

    As most downloads are imediatelly followed by an extraction, this
    function avoid some boilerplate code. It is equivalent to:
      extracted_dirs = dl_manager.extract(dl_manager.download(urls_info))

    Args:
      urls_info (UrlInfo): Same input as .download()

    Returns:
      extracted_dir (str): Same output as .extract()
    """
    # Maybe the input could accept a UrlExtractInfo(url_info, extract_info)
    # which would contains both UrlInfo and ExtractInfo as currently only
    # the url can be controlled through the arguments.
    return self.extract(self.download(urls_info))

  def execute_and_cache(self, process_fn, cache_key):
    """Execute the function and cache the associated path.

    This function executes the process_fn using a custom cached directory given
    by the key. This allow to use the download manager to cache and share
    additional data like vocabulary or other intermediate artifacts.

    The key are shared between datasets which allow re-using data from other
    datasets but also increase the risk of name collision. A good practice
    is to add the dataset name as prefix of the key.

      vocab_cache_key = os.path.join(self.name, 'vocab/en-fr')
      vocab_dir = dl_manager.execute_and_cache(
          generate_vocab_fn,
          cache_key=vocab_cache_key,
      )

    Args:
      process_fn (fct): Function with signature "(cache_dir) -> None" which
        takes as input a directory on which storing the additional data.
      cache_key (str): Key of the cache to store the data. If the key already
        exists, the process_fn isn't used and the previously cached dir is
        reused.

    Returns:
      cache_dir (str): The directory on which the additional data has been
        written.
    """
    # TODO(epot): Replace this and extract uri by the urijoin ?
    cache_uri = os.path.join('local://', cache_key.strip('/'))

    def inner_process(trial):
      # Create the directory on which generate the additional data
      gfile.MakeDirs(trial.output_path)
      # Call the function to process the additional data
      return process_fn(trial.output_path)

    return self._process_and_cache_uri(
        uri=cache_uri,
        uri_info=None,
        process_trial_fn=inner_process,
    )

  @property
  def manual_dir(self):
    """Returns the directory containing the manually extracted data."""
    if not gfile.Exists(self._manual_dir):
      raise AssertionError(
          'Manual directory {} does not exist. Create it and download/extract'
          'dataset artifacts in there.'.format(self._manual_dir)
      )
    return self._manual_dir

  # Internal functions

  def _process_and_cache_uri(self, uri, uri_info, process_trial_fn):
    """Internal function which manage the cache and launch the trials.

    Args:
      uri (str): Uri to search for in the cache
      uri_info (obj): Additional info for the uri (UrlInfo or ExtractInfo), will
        be saved with the trial
      process_trial_fn (fct): Function with the signature "(UriTrial) -> None"
        which is called if the uri isn't found in the cache

    Returns:
      output_path (str): The output from the Trial
    """

    # Maybe should normalize the uri so that semantically equivalent urls
    # are detected as the same (ex: "http://a.io/" and "http://a.io") and
    # stored in the same register. Or issue guidelines about the url format.

    # Either get the previous cached location or build a new trial
    trial = self._get_or_create_trial(uri=uri, uri_info=uri_info)

    if trial.status != download_pb2.UriTrial.COMPLETED:
      with self._process_trial_controllers(trial):
        process_trial_fn(trial)

    return trial.output_path  # Return cached or processed trial

  def _get_or_create_trial(self, uri, uri_info=None):
    """Create a new trial or get the previous one.

    The previous trials are recreated by looking at the content of the cached
    dir.

    Args:
      uri (str): Uri to create the trial for.
      uri_info (obj): Object containing additional info about the download
        or extraction (UrlInfo or ExtractInfo).

    Returns:
      trial (UriTrial): Result of the trial containing the destination, status,
        timestamp,...
    """
    # The generation is deterministic so generating keys for the same uri will
    # always gives the same result
    trial_id = '{}_{}'.format(
        util.escape_uri(uri),
        util.hash_uri(uri),
    )

    log = util.build_log(prefix=trial_id)

    # Generate a new trial to eventually use
    trial = download_pb2.UriTrial(
        id=trial_id,
        status=download_pb2.UriTrial.IN_PROGRESS,
        output_path=os.path.join(self._cache_dir, trial_id),
    )
    add_uri_info(trial, uri, uri_info)

    if gfile.Exists(trial.output_path):

      # If the directory exists, the previous trial was complete (as it was
      # renamed successfully from ".incomplete")
      if self._mode == util.GenerateMode.FORCE_REDOWNLOAD:
        log('Cleanup previous trial: {}', trial.output_path)
        gfile.DeleteRecursively(trial.output_path)
      else:
        log('Reusing previously cached data...')
        # Try to reuse the previous download
        trial.status = download_pb2.UriTrial.COMPLETED

        # For the downloads, the output_path contains the file
        # TODO(epot): Should instead write the meta-data on disk (in a
        # ._trial.json) and replace ListDirectory() by a version which filter
        # the metadata file.
        is_dl = not any(uri.startswith(p) for p in ('local://', 'extract://'))
        is_gz = (
            uri.startswith('extract://') and
            uri.endswith('.gz') and
            not uri.endswith('tar.gz')
        )
        if is_dl or is_gz:
          trial.output_path = get_download_filepath(trial)
    else:
      log('No cached value found.')

    return trial

  def _process_trial_controllers(self, trial):
    """Decorators to apply before and after the process_trial_fn."""
    return use_incomplete_dir(trial)

  def _download(self, trial):
    """Downloads a single url given by the trial (thread safe).

    Args:
      trial (UriTrial): Object containing info about download.

    Raises:
      ValueError: If the destination dir is not empty
    """
    log = util.build_log(prefix=trial.id)

    # Check the download dir is empty
    if (gfile.Exists(trial.output_path) and
        gfile.ListDirectory(trial.output_path)):
      raise ValueError('Download dir {} should be empty'.format(
          trial.output_path))

    gfile.MakeDirs(trial.output_path)

    log('Start downloading...')
    self._backend.download(trial)

    # TODO(epot): Compute the checksum

    # Update the output path
    trial.output_path = get_download_filepath(trial)

    log('Download complete at {}', trial.output_path)

  def _extract(self, trial):
    """Extract a single file given by the trial.

    Args:
      trial (UriTrial): Object containing info about the extraction.

    Raises:
      ValueError: If the format is incorrect
    """
    log = util.build_log(prefix=trial.id)

    src = trial.extract_info.path
    dst = trial.output_path

    rar_format = ExtractFormat(ext=TAR_EXT, fn=self._backend.extract_tar)
    zip_format = ExtractFormat(ext=['.zip'], fn=self._backend.extract_zip)
    gz_format = ExtractFormat(ext=['.gz'], fn=self._backend.extract_gzip)
    # Order matter as '.tar.gz' will call _extract_tar while '.gz' will
    # call _extract_gzip
    extraction_fns = collections.OrderedDict([
        (download_pb2.ExtractInfo.RAR, rar_format),
        (download_pb2.ExtractInfo.ZIP, zip_format),
        (download_pb2.ExtractInfo.GZ, gz_format),
    ])

    # Filetype explicitly defined
    if trial.extract_info.filetype != download_pb2.ExtractInfo.UNKNOWN:
      extract_filetype = trial.extract_info.filetype
      extract_fn = extraction_fns[extract_filetype]
    # Try to infer the filetype from the name
    else:
      for extract_filetype, extract_format in extraction_fns.items():
        if any(src.lower().endswith(ext) for ext in extract_format.ext):
          extract_fn = extract_format.fn
          break
      else:  # No break (unrecognized archive)
        raise ValueError(
            'Unsuported archive file {} for trial {}. If you think this is an '
            'error, you can try to explicitly define the type in the '
            'ExtractFileType'.format(src, trial.id))

    log('Extract {} with {}...', src, extract_fn.__name__)
    gfile.MakeDirs(dst)
    extract_fn(src, dst)

    if extract_filetype == download_pb2.ExtractInfo.GZ:
      trial.output_path = get_download_filepath(trial)

def to_url_info(value):
  """Convert the strings into UrlInfo."""
  if isinstance(value, six.string_types):
    return download_pb2.UrlInfo(url=value)
  elif isinstance(value, download_pb2.UrlInfo):
    return value
  else:
    raise ValueError('Could not convert {} to UrlInfo'.format(value))

def to_extract_info(value):
  """Convert the strings into ExtractInfo."""
  if isinstance(value, six.string_types):
    return download_pb2.ExtractInfo(path=value)
  elif isinstance(value, download_pb2.ExtractInfo):
    return value
  else:
    raise ValueError('Could not convert {} to UrlInfo'.format(value))

def add_uri_info(trial, uri, uri_info):
  """Add the uri info to the UriTrial."""
  # Add the eventual info
  if uri.startswith('local://'):
    pass
  elif uri.startswith('extract://'):
    trial.extract_info.CopyFrom(uri_info)
  else:
    trial.url_info.CopyFrom(uri_info)

def get_download_filepath(trial):
  """Extract the downloaded file from the completed trial."""
  # Get the downloaded file:
  files = list(gfile.ListDirectory(trial.output_path))
  if not files:  # Should have been catched before, but just in case
    raise ValueError('Download {} failed!'.format(trial.id))
  elif len(files) > 1:
    raise ValueError('Multiple files detected for {}.'.format(trial.id))

  # Update the output path
  filename = files[0]
  return os.path.join(trial.output_path, filename)

@contextlib.contextmanager
def use_incomplete_dir(trial):
  """Wrap the trial in a temporary .incomplete path while it is processed."""
  # Replace the output dir by a temporary dir
  output_path_original = trial.output_path
  # Should add random string to avoid collision with local download manager ?
  output_path_tmp = trial.output_path + '.incomplete'
  trial.output_path = output_path_tmp
  yield
  if not trial.output_path.startswith(output_path_tmp):
    raise ValueError(
        'The output path for {} has been modified to {} and do not match '
        'the original {}'.format(trial.id, trial.output_path, output_path_tmp))
  gfile.Rename(output_path_tmp, output_path_original)
  output_path_extension = util.lchop(trial.output_path, output_path_tmp)
  trial.output_path = output_path_original + output_path_extension

def _parallel_run(function, input_struct, max_workers=1):
  """Run the function on each element of data_struct using a pool of workers."""

  # Distribute the work in a pool
  launch_thread_pool = concurrent.futures.ThreadPoolExecutor
  with launch_thread_pool(max_workers=max_workers) as executor:

    def launch_worker(value):
      return executor.submit(function, value)

    output_struct = utils.map_nested(launch_worker, input_struct)

  # Gather all results once all workers have finished
  def gather_results(value):
    return value.result()

  return utils.map_nested(gather_results, output_struct)


import os
import json
import datetime
import re

insert_command = "INSERT INTO news(title, body, author_name, category, published_time, expire_time, source) VALUES("

MONTH = {
    'ianuarie': 1,
    'februarie': 2,
    'martie': 3,
    'aprilie': 4,
    'mai': 5,
    'iunie': 6,
    'iulie': 7,
    'august': 8,
    'septembrie': 9,
    'octombrie': 10,
    'noiembrie': 11,
    'decembrie': 12,
}

def get_real_data():
    commands = []
    #commands.append("CREATE INDEX ir_translation_ltns ON news ((md5(body)))")
    for f in os.listdir(os.path.join(os.getcwd(), 'jsons')):
        fpath = os.path.join(os.getcwd(), 'jsons', f)
        with open(fpath, 'rb') as fin:
            jfile = json.loads(fin.read().decode('utf-8'))
            jfile['content'] = jfile['content'].replace('\'', '')
            if 'FII' in jfile['source']:
                day = int(jfile['date'].split(' ')[0])
                month = MONTH[jfile['date'].split(' ')[1].split(',')[0]]
                year = int(jfile['date'].split(' ')[2])
                published_date = datetime.datetime(year=year, month=month, day=day)
                valabil_date = datetime.datetime(year=2020, month=month, day=day)
            elif 'UAIC' in jfile['source']:
                day = int(jfile['date'].split(' ')[0])
                month = MONTH[jfile['date'].split(' ')[1].split(',')[0]]
                year = int(jfile['date'].split(' ')[2])
                published_date = datetime.datetime(year=year, month=month, day=day)
                valabil_date = datetime.datetime(year=2020, month=month, day=day)
            elif 'ORAR' in jfile['source']:
                day = int(jfile['publish_date'].split('.')[0])
                month = int(jfile['publish_date'].split('.')[1])
                year = int(jfile['publish_date'].split('.')[2])
                published_date = datetime.datetime(year=year, month=month, day=day)
                day = int(jfile['valabil_date'].split('.')[0])
                month = int(jfile['valabil_date'].split('.')[1])
                year = int(jfile['valabil_date'].split('.')[2])
                valabil_date = datetime.datetime(year=year, month=month, day=day)

            #contor = News.objects.raw('SELECT COUNT(*) FROM news WHERE body=\'' + jfile['content']+'\'');
            #print(contor);

            jfile['content']=jfile['content'].replace('=/bin', '=https://www.info.uaic.ro/bin')
            #print(jfile['content'])
            # object.body.replace('127.0.0.1:8000', 'https://www.info.uaic.ro')

            vectorImg = re.findall('.*.*', jfile['content'])
            #for image in vectorImg:
                #jfile['content'] = jfile['content'].replace(image, '')
            #print(vectorImg)
            #
            for image in vectorImg:
                imagines =re.findall('', image);
                #print(imagines);
                for imagePiece in imagines:
                    print(imagePiece)
                    pieceSource=re.findall('src="[^"]*"', imagePiece);
                    print('')
                    jfile['content']=jfile['content'].replace(imagePiece, '')
                    #print(piece)

            new_command = insert_command + '\'' + jfile['title'] + '\', \'' + jfile['content'] + \
                            '\', \'\', \'\', \'' + str(published_date) + '\', \'' + str(valabil_date) + '\',' + '\'' + jfile['source'] + '\') '
            # print(new_command)

            commands.append(new_command)
            # return commands
    return commands

if __name__ == "__main__":
    get_real_data()


# -*- coding: utf-8 -*-
"""
Created on Mon Mar 16 09:33:04 2020

空氣污染監測網 網路爬蟲實作練習
能夠利用 selenium + BeautifulSoup 撰寫動態爬蟲
空氣污染監測網 網頁爬蟲
如果利用原本的 Request 的方式，會發現怎麼拉都是空資料。原因在於資料一樣是透過 JavaScript 動態載入的。

@author: chenlw
"""

# 打開瀏覽器

from selenium import webdriver
from selenium.webdriver.support.ui import Select
from bs4 import BeautifulSoup
import time

browser = webdriver.Chrome(executable_path='chromedriver')

browser.get("http://taqm.epa.gov.tw/taqm/tw/MonthlyAverage.aspx")

time.sleep(1)
# 模擬使用者操作行為，選擇/點擊

selectSite = Select(browser.find_element_by_id("ctl05_ddlSite"))
selectSite.select_by_value('29')
selectYear = Select(browser.find_element_by_id("ctl05_ddlYear"))
selectYear.select_by_value('2014')

browser.find_element_by_id('ctl05_btnQuery').click()
time.sleep(1)
# 取得資料，丟到 BeautifulSoup 解析

html_source = browser.page_source

soup = BeautifulSoup(html_source, 'html.parser')
table = soup.find('table', class_='TABLE_G')
print(table)

# 關閉瀏覽器
browser.quit();


# based on the idea from
# https://github.com/andsens/bootstrap-vz/blob/5250f8233215f6f2e3a571be2f5cf3e09accd4b6/docs/transform_github_links.py
#
# Copyright 2013-2014 Anders Ingemann <anders@ingemann.de>
# Copyright 2016 Darragh Bailey <dbailey@hpe.com>

from docutils import nodes
import os.path

def transform_github_links(app, doctree, fromdocname):
    """Convert file references for github to correct target

    Scans the doctree for links directly referencing ReSTructured
    text documents within this repository. It converts these links
    to a suitable target for sphinx generated docs.

    Such references as <file>.rst are used by source code hosting
    sites such as GitHub when rendering documents directly from
    individual source files without parsing the entire doctree.

    However referencing the original <file>.rst is not useful for
    sphinx generated documentation as <file>.rst will not exist in
    the resulting documentation as it will also have been converted
    to the chosen format e.g. <file>.html

    Supporting automatic conversion ensures that GitHub/BitBucket
    and any other git hosting site performing rendering on a file
    by file basis allowing users to navigate through the documentation,
    while still ensuring the output from fully generated sphinx docs
    will point to the correct target.
    """

    try:
        target_format = app.builder.link_suffix
    except AttributeError:
        # if the builder has no link_suffix, then no need to modify
        # the current links.
        return

    source_suffix = app.config.source_suffix
    # Links are either absolute against the repository or relative to
    # the current document's directory. Note that this is not
    # necessarily app.srcdir, which is the documentation root
    # directory. Instead rely on 'source' attribute of doctree to
    # identify the path of the file providing the current doctree
    try:
        doc_path = doctree.attributes['source']
        doc_dir = os.path.dirname(doc_path)
    except KeyError:
        # some doctrees added by other libraries through dynamic
        # generation do not have a source file. Assume paths are
        # relative to the repo.
        doc_dir = ""

    for node in doctree.traverse(nodes.reference):
        if 'refuri' not in node:
            continue
        if node['refuri'].startswith('http'):
            continue

        try:
            link, anchor = node['refuri'].split('#', 1)
            anchor = '#' + anchor
        except ValueError:
            link = node['refuri']
            anchor = ''

        if link is None:
            continue

        # Replace the suffix with the correct target format file ending,
        # but only if the link ends with both the correct source suffix
        # and refers to a local file.
        for src_suffix in source_suffix:
            if link.endswith(src_suffix):
                # absolute paths are considered relative to repo
                if link.startswith("/"):
                    basepath = ""
                # relative paths are against the current doctree source path
                else:
                    basepath = doc_dir
                if os.path.exists(os.path.join(basepath, link)):
                    node['refuri'] = (link[:-len(source_suffix)] + target_format +
                                    anchor)

def setup(app):
    app.connect('doctree-resolved', transform_github_links)
    return {'version': '0.1'}


import sqlite3
from typing import *
conn = sqlite3.connect('MoneyTransfer.db')

c = conn.cursor()

#c.execute("INSERT INTO users VALUES ('Taylor', 'Swift', 500)")

t = ('Taylor',)
#c.execute("SELECT * FROM users WHERE username=?", t)
#values = ('150000$q4CeejTK$74036e73646a965131f622b2ee371849cbba7e224a00ca3b32ebe7c60c5d7dd6',)
#c.execute("SELECT * FROM accessTokens WHERE token=?", values)
#values = ('Philippejlyu',)
#c.execute("SELECT username FROM users WHERE username=?", values)
#print(c.fetchone())
#t = (42, 'cindy',)
#c.execute("UPDATE users SET balance=? WHERE username=?", t)
#values = ('asdf', 'ghjk', 50.45, 234)
#c.execute("INSERT INTO transactions VALUES ('%s', '%s', %f, %i)" % values)
t = ('philippejlyu',)
c.execute("SELECT * FROM transactions WHERE recipient=?", t)
print(c.fetchall())
conn.commit()
conn.close()

'''
def get_user_record(username: str) -> Optional[Tuple]:
    """
    Return a tuple of the user information
    """
    conn = sqlite3.connect("MoneyTransfer.db")
    c = conn.cursor()
    t = (username,)
    c.execute("SELECT * FROM users WHERE username=?", t)
    result = c.fetchone()
    c.close()
    conn.commit()
    return result

print(get_user_record('cindy'))
print(type(get_user_record('cindy')[2]))
'''


# the input of the network is the original image (3 channels, RGB) and one channel to indicate the segmentation prediction of the last frame

import numpy as np
from PIL import Image
import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import DataLoader
from torchvision import transforms
import torchvision.models as models
import os
from datetime import datetime
from collections import OrderedDict
import cv2

from Config_4C import Config_4C
from Foreground4C import Foreground4C
from DataSet_4C_mask import DataSet_4C_mask

def generate_folder_dataset(root_image_path, root_label_path, root_prediction_path):
  target_frame_name_png_list = os.listdir(root_prediction_path)
  target_frame_name_png_list.sort()

  image_path_list = []
  label_path_list = []
  prediction_path_list = []

  for target_frame_name_png in target_frame_name_png_list:
    target_frame_name = target_frame_name_png.split(".")[0]
    target_frame_name_jpg = target_frame_name + ".jpg"

    image_path = os.path.join(root_image_path, target_frame_name_jpg)
    label_path = os.path.join(root_label_path, target_frame_name_png)
    prediction_path = os.path.join(root_prediction_path, target_frame_name_png)

    image_path_list.append(image_path)
    label_path_list.append(label_path)
    prediction_path_list.append(prediction_path)

  folder_dataset = {"image": image_path_list, "label": label_path_list, "prediction": prediction_path_list}
  return folder_dataset

def main():

  if Config_4C.first_train == True:
    deeplab = torch.hub.load('pytorch/vision:v0.4.2', 'deeplabv3_resnet101', pretrained=True)
    deeplab = deeplab.cuda()
    foreground = Foreground4C(deeplab)
  else:
    foreground = torch.load(Config_4C.mask_weight_path)

  foreground = foreground.cuda()

  folder_dataset = generate_folder_dataset(Config_4C.root_image_path, Config_4C.root_label_path, Config_4C.root_prediction_path)

  transform_image = transforms.Compose([
    transforms.Resize((Config_4C.trans_resize_size, Config_4C.trans_resize_size)),
    transforms.ToTensor(),
    transforms.Normalize(mean=[0.485, 0.456, 0.406, 0.4],
                         std=[0.229, 0.224, 0.225, 0.22])
  ])

  transform_label = transforms.Compose([
    transforms.Resize((Config_4C.trans_resize_size, Config_4C.trans_resize_size)),
    transforms.ToTensor(),
  ])

  dataset_4C = DataSet_4C_mask(folder_dataset, transform_image, transform_label)

  train_dataloader = DataLoader(dataset_4C,
                                shuffle=True,
                                num_workers=Config_4C.worker_num,
                                batch_size=Config_4C.batch_num)

  optimizer = optim.Adam(foreground.parameters(), lr=Config_4C.lr)

  criterion = nn.BCELoss(size_average=True)

  start_time = datetime.now()

  for epoch_id in range(Config_4C.epoch_num):
    for iter_id, data in enumerate(train_dataloader, 0):

      try:
        inputs, labels = data
        inputs = inputs.cuda()
        labels = labels.cuda()

        foreground.zero_grad()

        fx = foreground(inputs)

        fx = fx.view(-1)
        y = labels.view(-1)

        seg_loss = criterion(fx, y)
        seg_loss.backward()
        optimizer.step()

        loss_value = seg_loss.detach().cpu().numpy()

        current_time = datetime.now()
        used_time = current_time - start_time

        print(str(epoch_id) + " : " + str(iter_id) + ", loss: " + str(loss_value) + ", time: " + str(used_time))
      except:
        pass

    if (epoch_id%Config_4C.save_epochs==0) and (epoch_id>0):
      #torch.save(foreground.state_dict(), Config_3C.weight_path)
      torch.save(foreground, Config_4C.mask_weight_path)
      print("model saved")

if __name__ == '__main__':
  main()


import json
import numpy as np
import os
from sklearn.model_selection import train_test_split
import cv2
import torch

TUSIMPLE_PATH='./train_set'

##标注并结构化图森数据集
class CreateTusimpleData():

    def __init__(self,tusimple,line_width):

        self.tusimple=tusimple
        self.line_width=line_width

    def __call__(self):
        if not os.path.exists('./data/train_binary'):
            os.mkdir('./data/train_binary')
        if not os.path.exists('./data/cluster'):
            os.mkdir('./data/cluster')
        if not os.path.exists('./data/LaneImages'):
            os.mkdir('./data/LaneImages')
        jsons=[json for json in os.listdir(self.tusimple) if json.split('.')[-1]=='json']
        for j in jsons:
            data=[]
            with open(os.path.join(self.tusimple,j)) as f:
                for line in f.readlines():
                    data.append(json.loads(line))
            for entry in data:
                height=entry['h_samples']
                width=entry['lanes']
                for index,w in enumerate(width):
                    counter=list(set(w))
                    if counter[0]==-2 and len(counter)==1:
                        del width[index]
                clip=entry['raw_file']
                img=cv2.imread(os.path.join(self.tusimple,clip))
                cv2.imwrite(os.path.join('./data/LaneImages','_'.join(clip.split('/')[1:])),img)
                img=cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
                img_binary=np.zeros_like(img)
                img_cluster=np.zeros_like(img)
                for lane in range(len(width)):
                    queue=[]
                    for h,w in zip(height,width[lane]):
                        if w<0:
                            continue
                        else:
                            queue.insert(0,(w,h))
                            if len(queue)==2:
                                cv2.line(img_binary,queue[0],queue[1],255,self.line_width)
                                cv2.line(img_cluster,queue[0],queue[1],255-50*lane,self.line_width)
                            if len(queue)>1:
                                queue.pop()
                new_name='_'.join(clip.split('/')[1:])
                new_name='.'.join([new_name.split('.')[0],'png'])

                cv2.imwrite(os.path.join('./data/train_binary',new_name),img_binary)
                cv2.imwrite(os.path.join('./data/cluster',new_name),img_cluster)

if __name__=='__main__':

    creator=CreateTusimpleData(TUSIMPLE_PATH,5)
    creator()