from cv2 import cv2
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
import os
import sys

'''
Pre-Processing Techniques
We do Pre-Processing images to improve the chances of successful in recognition.
1. De-skew — When the document is not aligned properly when scanned, it may need to be titled a few degrees clockwise or counterclockwise.In order to take it perfectly horizontal or vertical.
2. Despeckle — remove positive and negative spots, smoothing edges
3. Binarisation — Convert an image from color or greyscale to black-and-white (called a “binary image” because there are two colors). The task of binarisation is performed as a simple way of separating the text (or any other desired image component) from the background.
4. Line removal — Cleans up non-glyph boxes and lines
5. Layout analysis or “zoning” — Identifies columns, paragraphs, captions, etc. as distinct blocks.
6. Line and word detection — Establishes baseline for word and character shapes, separates words if necessary.
7. Character isolation or “segmentation” — For per-character OCR, multiple characters that are connected due to image artifacts must be separated; single characters that are broken into multiple pieces due to artifacts must be connected.
8. Normalize aspect ratio and scale.
'''

#some things to do in preprocessing
#Good Quality Original Source
#Binarize Image
#Scaling To The Right Size
#Increase Contrast
#Remove Noise and Scanning Artefacts
#Deskew
#Layout Analysis (or Zone Analysis)

if __name__ == "__main__":
    if len(sys.argv) != 2:
        sys.stderr.write(
            "You must pass image name as argument !")
        sys.exit(1)
    path = os.path.dirname(os.path.abspath(__file__))
    file=os.path.join(path, sys.argv[1])
    print(file)

    matplotlib.interactive(True)
    image = cv2.imread(file, cv2.IMREAD_GRAYSCALE)
    plt.figure(figsize=(8, 8))
    plt.imshow(image, cmap='gray')
    input('press return to continue')
    #cv2.imshow("Gray", grayimg)
    #cv2.waitKey(0)

    '''
    image_enhanced = cv2.equalizeHist(image)
    plt.imshow(image_enhanced, cmap='gray')
    input('press return to continue')
    contrast_fixed = cv2.convertScaleAbs(image, alpha=2.5, beta=50)
    plt.figure(figsize=(8, 8))
    plt.imshow(contrast_fixed, cmap='gray')
    input('press return to continue')
    #cv2.imshow("Contrast", contrast_fixed)
    #cv2.waitKey(0)
    '''

    #_, th = cv2.threshold(image, 200, 255, cv2.THRESH_BINARY_INV)
    th=cv2.adaptiveThreshold(image,255,cv2.ADAPTIVE_THRESH_GAUSSIAN_C,cv2.THRESH_BINARY,11,2)
    cv2.imwrite(file+"-result.png", th)
    fig = plt.figure(figsize=(8, 8))
    fig.add_subplot(1, 2, 1)
    plt.imshow(image, cmap='gray')
    fig.add_subplot(1, 2, 2)
    plt.imshow(th, cmap='gray')
    input('press return to continue')
    #cv2.imshow("Binarization", binary_thresh)
    #cv2.waitKey(0)

    lines = cv2.HoughLinesP(th, 1, np.pi/180, 100,
                            minLineLength= 600/2.0, maxLineGap=20)
    angle = 0
    for line in lines:
        x1, y1, x2, y2 = line[0]
        r = np.arctan2(y2 - y1, x2 -x1)
        angle += np.arctan2(y2 - y1, x2 - x1)

    avg_radian = angle / len(lines)
    avg_angle = avg_radian * 180 / np.pi
    print("Average angle is %f degrees" % avg_angle)


## /elf/utils.py

## Built-in
import os
import datetime
import threading

## Package
import __init__

def timestamp():
    x = datetime.datetime.today()
    return '{}-{:02d}-{:02d} {:02d}:{:02d}:{:02d}'.format(
        x.year, x.month, x.day, x.hour, x.minute, x.second)

def log(logfile, msg='', outdir = '.'):

    with log.lock:
        try: lock = log.filelocks[logfile]
        except KeyError:
            lock = threading.Lock()
            log.filelocks[logfile] = lock
    with lock:
        with open(os.path.join(outdir, logfile), mode='a') as w:
            w.write('{}\n'.format(msg if isinstance(msg,str) else '\t'.join(tuple(map(str,msg)))))
log.filelocks = {}
log.lock = threading.Lock()


import logging
import numpy as np
import os
import pickle
from scipy import sparse as sp
import six
import tensorflow as tf

from .loss_graphs import AbstractLossGraph, RMSELossGraph
from .prediction_graphs import AbstractPredictionGraph, DotProductPredictionGraph
from .recommendation_graphs import (
    project_biases, split_sparse_tensor_indices, bias_prediction_dense, bias_prediction_serial, rank_predictions,
    densify_sampled_item_predictions, collapse_mixture_of_tastes, predict_similar_items
)
from .representation_graphs import AbstractRepresentationGraph, LinearRepresentationGraph
from .session_management import get_session
from .util import sample_items, calculate_batched_alpha

class TensorRec(object):

    def __init__(self,
                 n_components=100,
                 n_tastes=1,
                 user_repr_graph=LinearRepresentationGraph(),
                 item_repr_graph=LinearRepresentationGraph(),
                 attention_graph=None,
                 prediction_graph=DotProductPredictionGraph(),
                 loss_graph=RMSELossGraph(),
                 biased=True,
                 normalize_users=False,
                 normalize_items=False):
        """
        A TensorRec recommendation model.
        :param n_components: Integer
        The dimension of a single output of the representation function. Must be >= 1.
        :param n_tastes: Integer
        The number of tastes/reprs to be calculated for each user. Must be >= 1.
        :param user_repr_graph: AbstractRepresentationGraph
        An object which inherits AbstractRepresentationGraph that contains a method to calculate user representations.
        See tensorrec.representation_graphs for examples.
        :param item_repr_graph: AbstractRepresentationGraph
        An object which inherits AbstractRepresentationGraph that contains a method to calculate item representations.
        See tensorrec.representation_graphs for examples.
        :param attention_graph: AbstractRepresentationGraph or None
        An object which inherits AbstractRepresentationGraph that contains a method to calculate user attention. Any
        valid repr_graph is also a valid attention graph. If None, no attention process will be applied.
        :param prediction_graph: AbstractPredictionGraph
        An object which inherits AbstractPredictionGraph that contains a method to calculate predictions from a pair of
        user/item reprs.
        See tensorrec.prediction_graphs for examples.
        :param loss_graph: AbstractLossGraph
        An object which inherits AbstractLossGraph that contains a method to calculate the loss function.
        See tensorrec.loss_graphs for examples.
        :param biased: bool
        If True, a bias value will be calculated for every user feature and item feature.
        :param normalize_users: bool
        If True, every row of the user features will be l2 normalized.
        :param normalize_items: bool
        If True, every row of the item features will be l2 normalized.
        """

        # Arg-check
        if (n_components is None) or (n_tastes is None) or (user_repr_graph is None) or (item_repr_graph is None) \
                or (prediction_graph is None) or (loss_graph is None):
            raise ValueError("All arguments to TensorRec() must be non-None")
        if n_components < 1:
            raise ValueError("n_components must be >= 1")
        if n_tastes < 1:
            raise ValueError("n_tastes must be >= 1")
        if not isinstance(user_repr_graph, AbstractRepresentationGraph):
            raise ValueError("user_repr_graph must inherit AbstractRepresentationGraph")
        if not isinstance(item_repr_graph, AbstractRepresentationGraph):
            raise ValueError("item_repr_graph must inherit AbstractRepresentationGraph")
        if not isinstance(prediction_graph, AbstractPredictionGraph):
            raise ValueError("prediction_graph must inherit AbstractPredictionGraph")
        if not isinstance(loss_graph, AbstractLossGraph):
            raise ValueError("loss_graph must inherit AbstractLossGraph")
        if attention_graph is not None:
            if not isinstance(attention_graph, AbstractRepresentationGraph):
                raise ValueError("attention_graph must be None or inherit AbstractRepresentationGraph")
            if n_tastes == 1:
                raise ValueError("attention_graph must be None if n_tastes == 1")

        self.n_components = n_components
        self.n_tastes = n_tastes
        self.user_repr_graph_factory = user_repr_graph
        self.item_repr_graph_factory = item_repr_graph
        self.attention_graph_factory = attention_graph
        self.prediction_graph_factory = prediction_graph
        self.loss_graph_factory = loss_graph
        self.biased = biased
        self.normalize_users = normalize_users
        self.normalize_items = normalize_items

        # A list of the attr names of every graph hook attr
        self.graph_tensor_hook_attr_names = [

            # Top-level API nodes
            'tf_user_representation', 'tf_item_representation', 'tf_prediction_serial', 'tf_prediction', 'tf_rankings',
            'tf_predict_similar_items', 'tf_rank_similar_items',

            # Training nodes
            'tf_basic_loss', 'tf_weight_reg_loss', 'tf_loss',

            # Feed placeholders
            'tf_n_users', 'tf_n_items', 'tf_user_feature_indices', 'tf_user_feature_values', 'tf_item_feature_indices',
            'tf_item_feature_values', 'tf_interaction_indices', 'tf_interaction_values', 'tf_learning_rate', 'tf_alpha',
            'tf_sample_indices', 'tf_n_sampled_items', 'tf_similar_items_ids'
        ]
        if self.biased:
            self.graph_tensor_hook_attr_names += ['tf_projected_user_biases', 'tf_projected_item_biases']
        if self.attention_graph_factory is not None:
            self.graph_tensor_hook_attr_names += ['tf_user_attention_representation']

        self.graph_operation_hook_attr_names = [

            # AdamOptimizer
            'tf_optimizer',

        ]
        self._clear_graph_hook_attrs()

        # A map of every graph hook attr name to the node name after construction
        # Tensors and operations are stored separated because they are handled differently by TensorFlow
        self.graph_tensor_hook_node_names = {}
        self.graph_operation_hook_node_names = {}

    def _clear_graph_hook_attrs(self):
        for graph_tensor_hook_attr_name in self.graph_tensor_hook_attr_names:
            self.__setattr__(graph_tensor_hook_attr_name, None)
        for graph_operation_hook_attr_name in self.graph_operation_hook_attr_names:
            self.__setattr__(graph_operation_hook_attr_name, None)

    def _attach_graph_hook_attrs(self):
        session = get_session()

        for graph_tensor_hook_attr_name in self.graph_tensor_hook_attr_names:
            graph_tensor_hook_node_name = self.graph_tensor_hook_node_names[graph_tensor_hook_attr_name]
            node = session.graph.get_tensor_by_name(name=graph_tensor_hook_node_name)
            self.__setattr__(graph_tensor_hook_attr_name, node)

        for graph_operation_hook_attr_name in self.graph_operation_hook_attr_names:
            graph_operation_hook_node_name = self.graph_operation_hook_node_names[graph_operation_hook_attr_name]
            node = session.graph.get_operation_by_name(name=graph_operation_hook_node_name)
            self.__setattr__(graph_operation_hook_attr_name, node)

    def _create_feed_dict(self, interactions_matrix, user_features_matrix, item_features_matrix,
                          extra_feed_kwargs=None):

        # Check that input data is of a sparse type
        if (interactions_matrix is not None) and (not sp.issparse(interactions_matrix)):
            raise Exception('Interactions must be a scipy sparse matrix')
        if not sp.issparse(user_features_matrix):
            raise Exception('User features must be a scipy sparse matrix')
        if not sp.issparse(item_features_matrix):
            raise Exception('Item features must be a scipy sparse matrix')

        n_users, user_feature_indices, user_feature_values = self._process_matrix(user_features_matrix,
                                                                                normalize_rows=self.normalize_users)
        n_items, item_feature_indices, item_feature_values = self._process_matrix(item_features_matrix,
                                                                                normalize_rows=self.normalize_items)

        feed_dict = {self.tf_n_users: n_users,
                     self.tf_n_items: n_items,
                     self.tf_user_feature_indices: user_feature_indices,
                     self.tf_user_feature_values: user_feature_values,
                     self.tf_item_feature_indices: item_feature_indices,
                     self.tf_item_feature_values: item_feature_values, }

        if interactions_matrix is not None:
            _, interaction_indices, interaction_values = self._process_matrix(interactions_matrix)
            feed_dict[self.tf_interaction_indices] = interaction_indices
            feed_dict[self.tf_interaction_values] = interaction_values

        if extra_feed_kwargs:
            feed_dict.update(extra_feed_kwargs)

        return feed_dict

    def _create_user_feed_dict(self, user_features_matrix, extra_feed_kwargs=None):

        if not sp.issparse(user_features_matrix):
            raise Exception('User features must be a scipy sparse matrix')

        n_users, user_feature_indices, user_feature_values = self._process_matrix(user_features_matrix,
                                                                                normalize_rows=self.normalize_users)

        feed_dict = {self.tf_n_users: n_users,
                     self.tf_user_feature_indices: user_feature_indices,
                     self.tf_user_feature_values: user_feature_values}

        if extra_feed_kwargs:
            feed_dict.update(extra_feed_kwargs)

        return feed_dict

    def _create_item_feed_dict(self, item_features_matrix, extra_feed_kwargs=None):

        if not sp.issparse(item_features_matrix):
            raise Exception('Item features must be a scipy sparse matrix')

        n_items, item_feature_indices, item_feature_values = self._process_matrix(item_features_matrix,
                                                                                normalize_rows=self.normalize_items)

        feed_dict = {self.tf_n_items: n_items,
                     self.tf_item_feature_indices: item_feature_indices,
                     self.tf_item_feature_values: item_feature_values}

        if extra_feed_kwargs:
            feed_dict.update(extra_feed_kwargs)

        return feed_dict

    def _process_matrix(self, features_matrix, normalize_rows=False):

        if normalize_rows:
            if not isinstance(features_matrix, sp.csr_matrix):
                features_matrix = sp.csr_matrix(features_matrix)
            mag = np.sqrt(features_matrix.power(2).sum(axis=1))
            features_matrix = features_matrix.multiply(1.0 / mag)

        if not isinstance(features_matrix, sp.coo_matrix):
            features_matrix = sp.coo_matrix(features_matrix)

        # "Actors" is used to signify "users or items" -- unused for interactions
        n_actors = features_matrix.shape[0]
        feature_indices = [pair for pair in six.moves.zip(features_matrix.row, features_matrix.col)]
        feature_values = features_matrix.data

        return n_actors, feature_indices, feature_values

    def _create_batch_feed_dicts(self, interactions, user_features, item_features, extra_feed_kwargs,
                                 user_batch_size=None):

        if not isinstance(interactions, sp.csr_matrix):
            interactions = sp.csr_matrix(interactions)
        if not isinstance(user_features, sp.csr_matrix):
            user_features = sp.csr_matrix(user_features)

        n_users = user_features.shape[0]

        # Infer the batch size, if necessary
        if user_batch_size is None:
            user_batch_size = n_users

        feed_dicts = []

        start_batch = 0
        while start_batch < n_users:

            # min() ensures that the batch bounds doesn't go past the end of the index
            end_batch = min(start_batch + user_batch_size, n_users)

            batch_interactions = interactions[start_batch:end_batch]
            batch_user_features = user_features[start_batch:end_batch]

            # TODO its inefficient to make so many copies of the item features
            feed_dict = self._create_feed_dict(interactions_matrix=batch_interactions,
                                               user_features_matrix=batch_user_features,
                                               item_features_matrix=item_features,
                                               extra_feed_kwargs=extra_feed_kwargs)
            feed_dicts.append(feed_dict)

            start_batch = end_batch

        return feed_dicts

    def _build_tf_graph(self, n_user_features, n_item_features):

        # Initialize placeholder values for inputs
        self.tf_n_users = tf.placeholder('int64')
        self.tf_n_items = tf.placeholder('int64')
        self.tf_n_sampled_items = tf.placeholder('int64')

        # SparseTensor placeholders
        self.tf_user_feature_indices = tf.placeholder('int64', [None, 2])
        self.tf_user_feature_values = tf.placeholder('float', [None])
        self.tf_item_feature_indices = tf.placeholder('int64', [None, 2])
        self.tf_item_feature_values = tf.placeholder('float', [None])
        self.tf_interaction_indices = tf.placeholder('int64', [None, 2])
        self.tf_interaction_values = tf.placeholder('float', [None])
        self.tf_similar_items_ids = tf.placeholder('int64', [None])

        self.tf_sample_indices = tf.placeholder('int64', [None, None])
        self.tf_learning_rate = tf.placeholder('float', None)
        self.tf_alpha = tf.placeholder('float', None)

        # Construct the features and interactions as sparse matrices
        tf_user_features = tf.SparseTensor(self.tf_user_feature_indices, self.tf_user_feature_values,
                                           [self.tf_n_users, n_user_features])
        tf_item_features = tf.SparseTensor(self.tf_item_feature_indices, self.tf_item_feature_values,
                                           [self.tf_n_items, n_item_features])
        tf_interactions = tf.SparseTensor(self.tf_interaction_indices, self.tf_interaction_values,
                                          [self.tf_n_users, self.tf_n_items])

        # Collect the weights for normalization
        tf_weights = []

        # Build the item representations
        self.tf_item_representation, item_weights = \
            self.item_repr_graph_factory.connect_representation_graph(tf_features=tf_item_features,
                                                                      n_components=self.n_components,
                                                                      n_features=n_item_features,
                                                                      node_name_ending='item')
        tf_weights.extend(item_weights)

        tf_x_user, tf_x_item = split_sparse_tensor_indices(tf_sparse_tensor=tf_interactions, n_dimensions=2)
        tf_transposed_sample_indices = tf.transpose(self.tf_sample_indices)
        tf_x_user_sample = tf_transposed_sample_indices[0]
        tf_x_item_sample = tf_transposed_sample_indices[1]

        # These lists will hold the reprs and predictions for each taste
        tastes_tf_user_representations = []
        tastes_tf_predictions = []
        tastes_tf_prediction_serials = []
        tastes_tf_sample_prediction_serials = []

        # If this model does not use attention, Nones are used as sentinels in place of the attentions
        if self.attention_graph_factory is not None:
            tastes_tf_attentions = []
            tastes_tf_attention_serials = []
            tastes_tf_sample_attention_serials = []
            tastes_tf_attention_representations = []
        else:
            tastes_tf_attentions = None
            tastes_tf_attention_serials = None
            tastes_tf_sample_attention_serials = None
            tastes_tf_attention_representations = None

        # Build n_tastes user representations and predictions
        for taste in range(self.n_tastes):
            tf_user_representation, user_weights = \
                self.user_repr_graph_factory.connect_representation_graph(tf_features=tf_user_features,
                                                                          n_components=self.n_components,
                                                                          n_features=n_user_features,
                                                                          node_name_ending='user_{}'.format(taste))
            tastes_tf_user_representations.append(tf_user_representation)
            tf_weights.extend(user_weights)

            # Connect attention, if applicable
            if self.attention_graph_factory is not None:
                tf_attention_representation, attention_weights = \
                    self.attention_graph_factory.connect_representation_graph(tf_features=tf_user_features,
                                                                              n_components=self.n_components,
                                                                              n_features=n_user_features,
                                                                              node_name_ending='attn_{}'.format(taste))
                tf_weights.extend(attention_weights)

                tf_attention = self.prediction_graph_factory.connect_dense_prediction_graph(
                    tf_user_representation=tf_attention_representation,
                    tf_item_representation=self.tf_item_representation
                )
                tf_attention_serial = self.prediction_graph_factory.connect_serial_prediction_graph(
                    tf_user_representation=tf_attention_representation,
                    tf_item_representation=self.tf_item_representation,
                    tf_x_user=tf_x_user,
                    tf_x_item=tf_x_item,
                )
                tf_sample_attention_serial = self.prediction_graph_factory.connect_serial_prediction_graph(
                    tf_user_representation=tf_user_representation,
                    tf_item_representation=self.tf_item_representation,
                    tf_x_user=tf_x_user_sample,
                    tf_x_item=tf_x_item_sample,
                )

                tastes_tf_attentions.append(tf_attention)
                tastes_tf_attention_serials.append(tf_attention_serial)
                tastes_tf_sample_attention_serials.append(tf_sample_attention_serial)
                tastes_tf_attention_representations.append(tf_attention_representation)

            # Connect the configurable prediction graphs for each taste
            tf_prediction = self.prediction_graph_factory.connect_dense_prediction_graph(
                tf_user_representation=tf_user_representation,
                tf_item_representation=self.tf_item_representation
            )
            tf_prediction_serial = self.prediction_graph_factory.connect_serial_prediction_graph(
                tf_user_representation=tf_user_representation,
                tf_item_representation=self.tf_item_representation,
                tf_x_user=tf_x_user,
                tf_x_item=tf_x_item,
            )
            tf_sample_predictions_serial = self.prediction_graph_factory.connect_serial_prediction_graph(
                tf_user_representation=tf_user_representation,
                tf_item_representation=self.tf_item_representation,
                tf_x_user=tf_x_user_sample,
                tf_x_item=tf_x_item_sample,
            )

            # Append to tastes
            tastes_tf_predictions.append(tf_prediction)
            tastes_tf_prediction_serials.append(tf_prediction_serial)
            tastes_tf_sample_prediction_serials.append(tf_sample_predictions_serial)

        # If attention is in the graph, build the API node
        if self.attention_graph_factory is not None:
            self.tf_user_attention_representation = tf.stack(tastes_tf_attention_representations)

        self.tf_user_representation = tf.stack(tastes_tf_user_representations)
        self.tf_prediction = collapse_mixture_of_tastes(
            tastes_predictions=tastes_tf_predictions,
            tastes_attentions=tastes_tf_attentions
        )
        self.tf_prediction_serial = collapse_mixture_of_tastes(
            tastes_predictions=tastes_tf_prediction_serials,
            tastes_attentions=tastes_tf_attention_serials
        )
        tf_sample_predictions_serial = collapse_mixture_of_tastes(
            tastes_predictions=tastes_tf_sample_prediction_serials,
            tastes_attentions=tastes_tf_sample_attention_serials
        )

        # Add biases, if this is a biased estimator
        if self.biased:
            tf_user_feature_biases, self.tf_projected_user_biases = project_biases(
                tf_features=tf_user_features, n_features=n_user_features
            )
            tf_item_feature_biases, self.tf_projected_item_biases = project_biases(
                tf_features=tf_item_features, n_features=n_item_features
            )

            tf_weights.append(tf_user_feature_biases)
            tf_weights.append(tf_item_feature_biases)

            self.tf_prediction = bias_prediction_dense(
                tf_prediction=self.tf_prediction,
                tf_projected_user_biases=self.tf_projected_user_biases,
                tf_projected_item_biases=self.tf_projected_item_biases)

            self.tf_prediction_serial = bias_prediction_serial(
                tf_prediction_serial=self.tf_prediction_serial,
                tf_projected_user_biases=self.tf_projected_user_biases,
                tf_projected_item_biases=self.tf_projected_item_biases,
                tf_x_user=tf_x_user,
                tf_x_item=tf_x_item)

            tf_sample_predictions_serial = bias_prediction_serial(
                tf_prediction_serial=tf_sample_predictions_serial,
                tf_projected_user_biases=self.tf_projected_user_biases,
                tf_projected_item_biases=self.tf_projected_item_biases,
                tf_x_user=tf_x_user_sample,
                tf_x_item=tf_x_item_sample)

        tf_interactions_serial = tf_interactions.values

        # Construct API nodes
        self.tf_rankings = rank_predictions(tf_prediction=self.tf_prediction)
        self.tf_predict_similar_items = predict_similar_items(prediction_graph_factory=self.prediction_graph_factory,
                                                              tf_item_representation=self.tf_item_representation,
                                                              tf_similar_items_ids=self.tf_similar_items_ids)
        self.tf_rank_similar_items = rank_predictions(tf_prediction=self.tf_predict_similar_items)

        # Compose loss function args
        # This composition is for execution safety: it prevents loss functions that are incorrectly configured from
        # having visibility of certain nodes.
        loss_graph_kwargs = {
            'tf_prediction_serial': self.tf_prediction_serial,
            'tf_interactions_serial': tf_interactions_serial,
            'tf_interactions': tf_interactions,
            'tf_n_users': self.tf_n_users,
            'tf_n_items': self.tf_n_items,
        }
        if self.loss_graph_factory.is_dense:
            loss_graph_kwargs.update({
                'tf_prediction': self.tf_prediction,
                'tf_rankings': self.tf_rankings,
            })
        if self.loss_graph_factory.is_sample_based:
            tf_sample_predictions = densify_sampled_item_predictions(
                tf_sample_predictions_serial=tf_sample_predictions_serial,
                tf_n_sampled_items=self.tf_n_sampled_items,
                tf_n_users=self.tf_n_users,
            )
            loss_graph_kwargs.update({'tf_sample_predictions': tf_sample_predictions,
                                      'tf_n_sampled_items': self.tf_n_sampled_items})

        # Build loss graph
        self.tf_basic_loss = self.loss_graph_factory.connect_loss_graph(**loss_graph_kwargs)

        self.tf_weight_reg_loss = sum(tf.nn.l2_loss(weights) for weights in tf_weights)
        self.tf_loss = self.tf_basic_loss + (self.tf_alpha * self.tf_weight_reg_loss)
        self.tf_optimizer = tf.train.AdamOptimizer(learning_rate=self.tf_learning_rate).minimize(self.tf_loss)

        # Get node names for each graph hook
        for graph_tensor_hook_attr_name in self.graph_tensor_hook_attr_names:
            hook = self.__getattribute__(graph_tensor_hook_attr_name)
            self.graph_tensor_hook_node_names[graph_tensor_hook_attr_name] = hook.name
        for graph_operation_hook_attr_name in self.graph_operation_hook_attr_names:
            hook = self.__getattribute__(graph_operation_hook_attr_name)
            self.graph_operation_hook_node_names[graph_operation_hook_attr_name] = hook.name

    def fit(self, interactions, user_features, item_features, epochs=100, learning_rate=0.1, alpha=0.00001,
            verbose=False, out_sample_interactions=None, user_batch_size=None, n_sampled_items=None):
        """
        Constructs the TensorRec graph and fits the model.
        :param interactions: scipy.sparse matrix
        A matrix of interactions of shape [n_users, n_items].
        :param user_features: scipy.sparse matrix
        A matrix of user features of shape [n_users, n_user_features].
        :param item_features: scipy.sparse matrix
        A matrix of item features of shape [n_items, n_item_features].
        :param epochs: Integer
        The number of epochs to fit the model.
        :param learning_rate: Float
        The learning rate of the model.
        :param alpha:
        The weight regularization loss coefficient.
        :param verbose: boolean
        If true, the model will print a number of status statements during fitting.
        :param out_sample_interactions: scipy.sparse matrix
        A matrix of interactions of shape [n_users, n_items].
        If not None, and verbose == True, the model will be evaluated on these interactions on every epoch.
        :param user_batch_size: int or None
        The maximum number of users per batch, or None for all users.
        :param n_sampled_items: int or None
        The number of items to sample per user for use in loss functions. Must be non-None if
        self.loss_graph_factory.is_sample_based is True.
        """

        # Pass-through to fit_partial
        self.fit_partial(interactions=interactions,
                         user_features=user_features,
                         item_features=item_features,
                         epochs=epochs,
                         learning_rate=learning_rate,
                         alpha=alpha,
                         verbose=verbose,
                         out_sample_interactions=out_sample_interactions,
                         user_batch_size=user_batch_size,
                         n_sampled_items=n_sampled_items)

    def fit_partial(self, interactions, user_features, item_features, epochs=1, learning_rate=0.1,
                    alpha=0.00001, verbose=False, out_sample_interactions=None, user_batch_size=None,
                    n_sampled_items=None):
        """
        Constructs the TensorRec graph and fits the model.
        :param interactions: scipy.sparse matrix
        A matrix of interactions of shape [n_users, n_items].
        :param user_features: scipy.sparse matrix
        A matrix of user features of shape [n_users, n_user_features].
        :param item_features: scipy.sparse matrix
        A matrix of item features of shape [n_items, n_item_features].
        :param epochs: Integer
        The number of epochs to fit the model.
        :param learning_rate: Float
        The learning rate of the model.
        :param alpha:
        The weight regularization loss coefficient.
        :param verbose: boolean
        If true, the model will print a number of status statements during fitting.
        :param out_sample_interactions: scipy.sparse matrix
        A matrix of interactions of shape [n_users, n_items].
        If not None, and verbose == True, the model will be evaluated on these interactions on every epoch.
        :param user_batch_size: int or None
        The maximum number of users per batch, or None for all users.
        :param n_sampled_items: int or None
        The number of items to sample per user for use in loss functions. Must be non-None if
        self.loss_graph_factory.is_sample_based is True.
        """

        session = get_session()

        # Arg checking
        if self.loss_graph_factory.is_sample_based:
            if (n_sampled_items is None) or (n_sampled_items <= 0):
                raise ValueError("n_sampled_items must be an integer >0")
        if (n_sampled_items is not None) and (not self.loss_graph_factory.is_sample_based):
            logging.warning('n_sampled_items was specified, but the loss graph is not sample-based')

        # Check if the graph has been constructed by checking the dense prediction node
        # If it hasn't been constructed, initialize it
        if self.tf_prediction is None:
            # Numbers of features are learned at fit time from the shape of these two matrices and cannot be changed
            # without refitting
            self._build_tf_graph(n_user_features=user_features.shape[1], n_item_features=item_features.shape[1])
            session.run(tf.global_variables_initializer())

        if verbose:
            logging.info('Processing interaction and feature data')

        batched_feed_dicts = self._create_batch_feed_dicts(interactions=interactions,
                                                           user_features=user_features,
                                                           item_features=item_features,
                                                           user_batch_size=user_batch_size,
                                                           extra_feed_kwargs={self.tf_learning_rate: learning_rate})

        # This scales down the alpha based on the number of batches and inserts the new alpha in all feed dicts
        batched_alpha = calculate_batched_alpha(num_batches=len(batched_feed_dicts), alpha=alpha)
        for feed_dict in batched_feed_dicts:
            feed_dict[self.tf_alpha] = batched_alpha

        if verbose:
            logging.info('Beginning fitting')

        for epoch in range(epochs):
            for batch, feed_dict in enumerate(batched_feed_dicts):

                # Handle random item sampling, if applicable
                if self.loss_graph_factory.is_sample_based:
                    sample_indices = sample_items(n_users=feed_dict[self.tf_n_users],
                                                  n_items=feed_dict[self.tf_n_items],
                                                  n_sampled_items=n_sampled_items,
                                                  replace=self.loss_graph_factory.is_sampled_with_replacement)
                    feed_dict[self.tf_sample_indices] = sample_indices
                    feed_dict[self.tf_n_sampled_items] = n_sampled_items

                # TODO find something more elegant than these cascaded ifs
                if not verbose:
                    session.run(self.tf_optimizer, feed_dict=feed_dict)

                else:
                    _, loss, serial_predictions, wr_loss = session.run(
                        [self.tf_optimizer, self.tf_basic_loss, self.tf_prediction_serial, self.tf_weight_reg_loss],
                        feed_dict=feed_dict
                    )
                    mean_loss = np.mean(loss)
                    mean_pred = np.mean(serial_predictions)
                    weight_reg_l2_loss = alpha * wr_loss
                    logging.info('EPOCH {} BATCH {} loss = {}, weight_reg_l2_loss = {}, mean_pred = {}'.format(
                        epoch, batch, mean_loss, weight_reg_l2_loss, mean_pred
                    ))
                    if out_sample_interactions:
                        os_feed_dict = self._create_feed_dict(out_sample_interactions, user_features, item_features)
                        os_loss = self.tf_basic_loss.eval(session=session, feed_dict=os_feed_dict)
                        logging.info('Out-Sample loss = {}'.format(os_loss))

    def predict(self, user_features, item_features):
        """
        Predict recommendation scores for the given users and items.
        :param user_features: scipy.sparse matrix
        A matrix of user features of shape [n_users, n_user_features].
        :param item_features: scipy.sparse matrix
        A matrix of item features of shape [n_items, n_item_features].
        :return: np.ndarray
        The predictions in an ndarray of shape [n_users, n_items]
        """
        feed_dict = self._create_feed_dict(interactions_matrix=None,
                                           user_features_matrix=user_features,
                                           item_features_matrix=item_features)

        predictions = self.tf_prediction.eval(session=get_session(), feed_dict=feed_dict)

        return predictions

    def predict_similar_items(self, item_features, item_ids, n_similar):
        """
        Predicts the most similar items to the given item_ids.
        :param item_features: scipy.sparse matrix
        A matrix of user features of shape [n_users, n_user_features].
        :param item_ids: list or np.array
        The ids of the items of interest.
        E.g. [4, 8, 12] to get sims for items 4, 8, and 12.
        :param n_similar: int
        The number of similar items to get per item of interest.
        :return: list of lists of tuples
        The first level list corresponds to input arg item_ids.
        The second level list is of length n_similar and contains tuples of (item_id, score) for each similar item.
        """
        feed_dict = self._create_item_feed_dict(item_features_matrix=item_features)
        feed_dict[self.tf_similar_items_ids] = np.array(item_ids)

        sims = self.tf_predict_similar_items.eval(session=get_session(), feed_dict=feed_dict)

        results = []
        for i in range(len(item_ids)):
            item_sims = sims[i]
            best = np.argpartition(item_sims, -n_similar)[-n_similar:]
            item_results = sorted(zip(best, item_sims[best]), key=lambda x: -x[1])
            results.append(item_results)

        return results

    def predict_rank(self, user_features, item_features):
        """
        Predict recommendation ranks for the given users and items.
        :param user_features: scipy.sparse matrix
        A matrix of user features of shape [n_users, n_user_features].
        :param item_features: scipy.sparse matrix
        A matrix of item features of shape [n_items, n_item_features].
        :return: np.ndarray
        The ranks in an ndarray of shape [n_users, n_items]
        """
        feed_dict = self._create_feed_dict(interactions_matrix=None,
                                           user_features_matrix=user_features,
                                           item_features_matrix=item_features)

        rankings = self.tf_rankings.eval(session=get_session(), feed_dict=feed_dict)

        return rankings

    def predict_user_representation(self, user_features):
        """
        Predict latent representation vectors for the given users.
        :param user_features: scipy.sparse matrix
        A matrix of user features of shape [n_users, n_user_features].
        :return: np.ndarray
        The latent user representations in an ndarray of shape [n_users, n_components]
        """
        feed_dict = self._create_user_feed_dict(user_features_matrix=user_features)
        user_repr = self.tf_user_representation.eval(session=get_session(), feed_dict=feed_dict)

        # If there is only one user repr per user, collapse from rank 3 to rank 2
        if self.n_tastes == 1:
            user_repr = np.sum(user_repr, axis=0)

        return user_repr

    def predict_user_attention_representation(self, user_features):
        """
        Predict latent attention representation vectors for the given users.
        :param user_features: scipy.sparse matrix
        A matrix of user features of shape [n_users, n_user_features].
        :return: np.ndarray
        The latent user attention representations in an ndarray of shape [n_users, n_components]
        """

        if self.attention_graph_factory is None:
            raise ValueError("This TensorRec model does not use attention. Try re-building TensorRec with a valid "
                             "'attention_graph' arg.")

        feed_dict = self._create_user_feed_dict(user_features_matrix=user_features)
        user_attn_repr = self.tf_user_attention_representation.eval(session=get_session(), feed_dict=feed_dict)

        # If there is only one user attn repr per user, collapse from rank 3 to rank 2
        if self.n_tastes == 1:
            user_attn_repr = np.sum(user_attn_repr, axis=0)

        return user_attn_repr

    def predict_item_representation(self, item_features):
        """
        Predict representation vectors for the given items.
        :param item_features: scipy.sparse matrix
        A matrix of item features of shape [n_items, n_item_features].
        :return: np.ndarray
        The latent item representations in an ndarray of shape [n_items, n_components]
        """
        feed_dict = self._create_item_feed_dict(item_features_matrix=item_features)
        item_repr = self.tf_item_representation.eval(session=get_session(), feed_dict=feed_dict)
        return item_repr

    def predict_user_bias(self, user_features):
        """
        Predict bias values for the given users.
        :param user_features: scipy.sparse matrix
        A matrix of user features of shape [n_users, n_user_features].
        :return: np.ndarray
        The user biases in an ndarray of shape [n_users]
        """
        if not self.biased:
            raise NotImplementedError('Cannot predict user bias for unbiased model')
        feed_dict = self._create_user_feed_dict(user_features_matrix=user_features)
        predictions = self.tf_projected_user_biases.eval(session=get_session(), feed_dict=feed_dict)
        return predictions

    def predict_item_bias(self, item_features):
        """
        Predict bias values for the given items.
        :param item_features: scipy.sparse matrix
        A matrix of item features of shape [n_items, n_item_features].
        :return: np.ndarray
        The item biases in an ndarray of shape [n_items]
        """
        if not self.biased:
            raise NotImplementedError('Cannot predict item bias for unbiased model')
        feed_dict = self._create_item_feed_dict(item_features_matrix=item_features)
        predictions = self.tf_projected_item_biases.eval(session=get_session(), feed_dict=feed_dict)
        return predictions

    def save_model(self, directory_path):
        """
        Saves the model to files in the given directory.
        :param directory_path: str
        The path to the directory in which to save the model.
        :return:
        """

        if not os.path.exists(directory_path):
            os.makedirs(directory_path)

        saver = tf.train.Saver()
        session_path = os.path.join(directory_path, 'tensorrec_session.cpkt')
        saver.save(sess=get_session(), save_path=session_path)

        # Break connections to the graph before saving the python object
        self._clear_graph_hook_attrs()
        tensorrec_path = os.path.join(directory_path, 'tensorrec.pkl')
        with open(tensorrec_path, 'wb') as file:
            pickle.dump(file=file, obj=self)

        # Reconnect to the graph after saving
        self._attach_graph_hook_attrs()

    @classmethod
    def load_model(cls, directory_path):
        """
        Loads the TensorRec model and TensorFlow session saved in the given directory.
        :param directory_path: str
        The path to the directory containing the saved model.
        :return:
        """

        graph_path = os.path.join(directory_path, 'tensorrec_session.cpkt.meta')
        saver = tf.train.import_meta_graph(graph_path)

        session_path = os.path.join(directory_path, 'tensorrec_session.cpkt')
        saver.restore(sess=get_session(), save_path=session_path)

        tensorrec_path = os.path.join(directory_path, 'tensorrec.pkl')
        with open(tensorrec_path, 'rb') as file:
            model = pickle.load(file=file)
        model._attach_graph_hook_attrs()
        return model


import numpy as np
import matplotlib.pyplot as plt
from scipy.signal import find_peaks

inString = 'PsrAntiIn150nm.npy'
outString = 'PsrAntiOut150nm.npy'

frequencies = np.arange(0, 50, 0.05)
Psr1 = np.load(inString)
Psr2 = np.load(outString)
Psr = Psr1 + Psr2

omegas = np.load('Omegas.npy')

for omega in omegas:
    f = np.real(omega)
    idx = (np.abs(frequencies - f)).argmin()
    plt.plot(frequencies[idx], Psr[idx], "r.")

plt.plot(frequencies, Psr)
plt.xlabel("Frequency (GHz)")
plt.ylabel(r"$P_{\mathrm{sr}}$ (A.U.)")
plt.semilogy()
plt.savefig("CombinedHighQuality.png", dpi=1000)


import torch
from torch.autograd import Variable
import sys
sys.path.append('skip-thoughts.torch/pytorch')
from skipthoughts import *
import numpy as np
from collections import OrderedDict
import pickle
import json
import random

def build_dictionary(text):
    """
    Build a dictionary
    text: list of sentences (pre-tokenized)
    """
    wordcount = {}
    for cc in text:
        words = cc.split()
        for w in words:
            if w not in wordcount:
                wordcount[w] = 0
            wordcount[w] += 1

    sorted_words = sorted(list(wordcount.keys()), key=lambda x: wordcount[x], reverse=True)

    worddict = OrderedDict()
    for idx, word in enumerate(sorted_words):
        worddict[word] = idx+2 # 0: <eos>, 1: <unk>

    return worddict, wordcount

def convert_sentence_to_indices(sentence, worddict):

    indices = [
                  # assign an integer to each word, if the word is too rare assign unknown token
                  worddict.get(w) if worddict.get(w, VOCAB_SIZE + 1) < VOCAB_SIZE else UNK

                  for w in sentence.split()  # split into words on spaces
              ][: MAXLEN - 1]  # take only maxlen-1 words per sentence at the most.

    # last words are EOS
    indices += [EOS] * (MAXLEN - len(indices))

    indices = np.array(indices)
    indices = torch.from_numpy(indices)
    indices = indices.type(torch.LongTensor)
    indices = Variable(indices)
    if USE_CUDA:
        indices = indices.cuda()

    return indices

def encode(text, worddict, uniskip):
    def chunks(l, n):
        """Yield successive n-sized chunks from l."""
        for i in range(0, len(l), n):
            yield l[i:i + n]

    ret = []

    for i,chunk in enumerate(chunks(text, 100)):
        print("encoding chunk {} of size {}".format(i, len(chunk)))
        indices = [convert_sentence_to_indices(sentence, worddict) for sentence in chunk]
        indices = torch.stack(indices)
        indices = uniskip(indices)
        indices = indices.view(-1, 2400)
        indices = indices.data.cpu().numpy()

        ret.extend(indices)
    ret = np.array(ret)

    return ret

def load_all_captions(data_dir,filenames):
    def load_captions(caption_name):  # self,
        cap_path = caption_name
        with open(cap_path, "r") as f:
            captions = f.read().split('\n')
        captions = [cap.replace("\ufffd\ufffd", " ")
                    for cap in captions if len(cap) > 0]
        return captions

    caption_dict = {}
    for key in filenames:
        caption_name = '%s/text/%s.txt' % (data_dir, key)
        captions = load_captions(caption_name)
        caption_dict[key] = captions
    return caption_dict

def load_json_file(data_dir):
    train = json.load(open(data_dir, 'r'))
    out = {}

    for i, img in enumerate(train):
        imgid = img['image_id']
        sents = img['caption']
        out[imgid] = np.array(sents)

    return out

def create_embedding(data_dir='./data/',SEED = 500, split_percent = 15):
    # Load the dataset and extract features
    captions = load_json_file(data_dir)
    # shuffle captions and devide to train/test
    c = list(captions.items())
    random.seed(SEED)
    random.shuffle(c)
    split = int(len(c) * split_percent / 100)
    test = dict(c[0:split])
    train = dict(c[split::])

    for name,dataset in enumerate([train,test]):
        if name == 0:
            name = 'train'
        else:
            name = 'test'
        text = np.array(list(dataset.values()))
        np.save('text-{}'.format(name), text)
        filenames = list(dataset.keys())
        text_shape = text.size

        worddict, wordcount = build_dictionary(text)

        vocab = list(worddict.keys())
        dir_st = 'data/skip-thoughts/'
        uniskip = UniSkip(dir_st, vocab)
        biskip = BiSkip(dir_st,vocab)
        if USE_CUDA:
            uniskip.cuda()
            biskip.cuda()

        features_uni = encode(text, worddict, uniskip)
        features_uni = np.reshape(features_uni,(text_shape,1,2400))
        features_bi = encode(text, worddict, biskip)
        features_bi = np.reshape(features_bi, (text_shape, 1, 2400))
        features = np.concatenate((features_uni, features_bi), axis=2)
        np.save('skip-thought-embeddings-{}'.format(name), features)
        with open('filenames-{}.pickle'.format(name), 'wb') as f:
            pickle.dump(filenames, f)

USE_CUDA = True
VOCAB_SIZE = 20000
MAXLEN = 30
EOS = 0  # to mean end of sentence
UNK = 1  # to mean unknown token

create_embedding(data_dir='data/oxford/vis_oxford.json',SEED = 500, split_percent = 15)


from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.naive_bayes import MultinomialNB
from sklearn import metrics
from sklearn.pipeline import Pipeline

def split_data(labelled_tweets):
    X = labelled_tweets.lemmatized_content
    y = labelled_tweets.sentiment

    # X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state=42)

    return train_test_split(X, y, test_size = 0.1, random_state=42)

def train_model(labelled_tweets):
    X_train, X_test, y_train, y_test = split_data(labelled_tweets)
    count_vect = CountVectorizer()
    X_train_bow = count_vect.fit(X_train)
    X_train_bow = count_vect.transform(X_train)
    X_test_bow = count_vect.transform(X_test)

    tfidf_transformer = TfidfTransformer(use_idf=False).fit(X_train_bow)
    X_train_tfidf = tfidf_transformer.transform(X_train_bow)

    classifier = MultinomialNB().fit(X_train_tfidf, y_train)
    y_predict_class = classifier.predict(X_test_bow)

    print("Train data summary")

    print("Classifier Accuracy is {0}".format(metrics.accuracy_score(y_test, y_predict_class)))
    print(y_test.value_counts())

    return count_vect, classifier


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Jul 13 16:47:59 2020

@author: nathanyu
"""

import numpy as np
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split

class TreeNode():
    """树结点"""
    def __init__(self, feature_idx=None, feature_val=None, feature_name=None, node_val=None, child=None):
        """
        feature_idx:
            该结点对应的划分特征索引
        feature_val:
            划分特征对应的值
        feature_name:
            划分特征名
        node_val:
            该结点存储的值，**只有叶结点才存储类别**
        child:
            子树
        """
        self._feature_idx = feature_idx
        self._feature_val = feature_val
        self._feature_name = feature_name
        # 叶结点存储类别
        self._node_val = node_val
        # 非叶结点存储划分信息
        self._child = child

"""决策树算法Scratch实现"""
class DecisionTreeScratch():
    def __init__(self, feature_name, etype="gain", epsilon=0.01):
        """
        feature_name:
            每列特征名
        etype:
            可取值有
            gain: 使用信息增益
            ratio: 使用信息增益比
        epsilon:
            当信息增益或信息增益比小于该阈值时直接把对应结点作为叶结点
        """
        self._root = None
        self._fea_name = feature_name
        self._etype = etype
        self._epsilon = epsilon

    def fit(self, X, y):
        """
        模型训练
        X:
            训练集，每一行表示一个样本，每一列表示一个特征或属性
        y:
            训练集标签
        """
        self._root = self._build_tree(X, y)

    def predict(self, x):
        """给定输入样本，预测其类别"""
        return self._predict(x, self._root)

    def _build_tree(self, X, y):
        """
        构建树
        X:
            用于构建子树的数据集
        y:
            X对应的标签
        """
        # 最后只剩下一个特征时将该子树置为叶结点
        if X.shape[1] == 1:
            node_val = self._vote_label(y)
            return TreeNode(node_val=node_val)

        # 子树只剩下一个类别时直接置为叶结点
        if np.unique(y).shape[0] == 1:
            return TreeNode(node_val=y[0])

        n_feature = X.shape[1]
        # 最大信息增益或信息增益比
        max_gain = -np.inf
        # 信息增益最大或信息增益比最大的特征索引
        max_fea_idx = 0
        # 遍历每一个特征
        for i in range(n_feature):
            if self._etype == "gain":
                gain = self._calc_gain(X[:, i], y)
            else:
                gain = self._calc_gain_ration(X[:, i], y)
            if gain > max_gain:
                max_gain = gain
                max_fea_idx = i

        # 如果以该特征进行划分信息增益或增益比太小则不进行划分
        if max_gain < self._epsilon:
            node_val = self._vote_label(y)
            return TreeNode(node_val=node_val)

        feature_name = self._fea_name[max_fea_idx]
        child_tree = dict()
        # 遍历所选特征每一个可能的值，对每一个值构建子树
        feature_val = np.unique(X[:, max_fea_idx])
        for fea_val in feature_val:
            # 如果要对连续型数据进行划分，可将此处的==换成<=，类似CART
            child_X = X[X[:, max_fea_idx] == fea_val]
            child_y = y[X[:, max_fea_idx] == fea_val]
            child_X = np.delete(child_X, max_fea_idx, 1)
            # 构建子树
            child_tree[fea_val] = self._build_tree(child_X, child_y)
        return TreeNode(max_fea_idx, feature_name=feature_name, child=child_tree)

    def _predict(self, x, tree=None):
        """
        给定输入样本，将其划分到所属叶结点
        """
        # 根结点
        if tree is None:
            tree = self._root

        # 叶结点直接返回类别
        if tree._node_val is not None:
            return tree._node_val

        fea_idx = tree._feature_idx
        for fea_val, child_node in tree._child.items():
            if x[fea_idx] == fea_val:
                # 如果是叶结点直接返回类别
                if child_node._node_val is not None:
                    return child_node._node_val
                else:
                    # 否则继续去子树中找
                    return self._predict(x, child_node)

    def _vote_label(self, y):
        """统计y中次数出现最多的类别"""
        # 计算每个标签及其出现次数
        label, num_label = np.unique(y, return_counts=True)
        # 返回出现次数最多的类别作为叶结点类别
        return label[np.argmax(num_label)]

    def _calc_entropy(self, y):
        """
        计算熵
        y:
            数据集标签
        """
        entropy = 0
        # 计算每个类别的数量
        _, num_ck = np.unique(y, return_counts=True)
        for n in num_ck:
            p = n / y.shape[0]
            entropy -= p * np.log2(p)
        return entropy

    def _calc_condition_entropy(self, x, y):
        """
        计算条件熵
        x:
            数据集的某个特征对应的列向量，训练集中一行表示一个样本，列表示特征
        y:
            数据集标签
        """
        cond_entropy = 0
        # 计算特征x可能的取值及对应出现的次数
        xval, num_x = np.unique(x, return_counts=True)
        # 遍历该特征的每个值
        for v, n in zip(xval, num_x):
            # 该值所对应的划分
            y_sub = y[x == v]
            sub_entropy = self._calc_entropy(y_sub)
            p = n / y.shape[0]
            cond_entropy += p * sub_entropy
        return cond_entropy

    def _calc_gain(self, x, y):
        """
        计算信息增益
        x:
            数据集的某个特征对应的列向量，训练集中一行表示一个样本，列表示特征
        y:
            数据集标签
        """
        return self._calc_entropy(y) - self._calc_condition_entropy(x, y)

    def _calc_gain_ration(self, x, y):
        """
        计算信息增益比
        x:
            数据集的某个特征对应的列向量，训练集中一行表示一个样本，列表示特征
        y:
            数据集标签
        """
        return self._calc_gain(x, y) / self._calc_entropy(x)

def main():

    dataset = load_iris()
    xtrain, _, ytrain, _ = train_test_split(dataset.data, dataset.target, train_size=0.8, shuffle=True)
    feature_name = dataset.feature_names

    model = DecisionTreeScratch(feature_name, "gain", 0.1)
    model.fit(xtrain, ytrain)

    n_train = xtrain.shape[0]
    n_right = 0
    for i in range(n_train):
        y_pred = model.predict(xtrain[i])
        if y_pred == ytrain[i]:
            n_right += 1
        else:
            print("该样本真实标签为：{}，但是Scratch模型预测标签为：{}".format(ytrain[i], y_pred))
    print("DecisionTree模型在训练集上的准确率为：{}%".format(n_right * 100 / n_train))

if __name__ == "__main__":
    main()


import numpy as np
from data.get_dataset import get_dataset, PROBLEM_CLASSIFICATION, PROBLEM_DENOISING
from data.data_pipe import Pipe

class Data_manager:
    def __init__(self, data_path, problem=PROBLEM_CLASSIFICATION,
                 preproc_trn_X=None, preproc_trn_Y=None, preproc_val_X=None, preproc_val_Y=None,):

        self.problem = problem
        self.val_X, self.val_Y, self.trn_X, self.trn_Y = get_dataset(data_path, problem)

        self.set_preprocessings(preproc_trn_X, preproc_trn_Y, preproc_val_X, preproc_val_Y,)

    def set_preprocessings(self, preproc_trn_X=None, preproc_trn_Y=None, preproc_val_X=None, preproc_val_Y=None,):
        preprocesses = [preproc_trn_X, preproc_trn_Y, preproc_val_X, preproc_val_Y]
        preprocesses = [Pipe([]) if x is None else x for x in preprocesses]
        assert all([isinstance(x, Pipe) for x in preprocesses])
        self.preproc_trn_X, self.preproc_trn_Y, self.preproc_val_X, self.preproc_val_Y = preprocesses

    def get_next_trn_batch(self, batch_size=1):
        """
        yield batch_X, batch_Y, batch_number
        """
        self.p = 0
        self.b = 0
        L = len(self.trn_X)
        ids = np.random.permutation(L)
        last_p = self.p + batch_size
        while last_p <= L:
            batch_X_ = self.trn_X[ids[self.p:last_p]]
            batch_Y_ = self.trn_Y[ids[self.p:last_p]]
            batch_X = []
            batch_Y = []
            for x, y in zip(batch_X_, batch_Y_):
                x = self.preproc_trn_X(x)
                y = self.preproc_trn_Y(y)
                batch_X.append(x)
                batch_Y.append(y)
            batch_X = np.array(batch_X)
            batch_Y = np.array(batch_Y)
            self.b += 1
            self.p = last_p
            last_p = self.p + batch_size

            assert batch_X.shape[0] == batch_Y.shape[0]
            yield batch_X, batch_Y, self.b

    def get_next_trn_pair_classification(self):
        """
        yield batch_X, batch_Y, batch_number
        batch_size = 2
        """
        assert self.problem == PROBLEM_CLASSIFICATION
        self.b = 0
        L = len(self.trn_X)
        ids = list(range(0, L, 2))
        ids = np.random.permutation(ids)
        for id in ids:
            batch_X_ = self.trn_X[[id, id+1]]
            batch_Y_ = self.trn_Y[[id, id+1]]
            batch_X = []
            batch_Y = []
            for x, y in zip(batch_X_, batch_Y_):
                x = self.preproc_trn_X(x)
                y = self.preproc_trn_Y(y)
                batch_X.append(x)
                batch_Y.append(y)
            batch_X = np.array(batch_X)
            batch_Y = np.array(batch_Y)
            self.b += 1

            assert batch_X.shape[0] == batch_Y.shape[0]
            yield batch_X, batch_Y, self.b

    def get_next_val_pair_classification(self, part=1.0):
        """
        yield batch_X, batch_Y
        batch_size = 2
        """
        assert self.problem == PROBLEM_CLASSIFICATION
        L = len(self.val_X)
        ids = list(range(0, L, 2))
        if 0 < part < 1:
            part = int(round(len(ids)*part))
            ids = np.random.permutation(ids)[:part]
        elif part == 1:
            pass
        else:
            errmsg = "part is {}, but should be 0 < part <= 1".format(part)
            raise Exception(errmsg)

        for id in ids:
            batch_X_ = self.val_X[[id, id + 1]]
            batch_Y_ = self.val_Y[[id, id + 1]]
            batch_X = []
            batch_Y = []
            for x, y in zip(batch_X_, batch_Y_):
                x = self.preproc_val_X(x)
                y = self.preproc_val_Y(y)
                batch_X.append(x)
                batch_Y.append(y)
            batch_X = np.array(batch_X)
            batch_Y = np.array(batch_Y)

            assert batch_X.shape[0] == batch_Y.shape[0]
            yield batch_X, batch_Y

    def get_next_val_batch(self, batch_size=1, part=1.0):
        """
        yield batch_X, batch_Y
        """
        L = len(self.val_X)

        ids = np.random.permutation(L)
        L = int(round(L*part))
        ids = ids[:L]
        if batch_size == -1:
            batch_size = L

        def get_data():
            batch_X_ = self.val_X[ids[p:last_p]]
            batch_Y_ = self.val_Y[ids[p:last_p]]
            batch_X = []
            batch_Y = []
            for x, y in zip(batch_X_, batch_Y_):
                x = self.preproc_val_X(x)
                y = self.preproc_val_Y(y)
                batch_X.append(x)
                batch_Y.append(y)
            batch_X = np.array(batch_X)
            batch_Y = np.array(batch_Y)
            return batch_X, batch_Y

        p = 0
        last_p = p + batch_size
        while last_p < L:
            yield get_data()
            p = last_p
            last_p += batch_size

        for i in range(1):
            last_p = None
            yield get_data()

    def get_next_trn_pair_denoising(self):
        """
        yield batch_X, batch_Y, batch_number
        batch_size = 2
        """
        bs = 1
        self.p = 0
        self.b = 0
        L = len(self.trn_X)
        ids = np.random.permutation(L)
        last_p = self.p+bs
        while last_p <= L:
            batch_X_ = self.trn_X[ids[self.p:last_p]]
            batch_Y_ = self.trn_Y[ids[self.p:last_p]]
            batch_X = []
            batch_Y = []
            for x, y in zip(batch_X_, batch_Y_):
                x = self.preproc_trn_X(x)
                y = self.preproc_trn_Y(y)
                batch_X.append(x)
                batch_Y.append(y)
            batch_X = np.array(batch_X + batch_Y)
            batch_Y = np.array(batch_Y + batch_Y)
            self.b += 1
            self.p = last_p
            last_p = self.p+bs

            assert batch_X.shape[0] == batch_Y.shape[0]
            yield batch_X, batch_Y, self.b


# -*- coding: utf-8 -*-

import yaml
from Crypto.PublicKey import RSA
from boto import s3
from boto import iam
from boto import route53
from boto.s3.connection import OrdinaryCallingFormat
from boto.s3.key import Key
from boto.exception import S3ResponseError
import logging
from acme import client
from acme import messages
from acme import challenges
import io
import requests
import base64
import hashlib
from time import sleep
from OpenSSL import crypto
from datetime import datetime
from boto.ec2 import elb
from acme.jose.util import ComparableX509
import os

# this is the AWS region in which the function runs in
exec_region = os.environ['AWS_DEFAULT_REGION']

LOG = logging.getLogger("letslambda")
LOG.setLevel(logging.DEBUG)
handler = logging.StreamHandler()
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
# add formatter to ch
handler.setFormatter(formatter)
# add ch to logger
LOG.addHandler(handler)

def load_config(bucket):
    """
    Try to load the letlambda.yml out of the user bucket
    Will return None if the configuration file does not exist
    """
    conf = bucket.get_key("letslambda.yml")
    if conf == None:
        return None

    confString = conf.read()
    conf = yaml.load(confString)
    return conf

def loadAccountKey(bucket, conf):
    """
    Try to load the RSA account key from S3. If it doesn't
    succeed, it will create a new account key and try a registration
    with your provided information
    """
    LOG.info("Loading account key from s3")
    key = bucket.get_key("account.key.rsa")
    newAccountNeeded = False;
    if key == None:
        pem = createAndSaveKey(bucket, "account.key.rsa")
        newAccountNeeded = True
    else:
        pem = key.read()

    key = client.jose.JWKRSA.load(pem)
    if newAccountNeeded:
        registerNewAccount(conf, key)

    return key

def loadCSRKey(bucket, domain):
    LOG.info("Loading CSR key from S3")
    key = None
    if 'reuse_key' in domain.keys() and domain['reuse_key'] == True:
        LOG.info("Attempting to reuse old key for domain")
        name = domain['name'] + ".key.rsa"
        key = bucket.get_key(name)

    if key == None:
        pem = createAndSaveKey(bucket, name)
    else:
        pem = key.read()

    return crypto.load_privatekey(crypto.FILETYPE_PEM, pem)

def registerNewAccount(conf, key):
    """
    Attempt to create a new account on the ACME server
    with the key. No problem if it fails because this
    kye is already used.
    """
    LOG.info("Registering with ACME server with the new key")
    newReg = messages.NewRegistration(contact=tuple(conf['info']), key=key.public_key())
    acme_client = client.Client(conf['directory'], key)
    registration_resource = acme_client.register(newReg)
    LOG.info("Agreeing on the TOS on your behalf")
    acme_client.agree_to_tos(registration_resource)

def createAndSaveKey(bucket, name):
    """
    Generate an RSA 4096 key for general purpose (account or CSR)
    """
    LOG.info("Key not found! Generating new RSA key")
    key = RSA.generate(4096);
    saveToS3(bucket, key, name)
    return key.exportKey("PEM")

def saveToS3(bucket, rsakey, name):
    """
    Save the rsa key in PEM format to s3 .. for later use
    """
    LOG.info("Storing newly generated key")
    s3key = Key(bucket=bucket, name=name)
    fp = io.BytesIO(rsakey.exportKey("PEM"));
    s3key.set_contents_from_file(fp)

def get_authorization(client, domain):
    authorization_resource = client.request_domain_challenges(domain['name'])
    return authorization_resource;

def get_dns_challenge(authorization_resource):
    """
    Ask the ACME server to give us a list of challenges.
    Later, we will pick only the DNS one.
    """
    # Now let's look for a DNS challenge
    dns_challenges = filter(lambda x: isinstance(x.chall, challenges.DNS01), authorization_resource.body.challenges)
    return list(dns_challenges)[0]

def answer_dns_challenge(client, domain, challenge):
    """
    Compute the required answer and set it in the DNS record
    for the domain.
    """
    authorization = "{}.{}".format(
        base64.urlsafe_b64encode(challenge.get("token")).decode("ascii").replace("=", ""),
        base64.urlsafe_b64encode(client.key.thumbprint()).decode("ascii").replace("=", "")
        )

    dns_response = base64.urlsafe_b64encode(hashlib.sha256(authorization.encode()).digest()).decode("ascii").replace("=", "")

    # Let's update the DNS on our R53 account
    r53 = route53.connect_to_region(exec_region)
    zone = r53.get_zone(domain['r53_zone'])
    if zone == None:
        LOG.error("Cannot find R53 zone {}, are you controling it ?".format(domain['r53_zone']))
        exit(1)

    acme_domain = "_acme-challenge.{}".format(domain['name'])
    record = zone.find_records(name=acme_domain, type="TXT")
    if record:
        delete_status = zone.delete_record(record)

    add_status = zone.add_record("TXT", acme_domain, '"' + dns_response + '"')
    dns_updated = wait_until_sync(add_status)

    if dns_updated == False:
        LOG.error("We updated R53 but the servers didn't sync within 10 seconds. Bailing out.")
        exit(1)

    ## Now, let's tell the ACME server that we are ready
    challenge_response = challenges.DNS01Response(key_authorization=authorization)
    challenge_resource = client.answer_challenge(challenge, challenge_response)

    # We'd better poll to get the answer of this.

def generateCSR(bucket, conf):
    if 'reuse_key' in conf:
        key = loadCSRKey(bucket, conf)
    else:
        key = crypto.PKey()
        key.generate_key(crypto.TYPE_RSA, 4096)

    csr = crypto.X509Req()
    csr.get_subject().countryName = conf['countryName']
    csr.get_subject().CN = conf['name']
    csr.set_pubkey(key)
    csr.sign(key, "sha1")
    return (csr, key)

def requestCertificate(client, bucket, conf, auth_resource):
    (csr, key) = generateCSR(bucket, conf)
    # Gently wait for authorization to be valid
    # wait_until_valid(auth_resource)

    (certificate, ar) = client.poll_and_request_issuance(ComparableX509(csr), [auth_resource])
    return (certificate, key)

def wait_until_valid(auth):
    timeout = 30
    valid = False
    while valid and timeout > -1:
        status = requests.get(auth.uri)
        if status['status'] == "valid":
            valid = True

    return valid

def wait_until_sync(status):
    LOG.info("Waiting for DNS to synchronize with new TXT value")
    timeout = 30;
    while status.status != "INSYNC" and timeout > -1:
        sleep(1)
        timeout = timeout - 1;
        status.update()

    if timeout == -1:
        return False

    return True

def createIAMCertificate(domain, certificate, key):
    """
    Create a new IAM certificate from ACME and private key.
    It also fetched the chain certificate from ACME if provided
    """
    # First we fetch the chain certificate
    chain = requests.get(certificate.cert_chain_uri)
    chain_certificate = None
    if chain.status_code == 200:
        chain_certificate = crypto.load_certificate(crypto.FILETYPE_ASN1, chain.content)

    iam_connection = iam.connect_to_region(exec_region)
    res = iam_connection.upload_server_cert(
        domain['name'] + "-" + datetime.utcnow().strftime("%Y-%m-%dT%H-%M"),
        crypto.dump_certificate(crypto.FILETYPE_PEM, certificate.body.wrapped).decode("ascii"),
        crypto.dump_privatekey(crypto.FILETYPE_PEM, key).decode("ascii"),
        crypto.dump_certificate(crypto.FILETYPE_PEM, chain_certificate).decode("ascii"),
        "/")
    sleep(10)

    return res

def updateELB(conf, iam_certificate):
    LOG.info("Updating ELB with new certificate")
    elb_connection = elb.connect_to_region(conf['elb_region'])
    response = elb_connection.set_lb_listener_SSL_certificate(conf['elb'], 443, iam_certificate['upload_server_certificate_response']['upload_server_certificate_result']['server_certificate_metadata']['arn'])
    return response

def lambda_handler(event, context):
    bucket = event['bucket']
    region = event['region']

    LOG.info("Retrieving configuration file from bucket '{0}' in region '{1}' ".format(bucket, region))
    connection = s3.connect_to_region(region, calling_format=OrdinaryCallingFormat())
    try:
        bucket = connection.get_bucket(bucket);
    except S3ResponseError as e:
        print(e)
        LOG.error("Cannot fetch bucket : {}".format(bucket))
        exit(1)

    conf = load_config(bucket)
    if conf == None:
        LOG.error("Cannot find file 'letslambda.yml' in S3 bucket '{0}".format(bucket))
        exit(1)

    key = loadAccountKey(bucket, conf)
    acme_client = client.Client(conf['directory'], key)
    for domain in conf['domains']:
        authorization_resource = get_authorization(acme_client, domain)
        challenge = get_dns_challenge(authorization_resource)
        answer_dns_challenge(acme_client, domain, challenge)
        (certificate, key) = requestCertificate(acme_client, bucket, domain, authorization_resource)
        iam_cert = createIAMCertificate(domain, certificate, key)
        updateELB(domain, iam_cert)


import datetime

class MatchException(Exception):
    pass

class Match:
    MATCH_NOT_STARTED,\
    MATCH_IN_PROGRESS,\
    MATCH_OVER,\
    MATCH_CANCELLED\
    = range(4)

    def __init__(self, is_ranked=True):
        self.state = Match.MATCH_NOT_STARTED
        self.is_ranked = is_ranked

    def __str__(self):
        output = " MATCH SCORE ".center(41, "#") + "\n"
        output += "   {0}".format(self.player_1.name).ljust(17)
        output += "{0} - {1}".format(self.player_1.score, self.player_2.score).center(7)
        output += "{0}   ".format(self.player_2.name).rjust(17)
        output += "\n"
        output += "".center(41, "#")
        return output

    def start(self, card_id_1, card_id_2):
        self.state = Match.MATCH_IN_PROGRESS
        self.date = datetime.datetime.now()
        self.player_1 = MatchPlayer(card_id_1)
        self.player_2 = MatchPlayer(card_id_2)

    def update_score(self, player_number, increment = True):

        # Elect player and opponent given player_number
        player = self.player_1 if player_number == 1 else self.player_2
        opponent = self.player_2 if player_number == 1 else self.player_1

        # Evaluate new score
        new_score = player.score + (1 if increment else -1)
        if new_score < 0:
            raise MatchException("Player %s cannot have the score: %s" % (player_number, new_score))

        # Update score
        player.score = new_score

        # Evaluate match
        if player.score >= 11 and player.score > opponent.score + 1:
            # Game over
            self.state = Match.MATCH_OVER

    def cancel_match(self):
        self.state = Match.MATCH_CANCELLED

class MatchPlayer:
    def __init__(self, card_id):
        self.card_id = card_id
        self.score = 0


#!/usr/bin/python3

# -*-coding:utf-8 -*-

#Reference:**********************************************

# @Time    : 2019/10/2 12:46 下午

# @Author  : baozhiqiang

# @File    : k-core.py

# @User    : bao

# @Software: PyCharm

#Reference:**********************************************

import  commons
import  networkx as nx
class findk_core:
    def __init__(self):
        pass

    def main(self, dir):
            '''
            走来不要那么难，先搞定树吧。才能继续搞定图。
            :return:
            '''
            pre = '../../data/'
            last = '.txt'
            self.initG = commons.get_networkByFile(fileName=pre + dir + last)  # 获取图，
            max_sub_graph = commons.judge_data(self.initG)
            source_list = commons.product_sourceList(max_sub_graph, self.fix_number_source)
            self.true_Source_list = source_list
            self.infectG = commons.propagation1(max_sub_graph, self.true_Source_list)  # 开始传染

            subinfectG = commons.get_subGraph(self.infectG)
            subinfectG.to_undirected()
            subinfectG.remove_edges_from(subinfectG.selfloop_edges())
            data = nx.core_number(subinfectG)

            # Graph.to_undirected()
            print('data', data)
            data_sort = sorted(data.items(), key= lambda x :x[1], reverse= True)
            print('data_sort', data_sort)
            print('max-core',data_sort[0][1])
            core_number1 = 0
            #看下源点在那一层。
            for  node, core_number in data_sort:
                if node in source_list:
                    print('node,core_number', [node,core_number])
                    core_number1 = core_number
            return core_number1

    def cal_distanceError(self, dir):
            self.fix_number_source = 1
            distance = 0
            for i in range(20):
                self.main(dir)
                distance += self.main(dir)
            result = distance / 20
            # 导入time模块
            import time
            # 打印时间戳
            # print(time.time())
            pre = './result/'
            last = '.txt'
            with open(pre + dir + 'first' + last, 'a') as f:
                f.write(str(time.asctime(time.localtime(time.time()))) + '\n')
                f.write(str(20) + '     ' + str(dir) + '    ' + str(result))
            print(distance / 20)

test = findk_core()
filename = 'CA-GrQc'

test.cal_distanceError(filename)


from behave import given, when, then
from requests import get
from assertpy import assert_that

@given('an api url with valid response')
def step_imp(context):
    context.get_url = 'https://api.tmsandbox.co.nz/v1/Categories/6327/Details.json?catalogue=false'

@when('user sends request to get response')
def step_imp(context):
    context.response = get(context.get_url)

@then('it should display Name with value Carbon credits')
def step_imp(context):
    assert_that(context.response.status_code).is_equal_to(200)
    response = context.response.json()
    assert_that(response['Name']).described_as('Name Value').is_equal_to('Carbon credits')

@then('it should display CanRelist with boolean value True')
def step_imp(context):
    assert_that(context.response.status_code).is_equal_to(200)
    response = context.response.json()
    assert_that(response['CanRelist']).described_as('CanRelist value').is_true()

@then('Galery Promotion should display description with value 2x larger image')
def step_imp(context):
    assert_that(context.response.status_code).is_equal_to(200)
    response = context.response.json()

    promotion_name = 'Gallery'
    promotions = response['Promotions']
    promotion_counter = len(response['Promotions'])
    counter = 0
    while counter < promotion_counter:
        if promotions[counter]['Name'] == promotion_name:
            promotion_description = promotions[counter]['Description']
        counter = counter + 1

    assert_that(promotion_description).described_as('Galery Promotion Description').contains('2x larger image')


from cloudmesh_data.data.Provider import Provider
from cloudmesh_data.database import mongo
from deprecated import google_cloud_upload, google_cloud_delete, s3_delete, s3_download, s3_upload, \
    google_cloud_download
from cloudmesh_data.data.Config import Config

def get_files(service):
    config = Config()
    kind = config['cloud']['data'][service]
    provider = Provider()
    provider = provider.get_provider(kind)
    files_list = provider.list(provider["location"])

    return files_list

def get_file_by_name(provider, bucketname, filename, user_uuid):
    if provider == 'aws':
        file_path = s3_download.download_file(bucketname, filename)
        mongo.save_file_to_db('AWS', file_path, filename, user_uuid)
    elif provider == 'google':
        file_path = google_cloud_download.download_blob(bucketname, filename)
        mongo.save_file_to_db('GOOGLE', file_path, filename, user_uuid)

def upload_file_by_name(provider, bucketname, filename):
    if provider == 'aws':
        s3_upload.upload_file(bucketname, filename)
    elif provider == 'google':
        google_cloud_upload.upload_blob(bucketname, filename)

def copy_file(filename, provider, provider_bucket, dest, dest_bucket):
    if provider == dest:
        print("Target cloud needs to different than the source cloud")
        exit
    else:
        get_file_by_name(provider, provider_bucket, filename)
        upload_file_by_name(dest, dest_bucket, filename)

def rsync_file(filename, source, dest):
    print('')

def delete_file(provider, bucketname, filename):
    if provider == 'aws':
        s3_delete.delete_file(bucketname, filename)
    elif provider == 'google':
        google_cloud_delete.delete_blob(bucketname, filename)

def update_user_for_file(user_uuid, filename):
    mongo.update_user_for_file(user_uuid, filename)


# -*- coding: utf-8 -*-
import nltk

file_object  = open('testfile.txt', 'r')
output_object  = open('output.txt', 'w')

def word_to_output(pair): #format word/tag-pair and write it to output
	if pair[1] in ["NN", "NNS", "NNP", "NNPS", "PRP", "PRP$", "WP", "WP$"]:
		tag="NOUN"
	elif pair[1] in ["JJ", "JJR", "JJS"]:
		tag="ADJ"
	elif pair[1] in ["VB", "VBD", "VBG", "VBN", "VBP", "VBZ"]:
		tag="VERB"
	elif pair[1] in ["RB", "RBR", "RBS"]:
		tag="ADVERB"
	else: tag = pair[1]
	output_object.write(pair[0] + "(" + tag + ") ")
	if pair[0] == ".":
		output_object.write("\n")

for line in file_object:
	line = nltk.word_tokenize(line)
	line = nltk.pos_tag(line)
	ner = nltk.ne_chunk(line, binary=False)
	for subtree in ner:
		if type(subtree) is nltk.Tree: #named entity
			output_object.write("[NER] ")
			for pair in subtree:
				word_to_output(pair)
			output_object.write("[" + subtree.label() + "] ")
		else: #not a named entity
			word_to_output(subtree)


# coding=utf-8
'''
先把图片变换，后缀+R

json文件坐标变换，注意变换前后seg的个数，后缀名改变，Imgname那项要改掉

'''
import os
import argparse
import imageio
import imgaug as ia
from imgaug.augmentables.polys import Polygon, PolygonsOnImage
import numpy as np
import json
import imgaug.augmenters as iaa
import random

def parse_args():
    parser = argparse.ArgumentParser(description='Test with Mask RCNN networks.')
    # wjh
    parser.add_argument('--imgdir', type=str, default='',
                        help='Test images, use comma to split multiple.')
    parser.add_argument('--tardir', type=str, default='',
                        help='变换后的图像路径')
    parser.add_argument('--savedir', type=str, default='',
                        help='保存变换json的路径')
    parser.add_argument('--jsondir', type=str, default='',
                        help='总json文件的路径')
    parser.add_argument('--pretrained', type=str, default='True',
                        help='Load weights from previously saved parameters. You can specify parameter file name.')
    args = parser.parse_args()
    return args

def draw():
    image = imageio.imread('./flip/IMG_8038R.JPG')
    json_file = './flip/json/IMG_8038R.json'
    with open(json_file, 'rb') as fp:
        data = json.load(fp)
        all_point = []
        for shapes in data['shapes']:
            aaa = Polygon(shapes['points'])
            all_point.append(aaa)
        psoi = ia.PolygonsOnImage(all_point, shape=image.shape)
        aug = iaa.Sequential([
            iaa.Affine(scale=(1, 1), rotate=(0, 0), translate_percent=(0,0),
                       mode=["constant"], cval=0),  # 仿射变换

            # iaa.Fliplr(0.5), # 左右翻转
            # iaa.PerspectiveTransform((0.01, 0.1)), # 透视变换
            # iaa.AddToHueAndSaturation((-20, 20)),  #
            # iaa.LinearContrast((0.8, 1.2), per_channel=0.5),
            # iaa.Sometimes(0.75, iaa.Snowflakes())   # 高斯模糊
        ])
        image_aug, psoi_aug = aug(image=image, polygons=psoi)
        ia.imshow(psoi_aug.draw_on_image(image_aug, alpha_face=0.2, size_points=7))

# 旋转图片
def rotate_img(soudir, tardir, rate, json_dir, save_dir):
    pathdir = os.listdir(soudir)  # sourdir里不要有目录名
    filenum = len(pathdir)
    picknum = int(filenum*rate)
    sample = random.sample(pathdir, picknum)
    for name in sample:
        json_file = json_dir+name[:-4]+'.json'
        with open(json_file, 'rb') as fp:
            image = imageio.imread(soudir+name)
            data = json.load(fp)
            data['imagePath'] = data['imagePath'][:-4] + 'R.JPG'
            all_point = []
            for shapes in data['shapes']:
                aaa = Polygon(shapes['points'])
                all_point.append(aaa)
            psoi = ia.PolygonsOnImage(all_point, shape=image.shape)
            aug = iaa.Sequential([
                iaa.Affine(scale=(0.8, 1), rotate=(-10,10), translate_percent=(-0.1,0.1),
                           mode=["constant"], cval=0),  # 仿射变换

                # iaa.Fliplr(0.5), # 左右翻转
                # iaa.PerspectiveTransform((0.01, 0.1)), # 透视变换
                # iaa.AddToHueAndSaturation((-20, 20)),  #
                # iaa.LinearContrast((0.8, 1.2), per_channel=0.5),
                # iaa.Sometimes(0.75, iaa.Snowflakes())   # 高斯模糊
            ])
            image_aug, psoi_aug = aug(image=image, polygons=psoi)
            psoi_aug_removed = psoi_aug.remove_out_of_image(fully=True, partly=False)
            psoi_aug_removed_clipped = psoi_aug_removed.clip_out_of_image()
            assert len(psoi_aug.polygons) == len(psoi_aug_removed_clipped.polygons)
            for i,shapes in enumerate(data['shapes']):
                shapes['points'] = psoi_aug_removed_clipped.polygons[i].exterior.astype('int32').tolist()

            imageio.imwrite(tardir+name[:-4]+'R.JPG', image_aug)
            json.dump(data, open(save_dir + json_file[-13:-5] + 'R.json', 'w'), indent=4)

if __name__ == '__main__':
    args = parse_args()
    # rotate_img(args.imgdir, args.tardir, 1, args.jsondir, args.savedir)
    draw()