#!/usr/bin/env python
"""
q polynomials for grassmanians, etc.

See also: coxeter.py

"""

import string, os
from time import sleep, time
from functools import reduce
from functools import reduce, lru_cache
cache = lru_cache(maxsize=None)
from operator import matmul

import numpy
from numpy import alltrue, zeros, dot

from bruhat.argv import argv
from bruhat.smap import SMap
from bruhat.element import Z
from bruhat.poly import Poly
from bruhat.todd_coxeter import Schreier

# See:
# https://en.wikipedia.org/wiki/Q-analog
# https://math.ucr.edu/home/baez/week187.html

ring = Z
zero = Poly({}, ring)
one = Poly({():1}, ring)
q = Poly("q", ring)

class Coxeter(object):
    """
        Coxeter reflection group.
    """
    def __init__(self, ngen, rel):
        """
        rel : map pairs (i,j) of generators to m_{ij}
            (this is the Coxeter matrix)
        """
        for (i, j) in rel.keys():
            assert 0<=i<ngen
            assert 0<=j<ngen
        gen = list(range(ngen))
        for i in gen:
          for j in gen:
            if rel.get((i, j)) and not rel.get((j, i)):
                rel[j, i] = rel[i, j]
        for i in gen:
          for j in gen:
            m = rel.get((i, j))
            if m is None:
                rel[i, j] = 2 # default
        for i in gen:
          for j in gen:
            assert rel[i, j] == rel[j, i]
            #assert rel[i, j] in (2, 3, 4, 6)
        self.ngen = ngen
        self.rel = rel
        key = list(rel.items())
        key.sort()
        self.key = tuple(key)

    def __eq__(self, other):
        return self.key == other.key

    def __hash__(self):
        return hash(self.key)

    @cache
    def get_graph(self):
        return Schreier.make_reflection(self.ngen, self.rel)

    @cache
    def get_group(self):
        graph = self.get_graph()
        G = graph.get_group()
        return G

    def get_residual_graph(self, i):
        assert 0 <= i < self.ngen
        graph = Schreier.make_reflection(self.ngen, self.rel, False)
        hgens = [(j,) for j in range(self.ngen) if i!=j]
        graph.build(hgens)
        return graph

    def get_order(self):
        return len(self.get_group())

    @cache
    def get_poincare(self, ring, q):
        "the poincare polynomial"
        graph = self.get_graph()
        return graph.get_poincare(ring, q)

    def slow_get_poincare(self, ring, q):
        G = self.get_group()
        gen = G.gen
        I = G.identity
        found = {I:0}
        bdy = set([I])
        dist = 0
        while bdy:
            # bdy is the last thing we found
            dist += 1
            _bdy = set()
            for h in bdy:
              for g in gen:
                gh = g*h
                if gh in found:
                    continue
                found[gh] = dist
                _bdy.add(gh)
            bdy = _bdy
        found = list(found.values())
        p = ring.zero
        for i in range(dist):
            p = p + found.count(i) * q**i
        return p

    def subgroup(self, gen):
        lookup = dict((g, i) for (i,g) in enumerate(gen))
        rel = {}
        for i in gen:
          for j in gen:
            rel[lookup[i], lookup[j]] = self.rel[i, j]
        return Coxeter(len(gen), rel)

    def residual(self, i):
        assert 0<=i<self.ngen
        gen = [j for j in range(self.ngen) if j!=i]
        return self.subgroup(gen)

    def __mul__(self, other):
        ngen = self.ngen + other.ngen
        #lookup = dict((i, ngen+i))
        rel = dict(self.rel)
        for key,val in other.rel.items():
            i, j = key
            rel[i+self.ngen, j+self.ngen] = val
        return Coxeter(ngen, rel)

    @classmethod
    @cache
    def A(cls, n):
        rel = {(i, i+1):3 for i in range(n-1)}
        return cls(n, rel)

    @classmethod
    @cache
    def B(cls, n):
        rel = {}
        for i in range(n-1):
            rel[i, i+1] = 4 if i==n-2 else 3
        return cls(n, rel)
    C = B

    @classmethod
    @cache
    def D(cls, n):
        rel = {}
        for i in range(n-2):
            rel[i, i+1] = 3
        if n>2:
            rel[n-3, n-1] = 3
        return cls(n, rel)

    @classmethod
    @cache
    def E(cls, n):
        rel = {}
        for i in range(1, n-1):
            rel[i, i+1] = 3
        rel[0, 3] = 3
        return cls(n, rel)

    @classmethod
    @cache
    def F(cls):
        n = 4
        rel = {(0, 1):3, (1, 2):4, (2, 3): 3}
        return cls(n, rel)

    @classmethod
    @cache
    def G(cls):
        return cls(2, {(0,1):6})

A0 = Coxeter.A(0)
A1 = Coxeter.A(1)
A2 = Coxeter.A(2)
A3 = Coxeter.A(3)
A4 = Coxeter.A(4)

assert A0 is Coxeter.A(0)

G = A4.get_group()
assert len(G) == 120
assert G is A4.get_group()

G = (A2*A3).get_group()
assert len(G) == 6*24

assert A2 == A2
assert A2 != A3
assert A4.subgroup([1, 2]) == A2
assert A4.subgroup([0, 2, 3]) == A1 * A2
assert A4.residual(2) == A2*A1

B0 = Coxeter.B(0)
B1 = Coxeter.B(1)
B2 = Coxeter.B(2)
B3 = Coxeter.B(3)
B4 = Coxeter.B(4)

assert len(B2.get_group()) == 8
assert len(B3.get_group()) == 48
assert len(B4.get_group()) == 384

assert B3.residual(0) == B2
assert B3.residual(1) == A1*A1
assert B3.residual(2) == A2

D0 = Coxeter.D(0)
D1 = Coxeter.D(1)
D2 = Coxeter.D(2)
D3 = Coxeter.D(3)
D4 = Coxeter.D(4)
D5 = Coxeter.D(5)
D6 = Coxeter.D(6)
assert len(D4.get_group()) == 192
#assert D3 == A3 # needs re-indexing
assert D2 == A1*A1
assert D4.residual(1) == A1*A1*A1

E6 = Coxeter.E(6)
E7 = Coxeter.E(7)
E8 = Coxeter.E(8)
F4 = Coxeter.F()
G2 = Coxeter.G()

def shortstr(p):
    assert isinstance(p, Poly)
    keys = p.keys
    items = {}
    m = 0
    for key in keys:
        if len(key)==0:
            items[0] = p.cs[key]
        else:
            c, n = key[0]
            items[n] = p.cs[key]
            m = max(n, m)
    items = [items.get(i, 0) for i in range(m+1)]
    if max(items)>9:
        return "(%s)"%','.join(str(c) for c in items)
    else:
        return ''.join(str(c) for c in items)

class Poincare(object):
    def __init__(self, ring, q):
        self.ring = ring
        self.zero = ring.zero
        self.one = ring.one
        self.q = q

    def qbracket(self, n):
        q = self.q
        p = self.zero
        for i in range(n):
            p = p + q**i
        return p

    def qfactorial(self, n):
        q = self.q
        p = one
        for i in range(n):
            p = p * self.qbracket(i+1)
        return p

    def A(self, n):
        return self.qfactorial(n+1)

    def B(self, n):
        p = one
        for i in range(n):
            p = p * self.qbracket(2*(i+1))
        return p
    C = B

    def D(self, n):
        p = one
        if n==1:
            return 1+self.q # arghh...
        for i in range(n-1):
            p = p * self.qbracket(2*(i+1))
        if n>0:
            p = p * self.qbracket(n)
        return p

    def G2(self):
        q = self.q
        p = one + 2*q + 2*q**2 + 2*q**3 + 2*q**4 + 2*q**5 + q**6
        return p

def test():

    print("test()")

    poincare = Poincare(ring, q)
    A, B, D = poincare.A, poincare.B, poincare.D

    assert q**0 == one

    p = (1+q)**5

    assert poincare.qbracket(5) == 1 + q + q**2 + q**3 + q**4

    assert poincare.qfactorial(3) == 1 + 2*q + 2*q**2 + q**3

    # These count points of the maximal flag variety over the
    # field with q elements:
    assert shortstr(A(0)) == "1"
    assert shortstr(A(1)) == "11"
    assert shortstr(A(2)) == "1221"
    assert shortstr(A(3)) == "1356531"

    assert A(3) == A3.get_poincare(ring, q)

    assert shortstr(B(1)) == "11"
    assert shortstr(B(2)) == "12221"
    assert shortstr(B(3)) == "1357887531"

    assert B(3) == B3.get_poincare(ring, q)

    assert shortstr(D(1)) == "11"
    assert shortstr(D(2)) == "121"
    assert shortstr(D(3)) == "1356531"

    assert D(1) == D1.get_poincare(ring, q)
    assert D(2) == D2.get_poincare(ring, q)
    assert D(3) == D3.get_poincare(ring, q)
    assert D(4) == D4.get_poincare(ring, q)

    p = D(4)
    p = p / (A(1)**3)
    assert shortstr(p) == "1133443311"

    #print(p)

    get = lambda p : p.substitute((('q',2),))
    assert get(p) == 1575

    assert get(B(1) / A(0)) == 3

    assert get(B(2) / A(1)) == 15

    assert get(B(3) / B(2)) == 63
    assert get(B(3) / (A(1)**2)) == 315
    assert get(B(3) / A(2)) == 135

    assert get(B(4) / B(3)) == 255
    assert get(B(4) / (A(1) * B(2))) == 5355
    assert get(B(4) / (A(2) * A(1))) == 11475
    assert get(B(4) / A(3)) == 2295

    top = poincare.G2()
    bot = A(1)
    assert get(top / bot ) == 63

# here we stick B3 into F4...
calculation = """
111111
    111111
      111111
          111111
1111222222221111

 1112111
     1112111
         1112111
1111222222221111

11222211
   11222211
    11222211
      11222211
       11222211
         11222211
          11222211
             11222211
112345677888776543211

    1112111
      1112111
        1112111
          1112111
    1123344433211

112345677888776543211
1112111
  1112111
       1112111
       1112111
            1112111
              1112111
....2246688866422....
    11222211
    11222211
         11222211
         11222211
.......2224222.......

"""

if __name__ == "__main__":

    start_time = time()

    profile = argv.profile
    name = argv.next()
    _seed = argv.get("seed")
    if _seed is not None:
        print("seed(%s)"%(_seed))
        seed(_seed)

    if profile:
        import cProfile as profile
        profile.run("%s()"%name)

    elif name is not None:
        fn = eval(name)
        fn()

    else:
        test()

    t = time() - start_time
    print("finished in %.3f seconds"%t)
    print("OK!\n")


from fastapi import FastAPI
from contextlib import asynccontextmanager
from loguru import logger
import asyncio
from consumer import Worker as Consumer

# on_event deprecated -> lifespan으로 변경
@asynccontextmanager
async def lifespan(app: FastAPI):
    logger.info("lifespan start...")
    consumer = Consumer()
    task2 = asyncio.create_task(consumer.process())
    logger.info("lifespan start !")
    yield
    logger.info("lifespan close...")
    task2.cancel()
    if consumer:
        consumer.close()
    logger.info("lifespan closed!")

app = FastAPI(lifespan=lifespan)

@app.get("/")
async def get_root():
    # logger.debug("this is root!")
    return "this is root"

@app.get("/health")
async def get_health():
    return "healthy"


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

from preprocessing import classify_preprocessing,clustering_preprocessing
from sklearn import svm, tree, metrics
from sklearn.metrics import classification_report
import sklearn.metrics
from sklearn.feature_extraction.text import TfidfVectorizer, TfidfTransformer
from sklearn.model_selection import GridSearchCV
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.neighbors import KNeighborsClassifier
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.utils import shuffle
from sklearn.cluster import KMeans
import matplotlib.pyplot as plt
from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_score
from sklearn.datasets import make_blobs
from sklearn.cluster import KMeans
from sklearn import metrics
import matplotlib.pyplot as plt
import matplotlib.cm as cm
import numpy as np
from functions import learning_curve_graph, accuracy_plot, generate_error_eval,plot_confusion_matrix, clean_class, stopwords_gen
import pandas as pd
import json
from collections import Counter
from sklearn.neighbors import KNeighborsClassifier
from preprocessing import classify_preprocessing
import tensorflow as tf
from tensorflow import keras
import seaborn as sn
from sklearn.neural_network import MLPClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.calibration import calibration_curve
from sklearn.metrics import classification_report, confusion_matrix
from keras import models
from keras import layers
from keras.wrappers.scikit_learn import KerasClassifier
from sklearn.model_selection import cross_val_score
from sklearn.datasets import make_classification
from keras.callbacks import EarlyStopping
from urllib.request import urlopen

#number of cuisines
numCuisine = 400

with urlopen('https://raw.githubusercontent.com/salauddinaliahmed/EBC_7100-Assignment_1/master/train.json') as data_file:
    data = json.loads(data_file.read())

#Classification
complete_vector, x_train, x_test, y_train, y_test, df_sample, x_raw= classify_preprocessing(numCuisine)
df_y = df_sample["cuisine"]

cluster_vector, dictCuisineIngred, numCuisines, numIngred, cuisines, ingredients = clustering_preprocessing(data)

stop_words, stop_tuple = stopwords_gen(dictCuisineIngred)

#Cleaning_the_train : removing the stop words
list_x = []
for each_item in x_raw.items():
    new_dummy = []
    new_dummy.append((each_item[1]).split(','))
    list_x.append(new_dummy)
    new_dummy = []

clean_x_train= clean_class(list_x,stop_words)

new_clean_list = []
for each_t in clean_x_train:
    new_clean_list.append(",".join(each_t))

def comma_split(s):
        return s.split(',')
vectorizertr = TfidfVectorizer(tokenizer=comma_split,
                                ngram_range = ( 1 , 1 ),analyzer="word",
                                max_df = .57 , binary=False , token_pattern=r'\w+' , sublinear_tf=False, lowercase=False)
complete_vector = vectorizertr.fit_transform(new_clean_list)
x_train, x_test = train_test_split(complete_vector, test_size=0.20, random_state = 2)

def comma_split(s):
   return s.split(',')
cuisines  =     ['0:italian',
                 '1:mexican',
                 '2:southern_us',
                 '3:indian',
                 '4:chinese',
                 '5:french',
                 '6:cajun_creole',
                 '7:thai',
                 '8:japanese',
                 '9:greek',
                 '10:spanish',
                 '11:korean',
                 '12:vietnamese',
                 '13:moroccan',
                 '14:british',
                 '15:filipino',
                 '16:irish',
                 '17:jamaican',
                 '18:russian',
                 '19:brazilian']

k_folds = 10

#-------------------- TFIDF-SVM -------------------#
print ("=======================================================================")
print ("Support Vector Machine-----------------------------------------------------------------------")
print ("=======================================================================")
clf_svm = svm.SVC(C=1.0,
                  kernel='linear',
                  degree=3,
                  gamma='auto_deprecated',
                  coef0=0.0,
                  shrinking=True,
                  probability=False,
                  tol=0.001,
                  cache_size=2000,
                  class_weight=None,
                  verbose=False,
                  max_iter=-1,
                  decision_function_shape='ovr',
                  random_state=None)

clf_svm.fit(x_train, y_train)
print ("SVM Classifier accuracy for TFIDF: ", "{:.3%}".format(clf_svm.score(x_test,y_test)))

#Performing cross validation for SVM TFIDF
svm_tfidf_scores = cross_val_score(estimator=clf_svm,
                        X=complete_vector,
                        y=df_y,
                        cv=k_folds)

print('cross-validation accuracy scores TFIDF SVM: %s' % svm_tfidf_scores)
print('cross-validation accuracy: %.3f +/- %.3f' % (np.mean(svm_tfidf_scores), np.std(svm_tfidf_scores)))
#print ("Validation Curve for SVM TFIDF")
#validation_curve_graph(complete_vector, df_y, clf_svm, "gamma","Validation Curve with SVM")
print ("Learning Curve for SVM TFIDF")
learning_curve_graph(clf_svm, complete_vector, df_y)
print("Accuracy Plot fot SVM TFIDF")
accuracy_plot(k_folds, svm_tfidf_scores, clf_svm,"SVM")

#ERROR EVALUATION
print ("------------Error Evaluation for SVM-------------")
print ("Error Evaluation for SVM TFIDF")
svm_tfidf_confu_mat = generate_error_eval(clf_svm, complete_vector, df_y, cuisines, k_folds)

print("Graphs - SVM TFIDF")
plt.figure()
plot_confusion_matrix(svm_tfidf_confu_mat, classes= cuisines,
                     title='Confusion matrix, without normalization')
#Plot normalized confusion matrix
plt.figure()
plot_confusion_matrix(svm_tfidf_confu_mat, classes=cuisines, normalize=True,
                     title='Normalized confusion matrix')
plt.show()

#-------------------- TFIDF-KNN -------------------#
print ("=======================================================================")
print ("k-nearest neighbors--------------------------------------------------------------------")
print ("=======================================================================")
clf_knn = KNeighborsClassifier(weights='uniform',
                               algorithm='auto',
                               leaf_size=30,
                               p=2,
                               metric_params=None,
                               n_jobs=1,
                               n_neighbors=10)

clf_knn.fit(x_train, y_train)
print ("KNN Classifier accuracy for TFIDF: ", "{:.3%}".format(clf_knn.score(x_test,y_test)))

#Performing cross validation for KNN TFIDF
knn_tfidf_scores = cross_val_score(estimator=clf_knn,
                        X=complete_vector,
                        y=df_y,
                        cv=k_folds)

print('cross-validation accuracy scores TFIDF KNN: %s' % knn_tfidf_scores)
print('cross-validation accuracy: %.3f +/- %.3f' % (np.mean(knn_tfidf_scores), np.std(knn_tfidf_scores)))
#print ("Validation Curve for KNN TFIDF")
#validation_curve_graph(complete_vector, df_y, clf_knn,"n_neighbors","Validation Curve with KNN")
print ("Learning Curve for KNN TFIDF")
learning_curve_graph(clf_knn, complete_vector, df_y)
print("Accuracy Plot fot KNN TFIDF")
accuracy_plot(k_folds, knn_tfidf_scores, clf_knn,"KNN")

#ERROR EVALUATION
print ("------------Error Evaluation for KNN-------------")
print ("Error Evaluation for KNN TFIDF")
knn_tfidf_confu_mat = generate_error_eval(clf_knn, complete_vector, df_y,cuisines, k_folds)

print("Graphs - KNN TFIDF")
plt.figure()
plot_confusion_matrix(knn_tfidf_confu_mat, classes= cuisines,
                     title='Confusion matrix, without normalization')
#Plot normalized confusion matrix
plt.figure()
plot_confusion_matrix(knn_tfidf_confu_mat, classes=cuisines, normalize=True,
                     title='Normalized confusion matrix')
plt.show()
#-------------------- TFIDF-Decision Tree -------------------#
print ("=======================================================================")
print ("Decision Tree--------------------------------------------------------------------")
print ("=======================================================================")
clf_dtree = tree.DecisionTreeClassifier(criterion='gini',
                                        splitter='best',
                                        max_depth=None,
                                        min_samples_split=2,
                                        min_samples_leaf=1,
                                        min_weight_fraction_leaf=0.0,
                                        max_features=None,
                                        random_state=None,
                                        max_leaf_nodes=None,
                                        min_impurity_decrease=0.0,
                                        min_impurity_split=None,
                                        class_weight=None,
                                        presort=False)

clf_dtree.fit(x_train, y_train)
#Performing cross validation for Decision Tree TFIDF
dtree_tfidf_scores = cross_val_score(estimator=clf_dtree,
                        X=complete_vector,
                        y=df_y,
                        cv=k_folds)
print ("Decision Tree Classifier accuracy for TFIDF: ", "{:.3%}".format(clf_dtree.score(x_test,y_test)))
print('cross-validation accuracy scores TFIDF Decision Tree: %s' % dtree_tfidf_scores)
print('cross-validation accuracy: %.3f +/- %.3f' % (np.mean(dtree_tfidf_scores), np.std(dtree_tfidf_scores)))
#print ("Validation Curve for Decision Tree TFIDF")
#validation_curve_graph(complete_vector, df_y, clf_dtree,"max_depth","Validation Curve with Decision Tree")
print ("Learning Curve for Decision Tree TFIDF")
learning_curve_graph(clf_dtree, complete_vector, df_y)
print("Accuracy Plot fot Decision Tree TFIDF")
accuracy_plot(k_folds, dtree_tfidf_scores, clf_dtree,"Decision Tree")

#ERROR EVALUATION
print ("------------Error Evaluation for Decision Tree-------------")
print ("Error Evaluation for Decision Tree TFIDF")
dtree_tfidf_confu_mat = generate_error_eval(clf_dtree, complete_vector, df_y,cuisines, k_folds)

print("Graphs - Decision Tree TFIDF")
plt.figure()
plot_confusion_matrix(dtree_tfidf_confu_mat, classes= cuisines,
                     title='Confusion matrix, without normalization')
#Plot normalized confusion matrix
plt.figure()
plot_confusion_matrix(dtree_tfidf_confu_mat, classes=cuisines, normalize=True,
                     title='Normalized confusion matrix')
plt.show()

#-------------------- TFIDF-Random Forest -------------------#
print ("=======================================================================")
print ("Random Forest--------------------------------------------------------------------")
print ("=======================================================================")
clf_rf = RandomForestClassifier(bootstrap=True,
                                class_weight=None,
                                criterion='gini',
                                max_depth=2,
                                max_features='auto',
                                max_leaf_nodes=None,
                                min_impurity_decrease=0.0,
                                min_impurity_split=None,
                                min_samples_leaf=1,
                                min_samples_split=2,
                                min_weight_fraction_leaf=0.0,
                                n_estimators=100,
                                n_jobs=None,
                                oob_score=False,
                                random_state=0,
                                verbose=0,
                                warm_start=False)
clf_rf.fit(x_train, y_train)
#Performing cross validation for Random Forest TFIDF
rf_tfidf_scores = cross_val_score(estimator=clf_rf,
                        X=complete_vector,
                        y=df_y,
                        cv=k_folds)
print ("Random Forest Classifier accuracy for TFIDF: ", "{:.3%}".format(clf_rf.score(x_test,y_test)))
print('cross-validation accuracy scores TFIDF Random Forest: %s' % rf_tfidf_scores)
print('cross-validation accuracy: %.3f +/- %.3f' % (np.mean(rf_tfidf_scores), np.std(rf_tfidf_scores)))
print ("Learning Curve for Random Forest TFIDF")
learning_curve_graph(clf_rf, complete_vector, df_y)
print("Accuracy Plot fot Random Forest TFIDF")
accuracy_plot(k_folds, rf_tfidf_scores, clf_rf,"Random Forest")

#ERROR EVALUATION
print ("------------Error Evaluation for Random Forest-------------")
print ("Error Evaluation for Random Forest TFIDF")
rf_tfidf_confu_mat = generate_error_eval(clf_rf, complete_vector, df_y,cuisines, k_folds)

print("Graphs - Random Forest TFIDF")
plt.figure()
plot_confusion_matrix(rf_tfidf_confu_mat, classes= cuisines,
                     title='Confusion matrix, without normalization')
#Plot normalized confusion matrix
plt.figure()
plot_confusion_matrix(rf_tfidf_confu_mat, classes=cuisines, normalize=True,
                     title='Normalized confusion matrix')
plt.show()

##Taking user input here for prediction
user_input = input("Please enter a comma seperated ingredients: ")
myList = user_input.split(",")

#Removing the stop words from the given list of ingredients
clean_x_train_1= clean_class(list_x,stop_words)

clean_x_train_1.append(myList)

new_clean_list_1 = []
for each_t in clean_x_train:
    new_clean_list_1.append(",".join(each_t))

#Performing the TFIDF vector on the given list of ingredients
def comma_split(s):
        return s.split(',')
vectorizertr = TfidfVectorizer(tokenizer=comma_split,
                                ngram_range = ( 1 , 1 ),analyzer="word",
                                max_df = .57 , binary=False , token_pattern=r'\w+' , sublinear_tf=False, lowercase=False)
complete_vector_1 = vectorizertr.fit_transform(new_clean_list_1)

#Predicting the cusine based on the given set of ingredients
outputClass = clf_svm.predict(complete_vector_1[-1])
print ("This is the prediction value",outputClass)


'''
@Description: make mietable in a batch mode
@Author: Hejun Xie
@Date: 2020-07-02 08:35:04
@LastEditors: Hejun Xie
@LastEditTime: 2020-07-06 12:36:50
'''
# -*- coding: utf-8 -*-

import os
import sys
import datetime
import logging

if __name__ == "__main__":

    # [A]. logging configure
    log_datetime = datetime.datetime.now()
    log_filename = './log/{:%Y%m%d%H%M%S}.txt'.format(log_datetime)

    logger = logging.getLogger()
    logger.setLevel(logging.DEBUG)

    fh_debug = logging.FileHandler(log_filename + ".debug", mode='w')
    fh_debug.setLevel(logging.DEBUG)

    fh_info = logging.FileHandler(log_filename + ".info", mode='w')
    fh_info.setLevel(logging.INFO)

    ch = logging.StreamHandler()
    ch.setLevel(logging.INFO)

    formatter = logging.Formatter("%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s")
    fh_debug.setFormatter(formatter)
    fh_info.setFormatter(formatter)
    ch.setFormatter(formatter)

    logger.addHandler(fh_debug)
    logger.addHandler(fh_info)
    logger.addHandler(ch)

    # I/O
    config_dir = "./channels_dat_liuddashapes"
    rttov_home = "../../"
    output_dir = os.path.join(rttov_home, 'rtcoef_rttov12', 'mietable')
    tempdir    = "./mietable_temp"

    # start
    filelist = os.listdir(config_dir)

    # remove channels.dat
    if os.path.exists("channels.dat"):
        os.system("rm channels.dat")

    for configfile in filelist:
        config_filepath = os.path.join(config_dir, configfile)
        logger.info(config_filepath)
        os.system("cp {} ./channels.dat".format(config_filepath))

        # run
        command = "./mie_table_generation.ksh"
        logger.debug("\n" + command)
        pipe = os.popen(command)
        resp = pipe.read()
        logger.debug("\n" + resp)
        # os.system(command)

        os.system("rm channels.dat")

        # rename and move
        tempfile        = os.listdir(tempdir)[0]
        tempfilepath    = os.path.join(tempdir, tempfile)
        segments        = tempfile.split('.')
        shapename       = config_filepath.split('_')[-1]

        segments[0]     = '_'.join([segments[0], shapename])
        newname         = '.'.join(segments)
        newpath         = os.path.join(output_dir, newname)

        logger.info(tempfilepath)
        logger.info(newpath)

        os.system("mv {} {}".format(tempfilepath, newpath))


import base64
import uuid
import string
from random import randrange
import random

def random_char(y):
       return ''.join(random.choice(string.ascii_letters) for x in range(y))

code=input('Python code to be encrypt : ')

randomuuid=uuid.uuid4()
randomuuid=str(randomuuid).replace('-','')

code=code+'#'+randomuuid

encoded=base64.b64encode(code.encode('ascii'))

encoded=str(encoded, 'utf-8')

print(encoded)

shelter="import base64;exec(base64.b64decode('"+encoded+"'))"

encoded=base64.b64encode(shelter.encode('ascii'))

encoded=str(encoded, 'utf-8')

random_index = randrange(0,len(encoded))

replace=encoded[random_index]

print('replaced :'+replace)

encoded=encoded.replace(encoded[random_index],'*')

rand = random_char(10)

composition="import base64;"+rand+"='"+encoded+"';"+rand+"="+rand+".replace('*','"+replace+"');exec(base64.b64decode("+rand+"))"

print(composition)


import cv2
import sys
import os
import time
from enum import Enum
from djitellopy import Tello

class State (Enum):

    detectingFaces = 1
    goingToPerson = 2

state = State.detectingFaces

S=60
FPS=30

distanceThreshold= 200

speed= 10

currentScreenshot = 1

TOLERANCE_X = 4
TOLERANCE_Y = 4
SLOWDOWN_THRESHOLD_X = 20
SLOWDOWN_THRESHOLD_Y = 2
DRONE_SPEED_X = 10
DRONE_SPEED_Y = 10
SET_POINT_X = 480
SET_POINT_Y = 360

faceCascade = cv2.CascadeClassifier("faced.xml")
drone = Tello() #  tello o drone = Tello() declaramos tello como objeto.
drone.connect()
drone.takeoff()
drone.move_up(200)
time.sleep(2)
drone.streamon()  # start camera streaming

# video_capture = cv2.VideoCapture("udp://0.0.0.0:11111")
# video_capture = cv2.VideoCapture("rtsp://192.168.1.1")
# video_capture = cv2.VideoCapture(0)

while True:

    # ret, frame = video_capture.read()

    frame = drone.get_frame_read().frame  # capturing frame from drone
    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)  # turning image into gray scale

    faces = faceCascade.detectMultiScale(  # face detection
        gray,
        scaleFactor=1.1,
        minNeighbors=5,
        minSize=(30, 30),
        flags=cv2.CASCADE_SCALE_IMAGE
    )
    if state == State.detectingFaces:
        drone.rotate_clockwise(15)
        time.sleep(1)
    i = 0
    # Decorating image for debug purposes and looping through every detected face
    for (x, y, w, h) in faces:

        cv2.rectangle(frame, (x, y), (x+w, y+h), (255, 0, 0), 5)  # contour rectangle
        cv2.circle(frame, (int(x+w/2), int(y+h/2)), 12, (255, 0, 0), 1)  # face-centered circle
        # print(frame.shape)
        # cv2.line(frame, (int(x+w/2), int(720/2)), (int(960/2), int(720/2)), (0, 255, 255))

        cv2.circle(frame, (int(SET_POINT_X), int(SET_POINT_Y)), 12, (255, 255, 0), 8)  # setpoint circle
        i = i+1
        distanceX = x+w/2 - SET_POINT_X
        distanceY = y+h/2 - SET_POINT_Y

        up_down_velocity = 0
        right_left_velocity = 0
        if state == State.detectingFaces:
            state = State.goingToPerson

        if distanceX < -TOLERANCE_X:
            print("Mover dron a la izquierda")
            right_left_velocity = - DRONE_SPEED_X

        elif distanceX > TOLERANCE_X:
            print("Mover el dron a la derecha")
            right_left_velocity = DRONE_SPEED_X
        else:
            print("OK")

        if distanceY < -TOLERANCE_Y:
            print("Mover el dron para arriba")
            up_down_velocity = DRONE_SPEED_Y
        elif distanceY > TOLERANCE_Y:
            print("Mover el dron para abajo")
            up_down_velocity = - DRONE_SPEED_Y

        else:
            print("OK")

        if abs(distanceX) < SLOWDOWN_THRESHOLD_X:
            right_left_velocity = int(right_left_velocity / 2)
        if abs(distanceY) < SLOWDOWN_THRESHOLD_Y:
            up_down_velocity = int(up_down_velocity / 2)

        drone.send_rc_control(right_left_velocity, 0, up_down_velocity, 0)

    cv2.imshow('Video', frame)

    if cv2.waitKey(1) & 0xFF == ord('q'):  # quit from script
        break

# video_capture.release()
drone.land()
drone.streamoff()
cv2.destroyAllWindows()
sys.exit()


import requests
import pandas as pd

#   This time, I'm thinking I should develop a measure of how much different papers are covering crime.
# do newspapers in cities with more crime cover it more?   Or, rather, which newspapers take notice when
# there is a crime wave going on in their city?
#   There are limitations to using this "search" method.  Perhaps we should be checking for articles that have
# "yesterday", "this week", "this morning" or something to that effect in their text.  That way, we know they
# are discussing recent crime rather than a crime long ago.
#   Another limitation is that cities with higher crime may be more prone to writing summary articles.  When Chicago
# has 50 shootings in a weekend, even very crime-centric paper could be inclined to write a summary article of all of
# the shootings, and then perhaps cover only a few of the victims.  We can probably handle that by taking the square
# root of crime in the city.
#   I suppose this also means that we're going to focus in on a couple of major cities.  Perhaps just look at NYC,
# LA, and Seattle.  Then,

cities = [  'Atlanta',
            '/"District of Columbia/"',
            'Seattle',
            'St. Louis',
            'New York',
            'Madison',
            'Chicago',
            'Los Angeles',
            'San Francisco',
            'Miami',
            'Orlando',
            'Columbus',
            'Boston',
            'Portland',
            'Detroit',
            'Baltimore',
            'Pittsburgh',
            'Oakland',
            'Houston',
            'Philadelphia',
            'Newark',
            'Dallas',
            'Austin',
            'Cleveland',
            'Denver',
            'Tampa',
            'Sacramento',
            'Kansas City',
            'Las Vegas',
            'Phoenix',
            'Salt Lake City',
            'San Diego',
            'Cincinnati',
            'Indianapolis',
            'Oklahoma City',
            'San Antonio'
            ]
queries = ['any','violence','looting','arson']

df = pd.DataFrame({     'city' : sorted(cities * len(queries)),
                        'query': queries * len(cities),
                        'hits' : [0] * len(queries) * len(cities) })

strPath = '/home/chmpearson/keys/bing_sub_key.txt'
f = open(strPath)
subscription_key = f.read()

endpoint  = 'https://api.cognitive.microsoft.com/bing/v7.0/news/search'
since       = '2020-06-30'
headers = {"Ocp-Apim-Subscription-Key" : subscription_key}
params  = { "q": search_term,
            "sortBy": "Date",
            "since": since,
            "textDecorations": True,
            "textFormat": "HTML"
            }

for i, city, query in zip(df.index, df['city'], df['query'] ):
    print(i)
    print(city)
    print(query)
    if query == "any":
        params['q'] = city
    else:
        params['q'] = city + '+' + query
    df.iloc[i,2] = requests.get(endpoint, params = params, headers = headers).json()['totalEstimatedMatches']

df.to_csv('unrest.csv')


import gevent.monkey
gevent.monkey.patch_all(thread=False)

import gevent
from gevent import queue
import queue as builtin_queue

import click
import feedparser
import requests
from cachecontrol import CacheControl
from furl import furl
import dateutil.parser

import uuid
import hashlib
import random
import time
import sys
import textwrap
import logging

log = logging.getLogger(__name__)

orig_session = requests.session()
orig_session.headers = {'User-Agent': 'feedreader github.com/simonschmidt/feedreader'}

session = CacheControl(orig_session)

class Feed(object):
    """
    A self-refreshing feed.

    Arguments:
        url(str): Feed url
        interval(int): Polling interval
        include_initial(bool): Include items from first update

    Attributes:
        subscribers(set): Queues that will recieve content on updates

    Example:
        >>> feed = Feed('example://foo')
        >>> q = queue.Queue()
        >>> feed.subscribers.add(q)
    """
    RETRY_DELAY = 60

    def __init__(self, url, interval=60, include_initial=False, _spawn_greenlet=True):
        self.url = url
        self.interval = interval
        self._discard_next_update = not include_initial

        # Queues for interested parties
        self.subscribers = set()

        # All seen id's, worth limiting size of this?
        self._old_ids = set()

        # Start fetching
        if _spawn_greenlet:
            self._greenlet = gevent.spawn(self._fetcher)

    def update(self):
        """Manually trigger update"""
        items = self._get_new_items()

        if self._discard_next_update:
            self._discard_next_update = False
            items = []

        self._notify_subscribers(items)

        return items

    def _notify_subscribers(self, items):
        for item in items:
            for q in self.subscribers:
                try:
                    q.put_nowait((self, item))
                except Exception:
                    log.exception("Failed to enqueue item")

    def _get_new_items(self):
        items = _items_from_feed(self.url, skip_ids=self._old_ids)

        for item in items:
            self._old_ids.add(_item_id(item))

        return items

    def _fetcher(self):
        """Schedule updates"""
        while True:
            if len(self.subscribers) == 0:
                log.debug("Nobody cares about %s", self)
                gevent.sleep(1)
                continue

            try:
                log.debug("Updating feed %s", self)
                self.update()
                log.debug("Updated feed %s", self)
            except Exception as exc:
                retry_delay = min(self.RETRY_DELAY, self.interval)
                log.warning("Unable to update %s, retrying in %s seconds", self, retry_delay, exc_info=True)
                gevent.sleep(retry_delay)
                continue

            # BUG: If interval is lowered it wont take effect until after next update
            gevent.sleep(self.interval)

    def __repr__(self):
        return "Feed<url='{}', interval={}>".format(self.url, self.interval)

def _item_id(item):
    """ID for a feed item"""
    if item.id:
        return item.id

    # No ID, concat title and published *shrug*
    return hashlib.sha256("{}|{}".format(item.title, item.published)).hexdigest()

def _items_from_feed(url, skip_ids=None):
    scheme = furl(url).scheme

    if scheme == 'example':
        items = _fake_items(url)
    elif scheme in ('http', 'https'):
        r = session.get(url)
        r.raise_for_status()
        items = feedparser.parse(r.text).entries
    else:
        # It's a baby whale Jay!
        raise ValueError("Unsupported scheme '{}' in '{}'".format(scheme, url))

    items = [
        item
        for item in items
        if (skip_ids is None) or (_item_id(item) not in skip_ids)]
    items.sort(key=lambda x: x.published_parsed or x.id)

    return items

class FakeEntry(object):
    def __init__(self, id_=None):
        self.id = id_ or str(uuid.uuid4())
        self.title = "<example>"
        self.link = "example://{}".format(self.id)
        self.published = '2001-02-03 04:05:06'
        self.published_parsed = None

def _fake_items(url):
    if url != 'example://test':
        time.sleep(random.randint(1, 2))
        if random.random() < 0.3:
            raise RuntimeError("eth0 on fire")

    return [FakeEntry('old_id'), FakeEntry()]

def test_initial_items_excluded():
    f = Feed("example://test", _spawn_greenlet=False)

    assert len(f.update()) == 0
    assert len(f.update()) > 0

def test_item_included_exactly_once():
    f = Feed('example://test', include_initial=True, _spawn_greenlet=False)

    initial_ids = [_item_id(item) for item in f.update()]
    assert 'old_id' in initial_ids

    new_ids = [_item_id(item) for item in f.update()]
    assert len(new_ids) > 0
    assert 'old_id' not in new_ids

def test_items_are_queued():
    q = builtin_queue.Queue()
    f = Feed('example://test', include_initial=True, _spawn_greenlet=False)

    f.subscribers.add(q)
    f.update()

    (f_res, item) = q.get()
    assert f_res is f

@click.command()
@click.option(
    '--loglevel',
    type=click.Choice(['CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG']),
    default='ERROR')
@click.option('--interval', default=5 * 60, metavar='SECONDS', help='Update interval')
@click.option('--include-initial/--drop-initial', default=False)
@click.argument('urls', nargs=-1, metavar='URL...')
def main(*, interval, loglevel, include_initial, urls):
    """Feed reader for RSS and Atom"""
    logging.basicConfig(level=loglevel, stream=sys.stderr)

    print_queue = queue.Queue()
    for url in urls:
        feed = Feed(url, interval, include_initial=include_initial)
        feed.subscribers.add(print_queue)

    def printer():
        template = '\n'.join(("{date}", "{feed}", "  {title}", "  {link}", ""))
        for feed, item in print_queue:
            title_lines = '\n  '.join(textwrap.wrap(item.title))
            output = template.format(
                date=dateutil.parser.parse(item.published),
                feed=feed,
                title=title_lines,
                link=item.link)
            print(output)

    printer_greenlet = gevent.spawn(printer)
    gevent.wait([printer_greenlet])


# -*- coding: utf-8 -*-

# MIT License

from base64 import b64encode
import json
from os.path import dirname, join, exists
import unittest

from onelogin.saml2.response import OneLogin_Saml2_Response
from onelogin.saml2.settings import OneLogin_Saml2_Settings

class OneLogin_Saml2_SignedResponse_Test(unittest.TestCase):
    data_path = join(dirname(dirname(dirname(dirname(__file__)))), 'data')
    settings_path = join(dirname(dirname(dirname(dirname(__file__)))), 'settings')

    def loadSettingsJSON(self):
        filename = join(self.settings_path, 'settings1.json')
        if exists(filename):
            stream = open(filename, 'r')
            settings = json.load(stream)
            stream.close()
            return settings
        else:
            raise Exception('Settings json file does not exist')

    def file_contents(self, filename):
        f = open(filename, 'r')
        content = f.read()
        f.close()
        return content

    def testResponseSignedAssertionNot(self):
        """
        Tests the getNameId method of the OneLogin_Saml2_Response
        Case valid signed response, unsigned assertion
        """
        settings = OneLogin_Saml2_Settings(self.loadSettingsJSON())
        message = self.file_contents(join(self.data_path, 'responses', 'open_saml_response.xml'))
        response = OneLogin_Saml2_Response(settings, b64encode(message))

        self.assertEquals('someone@example.org', response.get_nameid())

    def testResponseAndAssertionSigned(self):
        """
        Tests the getNameId method of the OneLogin_Saml2_Response
        Case valid signed response, signed assertion
        """
        settings_info = self.loadSettingsJSON()
        settings_info['idp']['entityId'] = "https://federate.example.net/saml/saml2/idp/metadata.php"
        settings_info['sp']['entityId'] = "hello.com"

        settings = OneLogin_Saml2_Settings(settings_info)
        message = self.file_contents(join(self.data_path, 'responses', 'simple_saml_php.xml'))
        response = OneLogin_Saml2_Response(settings, b64encode(message))

        self.assertEquals('someone@example.com', response.get_nameid())

if __name__ == '__main__':
    runner = unittest.TextTestRunner()
    unittest.main(testRunner=runner)


import matplotlib.pyplot as plt
import matplotlib.lines as lines
import math
from numpy import array, linspace
import lib.sethares as sethares
import lib.tuning as tuning

class TuningPlot():
    def __init__(self, direction = 'h', size = 10, rangehigh = 1):
        self.pos = 0
        self.ticks = []
        self.ticklabels = []
        self.ratios = []
        self.direction = direction
        self.rangehigh = rangehigh
        self.fig, self.ax = plt.subplots(figsize=(size, size))

    def plotTuning(self, tuning, repeat = 1, notes = None, label = None):
        for i in range(len(tuning.intervals)*repeat):
            if (notes is None or i in notes):
                interval = tuning.getInterval(i)
                intervalName = tuning.getIntervalName(i)
                if (self.direction == 'v'):
                    self.ax.add_line(lines.Line2D([self.pos,self.pos+1], [interval, interval], lw=2, color="#a75d9b"))
                    self.ax.annotate(intervalName, xy=(self.pos, interval), xycoords='data', xytext=(3, 3), textcoords='offset points')
                else:
                    self.ax.add_line(lines.Line2D([interval, interval], [self.pos,self.pos+1], lw=2, color="#a75d9b"))
                    self.ax.annotate(intervalName, xy=(interval, self.pos+0.5), xycoords='data', xytext=(3, 3), textcoords='offset points')
        if (label is None):
            self.addCol(tuning.name)
        else:
            self.addCol(label)

    def plotMode(self, mode, root = 0, repeat = 1, label = None, wrap = False):
        self.plotTuning(mode.tuning, repeat, mode.getNotes(root, repeat, wrap), label)

    def plotChord(self, chord, root = 0, label = None):
        self.plotTuning(chord.mode.tuning, self.rangehigh, chord.getNotes(root), label)

    def plotDissonance(self, harmonics, scale = 4):
        freq = 500 * array(harmonics)
        amp = 0.88 ** (array(harmonics)-1)
        diss = sethares.dissonanceCurve(freq, amp, 2**self.rangehigh)
        yvalues = list(map(tuning.ratioToOctaves, linspace(1, 2**self.rangehigh, len(diss))))
        if (self.direction == 'v'):
            self.ax.plot((diss/scale)+self.pos, yvalues)
        else:
            self.ax.plot(yvalues, ((scale-diss)/scale)+self.pos)
        self.addCol("Dissonance")

    def plotRatios(self, ratios):
        self.ratios = ratios
        for n, d in ratios:
            if (self.direction == 'v'):
                plt.axhline(tuning.ratioToOctaves(n/d), color='silver')
            else:
                plt.axvline(tuning.ratioToOctaves(n/d), color='silver')

    def addCol(self, name):
        self.ticks.append(self.pos+0.5)
        self.ticklabels.append(name)
        self.pos = self.pos + 1

    def plot(self, aspect = 0.1):
        if (self.direction == 'v'):
            self.ax.set(xlim=(0, self.pos), ylim=(0, self.rangehigh), aspect=5)
            plt.xticks(self.ticks, self.ticklabels)
            plt.yticks([tuning.ratioToOctaves(n/d) for n, d in self.ratios], ['{}/{}'.format(n, d) for n, d in self.ratios])
        else:
            self.ax.set(xlim=(0, self.rangehigh), ylim=(self.pos, 0), aspect=aspect*self.rangehigh)
            plt.yticks(self.ticks, self.ticklabels)
            plt.xticks([tuning.ratioToOctaves(n/d) for n, d in self.ratios], ['{}/{}'.format(n, d) for n, d in self.ratios])
        self.ax.grid(False)
        plt.show()

class TuningPolarPlot():
    def __init__(self, size = 10, width = 1, height = 1, plots = 1, rangehigh = 1):
        self.pos = 0
        self.plots = plots
        self.ticks = []
        self.ticklabels = []
        self.ratios = []
        self.rangehigh = rangehigh
        self.width = width
        self.height = height
        self.plotnum = 0
        self.fig = plt.figure(figsize=(size*width, size*height))
        self.fig.tight_layout()
        self.newPlot()
        plt.subplots_adjust(wspace=0.3, hspace=0.3)

    def newPlot(self):
        self.plotnum = self.plotnum + 1
        self.ax = self.fig.add_subplot(self.height, self.width, self.plotnum, projection='polar')

    def plotDissonance(self, harmonics, scale = 4):
        freq = 500 * array(harmonics)
        amp = 0.88 ** (array(harmonics)-1)
        diss = sethares.dissonanceCurve(freq, amp, 2**self.rangehigh)
        yvalues = list(map(lambda r: tuning.ratioToOctaves(r)*2*math.pi, linspace(1, 2**self.rangehigh, len(diss))))
        self.ax.plot(yvalues, ((scale-diss)/scale))

    def plotRatios(self, ratios, offset=0):
        self.ratios = ratios
        self.ratiosOffset = offset

    def plotTuning(self, tuning, repeat = 1, notes = None, label = None):
        x_values = []
        y_values = []
        for i in range(len(tuning.intervals)*repeat):
            if (notes is None or i in notes):
                interval = tuning.getInterval(i)
                intervalName = tuning.getIntervalName(i)
                plt.plot(interval*2*math.pi, 1, marker='o', markersize=10, color="#008583")
                # self.ax.add_line(lines.Line2D([interval, interval], [self.pos,self.pos+1], lw=2))
                self.ax.annotate(intervalName, xy=(interval*2*math.pi, 1), xycoords='data', xytext=(interval*2*math.pi, 1.25),
                    textcoords='data', va='center', ha='center', color="#008583")

    def plotScale(self, scale, root = 0, repeat = 1, label = None, wrap = False):
        x_values = []
        y_values = []
        notes = scale.getNotes(root, repeat, wrap)
        for i in range(len(scale.tuning.intervals)*repeat):
            if (notes is None or i in notes):
                interval = scale.tuning.getInterval(i)
                intervalName = scale.tuning.getIntervalName(i)
                x_values.append(interval*2*math.pi)
                y_values.append(1)
                self.ax.annotate(intervalName, xy=(interval*2*math.pi, 1), xycoords='data', xytext=(interval*2*math.pi, 1.25),
                    textcoords='data', va='center', ha='center', color="#a75d9b")
        plt.plot(x_values, y_values, marker='o', markersize=10, color="#a75d9b", linestyle="-")
        self.ax.annotate(label, xy=(0, 0), xycoords='data', ha='center', color="#a75d9b", fontsize=16)
        # self.ax.title.set_text(label)

    def plotChord(self, chord, root = 0, label = None, color = "#e14200"):
        x_values = []
        y_values = []
        notes = chord.getNotes(root)
        notes = list(map(lambda n: n%len(chord.scale.tuning.intervals), notes))
        for i in range(len(chord.scale.tuning.intervals)):
            if (notes is None or i in notes):
                interval = chord.scale.tuning.getInterval(i)
                intervalName = chord.scale.tuning.getIntervalName(i)
                x_values.append(interval*2*math.pi)
                y_values.append(1)
                self.ax.annotate(intervalName, xy=(interval*2*math.pi, 1), xycoords='data', xytext=(interval*2*math.pi, 1.25),
                    textcoords='data', va='center', ha='center', color=color)
        x_values.append(x_values[0])
        y_values.append(y_values[0])
        plt.plot(x_values, y_values, marker='o', markersize=6, color=color, linestyle="--")
        self.ax.annotate(label, xy=(0, 0), xycoords='data', ha='center', color=color, fontsize=16)

    def plot(self):
        plt.yticks([1], [""])
        plt.xticks([(tuning.ratioToOctaves(n/d)+self.ratiosOffset)%1*2*math.pi for n, d in self.ratios], ['' for n, d in self.ratios])
        for i in range(len(self.ratios)):
            ratioLabel = '{}/{}'.format(self.ratios[i][0], self.ratios[i][1])
            x = (tuning.ratioToOctaves(self.ratios[i][0]/self.ratios[i][1])+self.ratiosOffset)%1*2*math.pi
            self.ax.annotate(ratioLabel, xy=(x, 0.7), xycoords='data', ha='center', color="black")
        self.ax.set_theta_direction(-1)
        self.ax.set_theta_offset(math.pi / 2.0)
        self.ax.set(ylim=(0, 1.1))
        self.ax.set_frame_on(False)
        plt.grid(axis='y', color='#008583', linestyle='-', linewidth=2)
        plt.grid(axis='x', color='silver', linestyle='-', linewidth=1)
        if(self.plotnum < self.plots):
            self.newPlot()

    def show(self):
        plt.show()


# =============================================================================
# find each ticker's name, industry and save into file
# =============================================================================
import time
import pandas as pd
import requests
from bs4 import BeautifulSoup

def get_ticker_profile(f_from, f_to):
    tickers_prof = pd.DataFrame(data={'ticker':[], 'name':[], 'industry':[]})
    tickers = [line.rstrip() for line in open(f_from)]
    count = 0

    for tik in tickers:
        time.sleep(1)
        count += 1
        print('https://finance.yahoo.com/quote/'+tik+'/profile?p='+tik)
        req = requests.get('https://finance.yahoo.com/quote/'+tik+'/profile?p='+tik, allow_redirects=False)
        soup = BeautifulSoup(req.content, 'html.parser')

        ticker_name = ''
        ticker_industry = ''

        if soup.find('h3', {'data-reactid':'6'}):
            ticker_name = [soup.find('h3', {'data-reactid':'6'})][0].string
        elif soup.find('h3', {'class':'Mb(5px) Mend(40px)'}):
            ticker_name = [soup.find('h3', {'class':'Mb(5px) Mend(40px)'})][0].string
        elif soup.find('span', {'data-reactid':'5'}):
            ticker_name = [soup.find('span', {'data-reactid':'5'})][0].string

        if soup.find('strong', {'data-reactid':'21'}):
            ticker_industry = [soup.find('strong', {'data-reactid':'21'})][0].string
        elif soup.find('span', {'class':'Fl(end)'}):
            ticker_industry = [soup.find('span', {'class':'Fl(end)'})][0].string
#                                'industry':str([soup.find('span', {'class':'Fl(end)'})][0].string+'-'+[soup.find('span', {'class':'Fl(end)', 'data-reactid':'29'})][0].string)},
        else:
            ticker_industry = ticker_name

        tickers_prof = tickers_prof.append({'ticker':tik,
                            'name':ticker_name,
                            'industry':ticker_industry},
                            ignore_index=True)

    tickers_prof.to_csv(f_to)


#!/usr/bin/python3

#=============
# homenetGrapher.py
# solely for depicting internal-to-internal network traffic, ideally running on a cron schedule and regularly monitored
# for SMALL to MEDIUM-sized networks
#
# v0.1 - 20 May 2019
# 	- first version, implements a configuration file requirement, basic labels on nodes, command line args, exclusions in the config
#
# use Zeek (Bro)'s conn.log and homenets.cfg to create visual images with GraphViz depicting traffic solely between internal hosts
#
# homenets.cfg
# add CIDRs representing your network ranges to homenets.cfg
# add individual IPs to ignore
#
# TODO
# - write to JSON and send to a web server running d3 and d3-force
# - display changes over time / over last x runs
# - Zeek uid pairing and/or additional triggering based off notices.log events
# - address coverage gaps between log rotation (look at a log that changed since last being read, which has also rotated and been gzip'd)#
# - better config parsing
#=============

import getopt, ipaddress, os, sys, time

now = time.strftime('%Y-%m-%d_%H:%M:%S')

def helpy():
	print()
	print('Usage:')
	print('\thomenet-graph.py --log [log-path] --config [config-path] [--dot | --circo | --neato] [--help]')
	print()
	print('\t-l / --log\t\tpath of the Zeek (Bro) conn.log to be visualized')
	print('\t-g / --config\t\tpath of the homenets.cfg (line-separated file containing your network\'s CIDR ranges, one-per-line')
	print('\t-o / --output\t\toutput location (NOT filename) for the final image')
	print('\t-d / --dot\t\toutputs image in dot format')
	print('\t-c / --circo\t\toutputs image in circo format')
	print('\t-n / --neato\t\toutputs image in neato format')
	print()
	print('Examples:')
	print('\thomenetGrapher.py --config /etc/opt/homenetGrapher/homenets.cfg --log /path-to-bro/conn.log --output /var/log --dot --circo --neato')
	print('\thomenetGrapher.py -g /etc/opt/homenetGrapher/homenets.cfg -l /path-to-bro/conn.log -o /var/log --dot --circo --neato')
	print()

try:
	opts, args = getopt.getopt(sys.argv[1:], "hcdng:l:o:", ["help", "circo", "dot", "neato", "config=", "log=", "output="])
except Exception as e:
	print(str(e))
	helpy()
	sys.exit(2)

makeimg = False
circ = False
dott = False
neat = False
try:
	for opt, arg in opts:
		if opt in ["-h", "--help"]:
			helpy()
			sys.exit(1)
		if opt in ["-g", "--config"]:
			configLocation = arg
		if opt in ["-o", "--output"]:
			outputPath = str(arg).rstrip('/')
			dotFile = str(outputPath+'/homenet-graph-{}-DOTFILE.dot'.format(now))
		if opt in ["-c", "--circo"]:
			circoImage = str(outputPath+'/homenet-graph-{}-circo.png'.format(now))
			circ = True
			makeimg = True
		if opt in ["-d", "--dot"]:
			dotImage = str(outputPath+'/homenet-graph-{}-dot.png'.format(now))
			dott = True
			makeimg = True
		if opt in ["-n", "--neato"]:
			neatoImage = str(outputPath+'/homenet-graph-{}-neato.png'.format(now))
			neat = True
			makeimg = True
		if opt in ["-l", "--log"]:
			logLocation = str(arg).rstrip('/')
except Exception as e:
	print(str(e))
	helpy()
	sys.exit(1)

with open(configLocation, mode='r', encoding="utf-8") as configfile:
	homenets = []
	exclude = []
	for line in configfile:
		if not line.startswith('#'):
			if len(line) >= 7:
				if not line.startswith('EXCLUDE'):
					homenets.append(line.rstrip('\n'))
				else:
					exclude.append(line.split()[1].rstrip('\n'))
	configfile.close()

def checkIps(s, d):
	for cidr in homenets:
		if s not in exclude and d not in exclude:
			if ipaddress.ip_address(s) in ipaddress.ip_network(cidr):
				for cidr in homenets:
					if ipaddress.ip_address(d) in ipaddress.ip_network(cidr):
						return True

connections = []
with open(logLocation, mode="r", encoding="utf-8") as logfile:
	c = 0
	for line in logfile:
		if not line.startswith('#'):
			l = line.split('\t')
			ts = l[0]
			sip = l[2]
			dip = l[4]
			spt = l[3] # if icmp, this is type
			dpt = l[5] # if icmp, this is code
			pro = l[6]
			srv = l[7]
			sta = l[11]
#			his = l[15]
			if checkIps(sip, dip):
				color = 'black'
				if pro == 'tcp':
					color = 'blue'
					if srv == 'ssl':
						color = 'cyan'
					if srv in ['ftp', 'ftp-data', 'ssh', 'scp', 'telnet', 'ms-wbt-server', 'tftp', 'ni-ftp', 'sftp', 'bftp', 'subntbcst_tftp', 'mftp', 'ftp-agent', 'pftp', 'ftps-data', 'ftps', 'tftp-mcast', 'etftp', 'utsftp', 'aaftp', 'gsiftp', 'odette-ftp', 'odette-ftps', 'tftps', 'kftp', 'kftp-data', 'mcftp', 'netconf-ssh', 'sdo-ssh', 'ssh-mgmt', 'rtelnet', 'telnets', 'skytelnet', 'hp-3000-telnet', 'tl1-telnet', 'telnetcpcd', 'scpi-telnet', 'ktelnet', 'rcp']:
						color = 'red'
				elif pro == 'udp':
					color = 'orange'
				elif pro == 'icmp':
					color = 'purple'
					dpt = '{}:{}'.format(spt, dpt)
				connections.append('"{}" -> "{}" [label="dpt:{}/{}/{} {}", color="{}"]'.format(sip, dip, dpt, pro, srv, sta, color))
	logfile.close()

if makeimg:
	print('Making {}'.format(dotFile))
	with open(dotFile, 'w') as o:
		o.write('digraph output {\nnode[shape = Mrecord];\nfontsize=16;\nnodesep=1.5;\nranksep=1;\nrankdir=LR;\n')
		connections = list(set(connections))
		for c in connections:
			o.write(c+';\n')
		o.write('\n}')
	o.close()
	print('Making output images...  these may take a minute to render.')
	try:
		if dott:
			print('Making {}'.format(dotImage))
			os.popen('dot -Tpng {} -o {}'.format(dotFile, dotImage))
		if circ:
			print('Making {}'.format(circoImage))
			os.popen('circo -Tpng {} -o {}'.format(dotFile, circoImage))
		if neat:
			print('Making {}'.format(neatoImage))
			os.popen('neato -Goverlap=scale -Tpng {} -o {}'.format(dotFile, neatoImage))
	except Exception as e:
		print(str(e))
		print('ERROR:  "dot", "circo", and/or "neato" not found via path variable - is GraphViz installed on this system?')
		sys.exit(5)
	# comment the remove line to preserve the dot-formatted text file used to generate the images via GraphViz
	# time.sleep(0.5) # this gives dot/circo/neato time to ingest the config before removing it
	# os.remove(dotFile)
else:
	print()
	print('You didn\'t specify a format for the output graph.')
	helpy()
	sys.exit(1)


import torch
import torch.nn as nn
import os
import time
import numpy as np
import random
import torch.nn.functional as F
import re
from pdb import set_trace

def gather_features(features, local_rank, world_size):
    features_list = [torch.zeros_like(features) for _ in range(world_size)]
    torch.distributed.all_gather(features_list, features)
    features_list[local_rank] = features
    features = torch.cat(features_list)
    return features

def sync_weights(model, except_key_words):
    state_dict = model.state_dict()
    for key, item  in state_dict.items():
        flag_sync = True
        for key_word in except_key_words:
            if key_word in key:
                flag_sync = False
                break

        if flag_sync:
            torch.distributed.broadcast(item, 0)

    model.load_state_dict(state_dict)
    return

# loss
def pair_cosine_similarity(x, eps=1e-8):
    n = x.norm(p=2, dim=1, keepdim=True)
    return (x @ x.t()) / (n * n.t()).clamp(min=eps)

def nt_xent(x, t=0.5):
    # print("device of x is {}".format(x.device))

    x = pair_cosine_similarity(x)
    x = torch.exp(x / t)
    idx = torch.arange(x.size()[0])
    # Put positive pairs on the diagonal
    idx[::2] += 1
    idx[1::2] -= 1
    x = x[idx]
    # subtract the similarity of 1 from the numerator
    x = x.diag() / (x.sum(0) - torch.exp(torch.tensor(1 / t)))

    return -torch.log(x).mean()

def normalize_fn(tensor, mean, std):
    """Differentiable version of torchvision.functional.normalize"""
    # here we assume the color channel is in at dim=1
    mean = mean[None, :, None, None]
    std = std[None, :, None, None]
    return tensor.sub(mean).div(std)

class NormalizeByChannelMeanStd(nn.Module):
    def __init__(self, mean, std):
        super(NormalizeByChannelMeanStd, self).__init__()
        if not isinstance(mean, torch.Tensor):
            mean = torch.tensor(mean)
        if not isinstance(std, torch.Tensor):
            std = torch.tensor(std)
        self.register_buffer("mean", mean)
        self.register_buffer("std", std)

    def forward(self, tensor):
        return normalize_fn(tensor, self.mean, self.std)

    def extra_repr(self):
        return 'mean={}, std={}'.format(self.mean, self.std)

def pgd_attack(model, images, labels, device, eps=8. / 255., alpha=2. / 255., iters=20):
    images = images.to(device)
    labels = labels.to(device)
    loss = nn.CrossEntropyLoss()

    # init
    delta = torch.rand_like(images) * eps * 2 - eps
    delta = torch.nn.Parameter(delta)

    for i in range(iters):
        outputs = model.eval()(images + delta)

        model.zero_grad()
        cost = loss(outputs, labels).to(device)
        cost.backward()

        delta.data = delta.data + alpha * delta.grad.sign()
        delta.grad = None
        delta.data = torch.clamp(delta.data, min=-eps, max=eps)
        delta.data = torch.clamp(images + delta.data, min=0, max=1) - images

    return images + delta

def eval_adv_test(model, device, test_loader, epsilon, alpha, criterion, log, attack_iter=40):
    batch_time = AverageMeter()
    losses = AverageMeter()
    top1 = AverageMeter()

    # fix random seed for testing
    torch.manual_seed(1)

    model.eval()
    end = time.time()
    for i, (input, target) in enumerate(test_loader):
        input, target = input.to(device), target.to(device)
        input_adv = pgd_attack(model, input, target, device, eps=epsilon, iters=attack_iter, alpha=alpha).data

        # compute output
        output = model.eval()(input_adv)
        loss = criterion(output, target)

        # measure accuracy and record loss
        prec1, = accuracy(output.data, target, topk=(1,))
        top1.update(prec1, input.size(0))
        losses.update(loss.item(), input.size(0))
        batch_time.update(time.time() - end)
        end = time.time()

        if (i % 10 == 0) or (i == len(test_loader) - 1):
            log.info(
                'Test: [{}/{}]\t'
                'Time: {batch_time.val:.4f}({batch_time.avg:.4f})\t'
                'Loss: {loss.val:.3f}({loss.avg:.3f})\t'
                'Prec@1: {top1.val:.3f}({top1.avg:.3f})\t'.format(
                    i, len(test_loader), batch_time=batch_time,
                    loss=losses, top1=top1
                )
            )

    log.info(' * Adv Prec@1 {top1.avg:.3f}'.format(top1=top1))
    return top1.avg

def setup_seed(seed):
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
    np.random.seed(seed)
    random.seed(seed)

    torch.backends.cudnn.benchmark = False
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.enabled = True

def accuracy(output, target, topk=(1,)):
    """Computes the precision@k for the specified values of k"""
    maxk = max(topk)
    batch_size = target.size(0)

    _, pred = output.topk(maxk, 1, True, True)
    pred = pred.t()
    correct = pred.eq(target.view(1, -1).expand_as(pred))

    res = []
    for k in topk:
        correct_k = correct[:k].reshape(-1).float().sum(0)
        res.append(correct_k.mul_(100.0 / batch_size))
    return res

class AverageMeter(object):
    """Computes and stores the average and current value"""
    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count

class logger(object):
    def __init__(self, path, log_name="log.txt", local_rank=0):
        self.path = path
        self.local_rank = local_rank
        self.log_name = log_name

    def info(self, msg):
        if self.local_rank == 0:
            print(msg)
            with open(os.path.join(self.path, self.log_name), 'a') as f:
                f.write(msg + "\n")

def fix_bn(model, fixto):
    if fixto == 'nothing':
        # fix none
        # fix previous three layers
        pass
    elif fixto == 'layer1':
        # fix the first layer
        for name, m in model.named_modules():
            if not ("layer2" in name or "layer3" in name or "layer4" in name or "fc" in name):
                m.eval()
    elif fixto == 'layer2':
        # fix the previous two layers
        for name, m in model.named_modules():
            if not ("layer3" in name or "layer4" in name or "fc" in name):
                m.eval()
    elif fixto == 'layer3':
        # fix every layer except fc
        # fix previous four layers
        for name, m in model.named_modules():
            if not ("layer4" in name or "fc" in name):
                m.eval()
    elif fixto == 'layer4':
        # fix every layer except fc
        # fix previous four layers
        for name, m in model.named_modules():
            if not ("fc" in name):
                m.eval()
    else:
        assert False

def change_batchnorm_momentum(module, value):
  if isinstance(module, torch.nn.modules.batchnorm._BatchNorm):
    module.momentum = value
  for name, child in module.named_children():
    change_batchnorm_momentum(child, value)

def get_negative_mask_to_another_branch(batch_size):
    negative_mask = torch.ones((batch_size, batch_size), dtype=bool)
    for i in range(batch_size):
        negative_mask[i, i] = 0

    return negative_mask

def nt_xent_only_compare_to_another_branch(x1, x2, t=0.5):
    out1 = F.normalize(x1, dim=-1)
    out2 = F.normalize(x2, dim=-1)
    d = out1.size()
    batch_size = d[0]

    neg = torch.exp(torch.mm(out1, out2.t().contiguous()) / t)

    mask = get_negative_mask_to_another_branch(batch_size).cuda()
    neg = neg.masked_select(mask).view(batch_size, -1)

    # pos score
    pos = torch.exp(torch.sum(out1 * out2, dim=-1) / t)

    # estimator g()
    Ng = neg.sum(dim=-1)

    # contrastive loss
    loss = (- torch.log(pos / (pos + Ng)))
    return loss.mean()

def nt_xent_compare_to_queue(out1, out2, queue, t=0.5, sampleWiseLoss=False):

    d = out1.size()
    batch_size = d[0]

    neg = torch.exp(torch.mm(out1, queue.clone().detach()) / t)

    # pos score
    pos = torch.exp(torch.sum(out1 * out2, dim=-1) / t)

    # estimator g()
    Ng = neg.sum(dim=-1)

    # contrastive loss
    loss = (- torch.log(pos / (pos + Ng)))

    if sampleWiseLoss:
        return loss
    else:
        return loss.mean()

def gatherFeatures(features, local_rank, world_size):
    features_list = [torch.zeros_like(features) for _ in range(world_size)]
    torch.distributed.all_gather(features_list, features)
    features_list[local_rank] = features
    features = torch.cat(features_list)
    return features

# loss
def pair_cosine_similarity(x, eps=1e-8):
    n = x.norm(p=2, dim=1, keepdim=True)
    return (x @ x.t()) / (n * n.t()).clamp(min=eps)

def nt_xent(x, t=0.5, sampleWiseLoss=False, return_prob=False):
    # print("device of x is {}".format(x.device))

    x = pair_cosine_similarity(x)
    x = torch.exp(x / t)
    idx = torch.arange(x.size()[0])
    # Put positive pairs on the diagonal
    idx[::2] += 1
    idx[1::2] -= 1
    x = x[idx]
    # subtract the similarity of 1 from the numerator
    x = x.diag() / (x.sum(0) - torch.exp(torch.tensor(1 / t)))

    if return_prob:
        return x.reshape(len(x) // 2, 2).mean(-1)

    sample_loss = -torch.log(x)

    if sampleWiseLoss:
        return sample_loss.reshape(len(sample_loss) // 2, 2).mean(-1)

    return sample_loss.mean()

def nt_xent_weak_compare(x, t=0.5, features2=None, easy_mining=0.9):
    if features2 is None:
        out = F.normalize(x, dim=-1)
        d = out.size()
        batch_size = d[0] // 2
        out = out.view(batch_size, 2, -1).contiguous()
        out_1 = out[:, 0]
        out_2 = out[:, 1]
    else:
        batch_size = x.shape[0]
        out_1 = F.normalize(x, dim=-1)
        out_2 = F.normalize(features2, dim=-1)

    # neg score
    out = torch.cat([out_1, out_2], dim=0)
    neg = torch.exp(torch.mm(out, out.t().contiguous()) / t)

    mask = get_negative_mask(batch_size).cuda()
    neg = neg.masked_select(mask).view(2 * batch_size, -1)

    total_neg_num = neg.shape[1]
    hard_sample_num = max(int((1 - easy_mining) * total_neg_num), 1)
    score = neg
    threshold = score.kthvalue(hard_sample_num, dim=1, keepdim=True)[0]
    neg = (score <= threshold) * neg
    Ng = neg.sum(dim=-1)

    # pos score
    pos = torch.exp(torch.sum(out_1 * out_2, dim=-1) / t)
    pos = torch.cat([pos, pos], dim=0)

    # contrastive loss
    loss = (- torch.log(pos / (pos + Ng)))

    return loss.mean()

def focal_loss(prob, gamma):
    """Computes the focal loss"""
    loss = (1 - prob) ** gamma * (-torch.log(prob))
    return loss.mean()

def fix_focal_loss(prob, fix_prob, gamma):
    """Computes the focal loss"""
    loss = (1 - fix_prob) ** gamma * (-torch.log(prob))
    return loss.mean()

class FocalLoss(nn.Module):
    def __init__(self, gamma=0.):
        super(FocalLoss, self).__init__()
        assert gamma >= 0
        self.gamma = gamma
        # self.weight = weight

    def forward(self, prob):
        return focal_loss(prob, self.gamma)

class DistillCrossEntropy(nn.Module):
    def __init__(self, T):
        super(DistillCrossEntropy, self).__init__()
        self.T = T
        return

    def forward(self, inputs, target):
        """
        :param inputs: prediction logits
        :param target: target logits
        :return: loss
        """
        log_likelihood = - F.log_softmax(inputs / self.T, dim=1)
        sample_num, class_num = target.shape
        loss = torch.sum(torch.mul(log_likelihood, torch.softmax(target / self.T, dim=1)))/sample_num

        return loss

class FocalLoss_fix(nn.Module):
    def __init__(self, gamma=0., fix_probs=None):
        super(FocalLoss_fix, self).__init__()
        '''
        fix_prob: FloatTensor, the prob for each sample, it follows the order of index
        '''

        assert gamma >= 0
        self.gamma = gamma
        assert fix_probs is not None
        self.fix_probs = fix_probs
        # self.weight = weight

    def forward(self, prob, idxs):
        fix_prob = self.fix_probs[idxs]
        return fix_focal_loss(prob, fix_prob, self.gamma)

def get_negative_mask(batch_size):
    negative_mask = torch.ones((batch_size, 2 * batch_size), dtype=bool)
    for i in range(batch_size):
        negative_mask[i, i] = 0
        negative_mask[i, i + batch_size] = 0

    negative_mask = torch.cat((negative_mask, negative_mask), 0)
    return negative_mask

def nt_xent_instance_large(x, t=0.5, return_porbs=False):
    out = F.normalize(x, dim=-1)
    d = out.size()
    batch_size = d[0] // 2
    out = out.view(batch_size, 2, -1).contiguous()
    out_1 = out[:, 0]
    out_2 = out[:, 1]
    out = torch.cat([out_1, out_2], dim=0)

    # doesn't give gradient
    losses = []
    probs = []

    with torch.no_grad():
        for cnt in range(batch_size):
            # pos score
            pos = torch.exp(torch.sum(out_1[cnt] * out_2[cnt]) / t)
            pos = torch.stack([pos, pos], dim=0)

            Ng1 = torch.exp((out_1[cnt].unsqueeze(0) * out).sum(1) / t).sum() - torch.exp(torch.Tensor([1 / t,])).to(out.device)
            Ng2 = torch.exp((out_2[cnt].unsqueeze(0) * out).sum(1) / t).sum() - torch.exp(torch.Tensor([1 / t,])).to(out.device)
            Ng = torch.cat([Ng1, Ng2], dim=0)

            if not return_porbs:
                # contrastive loss
                losses.append(- torch.log(pos / Ng).mean())
            else:
                # contrastive loss
                probs.append((pos / Ng).mean())

    if not return_porbs:
        losses = torch.stack(losses)
        return losses
    else:
        probs = torch.stack(probs)
        return probs

def nt_xent_inter_batch_multiple_time(x, t=0.5, batch_size=512, repeat_time=10, return_porbs=False):
    out = F.normalize(x, dim=-1)
    d = out.size()
    dataset_len = d[0] // 2
    dataset_features = out.view(dataset_len, 2, -1).contiguous()

    # doesn't give gradient
    losses_all = []

    with torch.no_grad():
        for cnt in range(repeat_time):
            losses_batch = []
            # order features
            random_order = torch.randperm(dataset_len, device=out.device)
            order_back = torch.argsort(random_order)
            dataset_features_1 = dataset_features[:, 0]
            dataset_features_2 = dataset_features[:, 1]
            # dataset_features_1 = dataset_features_1[random_order]
            # dataset_features_2 = dataset_features_2[random_order]

            # get the loss
            assert dataset_len >= batch_size
            for i in range(int(np.ceil(dataset_len / batch_size))):
                if (i + 1) * batch_size < dataset_len:
                    samplingIdx = random_order[i * batch_size: (i + 1) * batch_size]
                    offset = 0
                else:
                    samplingIdx = random_order[dataset_len - batch_size:]
                    offset = i * batch_size - (dataset_len - batch_size)

                # calculate loss
                out1 = dataset_features_1[samplingIdx]
                out2 = dataset_features_2[samplingIdx]

                out = torch.stack([out1, out2], dim=1).view((batch_size * 2, -1))
                losses_or_probs = nt_xent(out, t=t, sampleWiseLoss=True, return_prob=return_porbs)[offset:]
                losses_batch.append(losses_or_probs)

            # reset the order
            losses_batch = torch.cat(losses_batch, dim=0)
            losses_batch = losses_batch[order_back]
            losses_all.append(losses_batch)

        # average togather
        losses_all = torch.stack(losses_all).mean(0)

        return losses_all

def nt_xent_debiased(x, t=0.5, tau_plus=0.5, debiased=False, weightIns=None, distanceWeightingMode=False, sampleWiseLoss=False, features2=None, returnProb=False):
    if features2 is None:
        out = F.normalize(x, dim=-1)
        d = out.size()
        batch_size = d[0] // 2
        out = out.view(batch_size, 2, -1).contiguous()
        out_1 = out[:, 0]
        out_2 = out[:, 1]
    else:
        batch_size = x.shape[0]
        out_1 = F.normalize(x, dim=-1)
        out_2 = F.normalize(features2, dim=-1)

    # neg score
    out = torch.cat([out_1, out_2], dim=0)
    # print("temperature is {}".format(t))
    neg = torch.exp(torch.mm(out, out.t().contiguous()) / t)

    if returnProb:
        assert not distanceWeightingMode

    if not distanceWeightingMode:
        mask = get_negative_mask(batch_size).cuda()
        neg = neg.masked_select(mask).view(2 * batch_size, -1)

        # pos score
        pos = torch.exp(torch.sum(out_1 * out_2, dim=-1) / t)
        pos = torch.cat([pos, pos], dim=0)

        # estimator g()
        if debiased:
            N = batch_size * 2 - 2
            Ng = (-tau_plus * N * pos + neg.sum(dim=-1)) / (1 - tau_plus)
            # constrain (optional)
            Ng = torch.clamp(Ng, min=N * np.e ** (-1 / t))
        else:
            Ng = neg.sum(dim=-1)

        # contrastive loss
        if returnProb:
            return pos / (pos + Ng)

        loss = (- torch.log(pos / (pos + Ng)))

        assert weightIns is None
        if sampleWiseLoss:
            return (loss[batch_size:] + loss[:batch_size]) / 2
        else:
            return loss.mean()
    else:
        assert not sampleWiseLoss

        mask = get_negative_mask(batch_size).cuda()
        neg = neg * mask
        negWeight = torch.cat([weightIns, weightIns], dim=0).unsqueeze(0) * mask
        negWeightNormalized = negWeight / negWeight.sum(0, keepdim=True) * mask.sum(0, keepdim=True)
        neg = neg * negWeightNormalized

        # pos score
        pos = torch.exp(torch.sum(out_1 * out_2, dim=-1) / t)
        pos = torch.cat([pos, pos], dim=0)

        # estimator g()
        assert not debiased
        Ng = neg.sum(dim=-1)

        # contrastive loss
        loss = (- torch.log(pos / (pos + Ng)))

        return loss.mean()

def nt_xent_sigmoid_focal(x, t, gamma, features2=None):
    if features2 is None:
        out = F.normalize(x, dim=-1)
        d = out.size()
        batch_size = d[0] // 2
        out = out.view(batch_size, 2, -1).contiguous()
        out_1 = out[:, 0]
        out_2 = out[:, 1]
    else:
        batch_size = x.shape[0]
        out_1 = F.normalize(x, dim=-1)
        out_2 = F.normalize(features2, dim=-1)

    # neg score
    out = torch.cat([out_1, out_2], dim=0)
    neg = torch.mm(out, out.t().contiguous()) / t

    mask = get_negative_mask(batch_size).cuda()
    neg = neg.masked_select(mask)
    # pos score
    pos = torch.sum(out_1 * out_2, dim=-1) / t
    pos = torch.cat([pos, pos], dim=0)

    # print("pos is {}, neg is {}".format(pos, neg))

    # get sigmoid prob
    p_pos = torch.sigmoid(pos)
    p_neg = torch.sigmoid(-neg)
    p_all = torch.cat([p_pos, p_neg])

    # print("after sigmoid, pos is {}, neg is {}".format(p_pos, p_neg))

    # contrastive loss
    loss = - ((1 - p_all) ** gamma * torch.log(p_all)).sum() / (batch_size * 2)

    return loss

def getStatisticsFromTxt(txtName, num_class=1000):
      statistics = [0 for _ in range(num_class)]
      with open(txtName, 'r') as f:
        lines = f.readlines()
      for line in lines:
            s = re.search(r" ([0-9]+)$", line)
            if s is not None:
              statistics[int(s[1])] += 1
      return statistics

def getAllClassNamesFromTxt(txtName):
  names = []
  with open(txtName, 'r') as f:
    lines = f.readlines()
  for line in lines:
    s = re.search(r"train/(n[0-9]+)/n[0-9]+_[0-9]+\.JPEG ([0-9]+)", line)
    if (s is not None):
      names.append(str(s[1]))

  names = np.unique(names).tolist()
  return names

def getAllClassNamesFromTxtPlaces(txtName):
  names = []
  with open(txtName, 'r') as f:
    lines = f.readlines()
  for line in lines:
    s = re.search(r"data_256/([a-zA-Z/]+)/[0-9]+\.jpg ([0-9]+)", line)
    if (s is not None):
      names.append(str(s[1]))

  names = np.unique(names).tolist()
  return names

def gather_tensor(tensor, local_rank, world_size):
    # gather features
    tensor_list = [torch.zeros_like(tensor) for _ in range(world_size)]
    torch.distributed.all_gather(tensor_list, tensor)
    tensor_list[local_rank] = tensor
    tensors = torch.cat(tensor_list)
    return tensors

def reOrderData(idxs, labels, features):
    # sort all losses and idxes
    labels_new = []
    features_new = []
    idxs_new = []

    # reorder
    for idx, label, feature in zip(idxs, labels, features):
        order = np.argsort(idx)
        idxs_new.append(idx[order])
        labels_new.append(label[order])
        features_new.append(feature[order])

    # check if equal
    for cnt in range(len(idxs_new) - 1):
        if not np.array_equal(idxs_new[cnt], idxs_new[cnt+1]):
            raise ValueError("idx for {} and {} should be the same".format(cnt, cnt+1))

    return idxs_new, labels_new, features_new

def cosine_annealing(step, total_steps, lr_max, lr_min, warmup_steps=0):
    assert warmup_steps >= 0

    if step < warmup_steps:
        lr = lr_max * step / warmup_steps
    else:
        lr = lr_min + (lr_max - lr_min) * 0.5 * (1 + np.cos((step - warmup_steps) / (total_steps - warmup_steps) * np.pi))

    return lr

def get_CUB200_data_split(root, customSplit):
    if os.path.isdir(root):
        root = root
    else:
        if os.path.isdir("../../data/CUB_200_2011/images"):
            root = "../../data/CUB_200_2011/images"
        elif os.path.isdir("/mnt/models/dataset/CUB_200_2011/images"):
            root = "/mnt/models/dataset/CUB_200_2011/images"
        else:
            assert False

    txt_train = "split/CUB_200/train_split.txt"
    txt_val = "split/CUB_200/val_split.txt"
    txt_test = "split/CUB_200/test_split.txt"

    if customSplit != '':
        txt_train = "split/CUB_200/{}.txt".format(customSplit)

    return root, txt_train, txt_val, txt_test

def remove_state_dict_module(state_dict):
    # rename pre-trained keys
    for k in list(state_dict.keys()):
        # retain only encoder_q up to before the embedding layer
        if k.startswith('module.'):
            # remove prefix
            state_dict[k.replace("module.", "")] = state_dict[k]
            # delete renamed or unused k
            del state_dict[k]

    return state_dict

def fix_backbone(model, log):
    # fix every layer except fc
    # fix previous four layers
    log.info("fix backbone")
    for name, param in model.named_parameters():
        if not ("fc" in name):
            param.requires_grad = False

    for name, m in model.named_modules():
        if not ("fc" in name):
            m.eval()

def fix_agent(model, log):
    # fix every layer except fc
    # fix previous four layers
    log.info("fix agent")
    for name, param in model.named_parameters():
        param.requires_grad = False

    for name, m in model.named_modules():
        m.eval()

def modify_model_weight(model, random_prune_ratio):
    for module in model.modules():
        if isinstance(module, nn.Conv2d):
            module.weight.data.copy_(module.weight.data * (1 - random_prune_ratio))

if __name__ == "__main__":
    a = torch.rand([8, 128], device="cuda")

    loss_ori = nt_xent_debiased(a, t=0.5, sampleWiseLoss=True)
    # loss = nt_xent_inter_batch_multiple_time(a, t=0.5, batch_size=4, repeat_time=5)
    loss = nt_xent(a, t=0.5, sampleWiseLoss=True)

    print("loss_ori is {}, loss new is {}".format(loss_ori, loss))


import pywikibot, math
from datetime import datetime

MIN_YEAR = -625
MAX_YEAR = int(datetime.now().year)+5

BIRTH_PAGE_PART = "قالب:ناس تزادو ف"
DEATH_PAGE_PART = "قالب:ناس توفاو ف"
EVENT_PAGE_PART = "قالب:حوايج وقعو ف"
EXPCT_PAGE_PART = "قالب:أحدات مبرمجة ؤلا متوقعة ف"
BC = " قبل لميلاد"
BC_REDUCED = "ق.م"
BH_REDUCED = "ق.هـ"
BCH_REDUCED = "ق.ش"

YEAR_NAV_PTRN = "{{Year nav|{raw_year}}}"
CENT_TEMPLATE_TAG = "{{C{century} year in topic}}"

YEAR = "[[عام]]"
LEAP_YEAR = "[[عام مكبس]]"

CENTURY = "قرن "
DECADE_PART = "عوام "

EN_ARY_DAYS = {"Sunday":"لحد"
              ,"Monday":"تنين"
              ,"Tuesday":"تلات"
              ,"Wednesday":"لاربع"
              ,"Thursday":"لخميس"
              ,"Friday":"جمعة"
              ,"Saturday":"سبت"
              }

ROMAN_PART = "، ؤ {روماني} ف [[تقويم روماني|تّقويم رّوماني]]"

SOURCE = "<ref>{{Cite web|url=https://www.timeanddate.com/calendar/?year={عام}&country=106|title={عام}|language=en}}</ref>"

list_page_parts = [BIRTH_PAGE_PART, DEATH_PAGE_PART, EVENT_PAGE_PART, EXPCT_PAGE_PART]

YEAR_PART = "(عام)"

MAIN_TEXT = """{{Year nav|yy}}
{{C{cent} year in topic}}
'''{عام}''' هوّا {نوع} ف [[تقويم ڭريڭوري|تّقويم لڭريڭوري]] [[عوام ميلاديين بداو نهار {نهار1}|بدا نهار {نهار1}]] ؤ {مسالية} نهار {نهار2}. هوّا لعام نمرة {عام} ف [[إيرا عامة|لإيرا لعامة]] ؤ لفترة د [[بعد لميلاد]]، نمرة {فلقرن} ف [[لقرن {قرن}]]، ؤ نمرة {فلعقد} ف ل[[عقد]] ديال [[عوام {بدية}]].{source}

{عام} كيوافق {إسلامي} ف [[تقويم إسلامي|تّقويم لهيجري]] ؤ {أمازيغي} ف [[تقويم أمازيغي|تّقويم لأمازيغي]] ؤ {هولوسيني} ف [[تقويم هولوسيني|تّقويم لهولوسيني]].

{{ناس تزادو ف {عام}}}

{{ناس توفاو ف {عام}}}

{{حوايج وقعو ف {عام}}}

{{قالب:أحدات مبرمجة ؤلا متوقعة ف {موستقبال}}}

== عيون لكلام ==
{{عيون}}
{{ضبط مخازني}}

[[تصنيف:{عام}]]
[[تصنيف:عوام د تقويم لميلادي]]
[[تصنيف:مقالات زادهوم داريجابوت]]
"""

MAIN_TEXT_BC = """{{Year nav|yy}}
{{CX year in topic}}
'''{عام}''' هوّا عام ف [[تقويم يولياني|تّقويم ليولياني]] ف لفترة د [[قبل لميلاد]]، نمرة {فلقرن} ف [[لقرن {قرن}]]، ؤ نمرة {فلعقد} ف ل[[عقد]] ديال [[عوام {بدية}]].<ref>{{Cite web|url=https://keisan.casio.com/exec/system/1247132711|title=Day of Week Calculator|language=en}}</ref>

{عام} كيوافق {إسلامي} ف [[تقويم إسلامي|تّقويم لهيجري]] ؤ {أمازيغي} ف [[تقويم أمازيغي|تّقويم لأمازيغي]]، ؤ {روماني} ف [[تقويم روماني|تّقويم رّوماني]]، ؤ {هولوسيني} ف [[تقويم هولوسيني|تّقويم لهولوسيني]].

{{ناس تزادو ف {عام}}}

{{ناس توفاو ف {عام}}}

{{حوايج وقعو ف {عام}}}

== عيون لكلام ==
{{عيون}}
{{ضبط مخازني}}

[[تصنيف:{عام}]]
[[تصنيف:عوام د تقويم لميلادي]]
[[تصنيف:مقالات زادهوم داريجابوت]]"""

TMP_PAGE_SAVE_MESSAGE = "پاج ديال لقالب تقادّات خاوية"

PAGE_SAVE_MESSAGE = "پاج د لعام تقادّات"

def get_year_text(year):
    if year > 0:
        return str(year)
    else:
        return str(abs(year))+BC

def is_year_templates_available(site, year):

    year_text = get_year_text(year)
    for page_part in list_page_parts:
        title = page_part+" "+year_text
        page = pywikibot.Page(site,title)
        #print(page.title())
        if page.text != "":
            #print(page.title())
            return True

    return False

"""
def get_islamic_year_range(year):
    islamicMult = 1.030684 # the factor to multiply by
    islamicSub = 621.5643 # the factor to subtract by
    if (year - 621) > 0:
	#year1 = math.floor(islamicMult*(year-islamicSub ))
	year1 = math.floor(islamicMult*(year-islamicSub))
	#year2 = math.floor( islamicMult * ( year - islamicSub + 1 ) )
	year2 = math.floor(islamicMult*(year-islamicSub+1))
	return str(year1)+'-'+str(year2)
    else:
	year1 = math.ceil( -islamicMult * ( year - islamicSub ) )
	year2 = math.ceil( -islamicMult * ( year - islamicSub + 1 ) )
	return str(year1)+" "+BH_REDUCED+'-'+str(year2)+" "+BH_REDUCED
"""
def get_islamic_year_range(year):
    mult = 1.030684
    sub = 621.5643
    if year > 621:
        year1 = math.floor(mult * (year - sub))
        year2 = math.floor(mult * (year - sub + 1))
        return str(year1)+'-'+str(year2)
    else:
        year1 = math.ceil(-mult * (year - sub))
        year2 = math.ceil(-mult * (year - sub + 1))
        return str(year1)+" "+BH_REDUCED+'-'+str(year2)+" "+BH_REDUCED

def get_year_type(year):
    if year < 0:
        return YEAR
    elif year % 400 == 0:
        return LEAP_YEAR
    elif year % 100 == 0:
        return YEAR
    elif year % 4 == 0:
        return LEAP_YEAR

    return YEAR

def get_start_end_of_year_day_name(year):
    start_year = datetime.strptime('Jan 1 '+str(year).rjust(4,'0'), '%b %d %Y')
    end_year = datetime.strptime('Dec 31 '+str(year).rjust(4,'0'), '%b %d %Y')

    start, end = start_year.strftime("%A"),end_year.strftime("%A")

    return EN_ARY_DAYS[start],EN_ARY_DAYS[end]

def get_number_in_century(year):
    if year > 0:
        r = year%100
        return (lambda r:100 if r == 0 else r)(r)
    else:
        return 10 - (year + 1)%10

def get_number_in_decade(year):
    """
    Returns the year number within the decade.
    """
    if year > 9:
        return year%10+1
    elif year > -10:
        return year%10
    else:
        if year%10 == 0:
            return 10
        else:
            return year%10

def get_decade(year):
    decade_number = abs(year)//10*10
    if year > 0:
        return str(decade_number)
    else:
        return str(decade_number)+BC

def get_century(year):
    century_number = abs(year)//100+1
    return (lambda year:str(century_number) if year > 0 else str(century_number)+BC)(year)

def get_amazigh_year(year):
    K = 950
    if year > 0:
        return year + K
    elif year < - K:
        return str(year+K+1)+" "+BCH_REDUCED
    else:
        return year + K + 1

def get_holocene_year(year):
    K = 10000
    if year > 0:
        return year + K
    else:
        return year + K + 1

def get_roman_year(year):
    K = 753
    if year > 0:
        return year + K
    elif year < - K:
        return None
    else:
        return year + K + 1

def get_full_text(year):
    if year > 0:
        text = MAIN_TEXT.replace("yy",str(year))
        year_text = str(year)
        start, end = get_start_end_of_year_day_name(year)
        if year >= datetime.now().year:
            end_verb = "غادي يسالي"
            text = text.replace("{موستقبال}",year_text)
        else:
            end_verb = "سالا"
            text = text.replace("{{قالب:أحدات مبرمجة ؤلا متوقعة ف {موستقبال}}}","")

        century = int(get_century(year))
        if century > 17:
            cent = str(century)
        else:
            cent = "X"
        text = text.replace("{source}",SOURCE).replace("{عام}",year_text).replace("{نوع}",get_year_type(year)).replace("{نهار1}",start).replace("{مسالية}",end_verb).replace("{نهار2}",end) \
               .replace("{فلقرن}",str(get_number_in_century(year))).replace("{قرن}",str(get_century(year))).replace("{فلعقد}",str(get_number_in_decade(year))) \
               .replace("{بدية}",str(get_decade(year))).replace("{إسلامي}",get_islamic_year_range(year)).replace("{أمازيغي}",str(get_amazigh_year(year))) \
               .replace("{هولوسيني}",str(get_holocene_year(year))).replace("{cent}",cent)
    else:
        text = MAIN_TEXT_BC.replace("yy",str(year))
        year_text = str(abs(year))+BC
        text = text.replace("{عام}",year_text).replace("{نوع}",get_year_type(year)).replace("{فلقرن}",str(get_number_in_century(year))) \
               .replace("{قرن}",str(get_century(year))).replace("{فلعقد}",str(get_number_in_decade(year))) \
               .replace("{بدية}",str(get_decade(year))).replace("{إسلامي}",get_islamic_year_range(year)).replace("{أمازيغي}",str(get_amazigh_year(year))) \
               .replace("{هولوسيني}",str(get_holocene_year(year)))
        roman_year = get_roman_year(year)
        print(roman_year)
        if roman_year is None:
            text = text.replace(ROMAN_PART,"")
        else:
            text = text.replace("{روماني}",str(roman_year))

    return text

def run_for_years_list(years, site):

    for year in years:
        print(year)
        if is_year_templates_available(site, year):
            year_text = get_year_text(year)
            print(year_text)
            if year < 32 and year > 0:
                year_text2 = year_text+" "+YEAR_PART
                page = pywikibot.Page(site,year_text2)
            else:
                page = pywikibot.Page(site,year_text)
            if page.text == "":

                for page_part in list_page_parts[:-1]:
                    title = page_part+" "+year_text
                    tmp_page = pywikibot.Page(site,title)
                    if tmp_page.text == "":
                        tmp_page.text = ""
                        tmp_page.save(TMP_PAGE_SAVE_MESSAGE)

                page.text = get_full_text(year).strip().replace("\n\n\n","\n")
                page.save(PAGE_SAVE_MESSAGE)

if __name__ == '__main__':
    years = list(range(1,MAX_YEAR+1))

    site = pywikibot.Site()

    run_for_years_list(years, site)

    years = list(range(MIN_YEAR,0))
    #print(years)

    run_for_years_list(years, site)

    #"""

Task 23 - Add year pages/Wikipedia/add_year_pages.py


from typing import List
class Solution:
    def productExceptSelf(self, nums: List[int]) -> List[int]:
        left = [1]
        right = [1]
        l = len(nums)
        result = []
        for i in range(0,l-1):
            left.append(left[i]*nums[i])
            right.insert(0,right[0]*nums[l-1-i])
        for i in range(0,l):
            result.append(left[i] * right[i])
        return result


"""

convert_waveform_txt_to_parquet.py

Simple file for converting the .txt wavefile to the standard root format.

"""
import sipmanalyze.formats as forms
import argparse
import os
import awkward

if __name__ == "__main__":
  parser = argparse.ArgumentParser(
    "Converting a waveform file to a standard root file format.")
  parser.add_argument('input', type=str, nargs='+', help='input .txt file')
  args = parser.parse_args()
  for idx, in_f in enumerate(args.input):
    try:
      out_f = in_f.replace('.txt', '.root')
      print(f'Converting file {in_f} [{idx+1}/{len(args.input)}]')
      std_cont = forms.standard.standard_container.from_txt(in_f)
      std_cont.data['lumival'] = std_cont.data.payload[:, 0]
      std_cont.data['uncval'] = std_cont.data.payload[:, 1]
      std_cont.data = awkward.zip(
        {f: std_cont.data[f]
         for f in std_cont.data.fields
         if f != 'payload'})

      std_cont.save_to_file(out_f)
      del std_cont
    except Exception as err:
      print(err)
      pass