#
# Copyright (c) 2019-2020, NVIDIA CORPORATION.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import sys

import numpy as np
import rmm

from xbb_tools.utils import (
    benchmark,
    tpcxbb_argparser,
    train_clustering_model,
    run_dask_cudf_query,
)
from xbb_tools.readers import build_reader
from dask import delayed

try:
    cli_args = tpcxbb_argparser()
except:
    cli_args = {}

# q25 parameters
Q25_DATE = "2002-01-02"
N_CLUSTERS = 8
CLUSTER_ITERATIONS = 20
N_ITER = 5

@benchmark(dask_profile=cli_args.get("dask_profile"),)
def read_tables():
    table_reader = build_reader(basepath=cli_args["data_dir"])

    ss_cols = ["ss_customer_sk", "ss_sold_date_sk", "ss_ticket_number", "ss_net_paid"]
    ws_cols = [
        "ws_bill_customer_sk",
        "ws_sold_date_sk",
        "ws_order_number",
        "ws_net_paid",
    ]
    datedim_cols = ["d_date_sk", "d_date"]

    ss_ddf = table_reader.read("store_sales", relevant_cols=ss_cols, index=False)
    ws_ddf = table_reader.read("web_sales", relevant_cols=ws_cols, index=False)
    datedim_ddf = table_reader.read("date_dim", relevant_cols=datedim_cols, index=False)

    return (ss_ddf, ws_ddf, datedim_ddf)

def agg_count_distinct(df, group_key, counted_key, client):
    """Returns a Series that is the result of counting distinct instances of 'counted_key' within each 'group_key'.
    The series' index will have one entry per unique 'group_key' value.
    Workaround for lack of nunique aggregate function on Dask df.
    """

    ### going via repartition for split_out drop duplicates
    ### see issue: https://github.com/rapidsai/tpcx-bb-internal/issues/492
    unique_df = df[[group_key, counted_key]].map_partitions(
        lambda df: df.drop_duplicates()
    )
    unique_df = unique_df.repartition(columns=[group_key])
    unique_df = unique_df.map_partitions(lambda df: df.drop_duplicates())

    return unique_df.groupby(group_key)[counted_key].count(split_every=2)

def convert_datestring_to_days(df, date_col="d_date", date_format="%Y-%m-%d"):
    """
    Utility to convert datestring to int representing days
    """
    datetime_array = rmm.device_array(len(df), dtype=np.int64)
    df[date_col].str.timestamp2int(
        format=date_format, units="D", devptr=datetime_array.device_ctypes_pointer.value
    )
    df[date_col] = datetime_array
    return df

@benchmark(dask_profile=cli_args.get("dask_profile"))
def get_clusters(client, ml_input_df):
    ml_tasks = [
        delayed(train_clustering_model)(df, N_CLUSTERS, CLUSTER_ITERATIONS, N_ITER)
        for df in ml_input_df.to_delayed()
    ]
    results_dict = client.compute(*ml_tasks, sync=True)

    output = ml_input_df.index.to_frame().reset_index(drop=True)

    labels_final = dask_cudf.from_cudf(
        results_dict["cid_labels"], npartitions=output.npartitions
    )
    output["label"] = labels_final.reset_index()[0]

    # Based on CDH6.1 q25-result formatting
    results_dict["cid_labels"] = output
    return results_dict

@benchmark(dask_profile=cli_args.get("dask_profile"))
def main(client):
    ss_ddf, ws_ddf, datedim_ddf = read_tables()
    datedim_ddf = datedim_ddf.map_partitions(convert_datestring_to_days)
    min_date = np.datetime64(Q25_DATE, "D").astype(int)
    # Filter by date
    valid_dates_ddf = datedim_ddf[datedim_ddf["d_date"] > min_date].reset_index(
        drop=True
    )

    f_ss_ddf = ss_ddf[ss_ddf["ss_customer_sk"].notnull()].reset_index(drop=True)
    f_ws_ddf = ws_ddf[ws_ddf["ws_bill_customer_sk"].notnull()].reset_index(drop=True)

    # Merge
    ss_merged_df = f_ss_ddf.merge(
        valid_dates_ddf, left_on="ss_sold_date_sk", right_on="d_date_sk", how="inner"
    )
    ws_merged_df = f_ws_ddf.merge(
        valid_dates_ddf, left_on="ws_sold_date_sk", right_on="d_date_sk", how="inner"
    )

    # Roll up store sales
    agg_store_sales_ddf = ss_merged_df.groupby("ss_customer_sk").agg(
        {"ss_sold_date_sk": "max", "ss_net_paid": "sum"}
    )

    agg_store_sales_ddf["frequency"] = agg_count_distinct(
        ss_merged_df, "ss_customer_sk", "ss_ticket_number", client=client
    )  # Simulate count distinct

    # Same rollup, just different columns for web sales
    agg_web_sales_ddf = ws_merged_df.groupby("ws_bill_customer_sk").agg(
        {"ws_sold_date_sk": "max", "ws_net_paid": "sum"}
    )

    agg_web_sales_ddf["frequency"] = agg_count_distinct(
        ws_merged_df, "ws_bill_customer_sk", "ws_order_number", client=client
    )  # Simulate count distinct

    agg_store_sales_ddf = agg_store_sales_ddf.reset_index()
    agg_web_sales_ddf = agg_web_sales_ddf.reset_index()

    shared_columns = ["cid", "most_recent_date", "amount", "frequency"]
    agg_store_sales_ddf.columns = shared_columns
    agg_web_sales_ddf.columns = shared_columns
    agg_sales_ddf = dask_cudf.concat([agg_store_sales_ddf, agg_web_sales_ddf])

    cluster_input_ddf = agg_sales_ddf.groupby("cid").agg(
        {"most_recent_date": "max", "frequency": "sum", "amount": "sum"}
    )

    cluster_input_ddf["recency"] = (37621 - cluster_input_ddf["most_recent_date"]) < 60

    # Reorder to match refererence examples
    cluster_input_ddf = cluster_input_ddf[["recency", "frequency", "amount"]]

    # Prepare df for KMeans clustering
    cluster_input_ddf["recency"] = cluster_input_ddf["recency"].astype("int64")
    cluster_input_ddf["amount"] = cluster_input_ddf["amount"].astype("float64")

    cluster_input_ddf = cluster_input_ddf.persist()

    results_dict = get_clusters(client=client, ml_input_df=cluster_input_ddf)
    return results_dict

if __name__ == "__main__":
    from xbb_tools.cluster_startup import attach_to_cluster
    import cudf
    import dask_cudf

    client = attach_to_cluster(cli_args)

    run_dask_cudf_query(cli_args=cli_args, client=client, query_func=main)


__author__ = 'Teddy'
import numpy as np
import cv2
import argparse
from PIL import Image

BASE_DIR = "trainData/"
FILE_NAME = "/1.png"

class KNearestClass:
    def kNearTrain(self):
        #The following line is modified for OpenCV 3.0
        trainData, labels = self.getTrainingData()
        knn = cv2.ml.KNearest_create()
        #knn.train(trainData,labels)
        #print(trainData.shape, labels.shape)
        knn.train(trainData, cv2.ml.ROW_SAMPLE, labels)
        return knn

    def getTrainingData(self):
        trainData = np.array([cv2.imread(BASE_DIR + str(0) + FILE_NAME).flatten()])
        labels = np.array([0])
        for x in range(1,10):
            trainData = np.vstack([trainData, (cv2.imread(BASE_DIR + str(x) + FILE_NAME).flatten())])
            labels = np.vstack([labels, [x]])
        return trainData.astype(np.float32), labels

    def solve(self, cv2im):
        input = np.array([cv2im.flatten()]).astype(np.float32)
        #print(input.shape)
        res = self.model.findNearest(input,1)
        return res

    def __init__(self):
        self.model = self.kNearTrain()

if __name__ == "__main__":
    model = KNearestClass()
    im = cv2.imread("2.png")
    res = model.solve(im)
    print(res[0])


'''
版本更新内容：
1.0：   根据Otsu原理 自己写代码 实现Otsu算法（写有大量注释）
1.2：   将自己实现的Otsu算法 封装在myotsu方法中,并且删除大量注释和多于代码
2.0:    对比开闭运算处理的结果，选取开运算消除噪声点
3.0：   用学过的方法实现提取分割区域的轮廓（写有大量注释）
3.2：   删除了大量注释，保留了最精简的代码
3.contours.1:   单独 写一个计算contours的算法
3.contours.2:   单独 写一个计算contours的算法,优化边界问题
3.contours2.0： 代码部分优化 精简+注释(写有大量注释 并包含试错代码)
3.contours2.9： 封装my_contours，并 精简注释删除大部分无用代码
3.contours3.0： 准备提取的轮廓彩色化---在my_contours中添加代码获取每个轮廓点的X【。。。】Y【。。。】坐标列表  range从0开始技术的；要重新调for循环
3.contours3.9： R化的轮廓 在原图上进行标注

'''

import cv2
from matplotlib import pyplot as plt
import numpy as np
from PIL import Image, ImageStat
# from __future__ import division
import __future__

from numpy.lib import npyio

def cv_show(img,name):
    cv2.imshow(name,img)
    cv2.waitKey()
    cv2.destroyAllWindows()

def myotsu(imge):
    #? ↓my_otsu算法部分
    rows,cols=np.shape(imge)#!统计出行数列数
    n0=0#前景像素个数
    n=rows*cols#总像素数
    n1=n-n0#后景像素格式

    #? 二维数组求和
    s=sum(map(sum,imge))#图片总像素值
    u=s/n#图片平均像素值

    mythreshold=0
    my_g=0#存放方差最终结果
    my_th=0#存放阈值最终结果

    #? ↓ for循环部分：求前后景平均像素值 等参数，并计算出最佳阈值
    for i in range(256):

        np.where(imge<mythreshold)#返回坐标，二维图像返回两个数组 一个数组包含x坐标，另一个数组包含y坐标
        n0=np.where(imge<=mythreshold)[0].shape[0]#前景像素点个数    [<=]
        n1=np.where(imge>mythreshold)[0].shape[0]#后景像素点个数     [>]
        '''
        where[0]包含符合条件的所有x坐标，where[1]是y坐标，【0】【1】元素个数一样
        shape永远是[0] ，只有一个值 保存在shape[0] 保存的是复合条件的点的个数
        '''
        if(n0==0 or n1==0): #‘分母’为0的话 直接跳出for，不然最后会得到nan值
            mythreshold +=1
            continue

        w0 = n0/n#前景所占比例
        w1 = n1/n#后景所占比例

        re0, thr0 = cv2.threshold(imge, mythreshold, 255, cv2.THRESH_TOZERO_INV)#前景图片
        #! THRESH_TOZERO_INV  像素值大于阈值时，设置为0，否则不改变
        s0=sum(map(sum,thr0))#前景图片总像素值
        u0=s0/n0#前景图片平均像素值

        re1, thr1 = cv2.threshold(imge, mythreshold, 255, cv2.THRESH_TOZERO)#后景图片
        #! THRESH_TOZERO 像素值大于阈值时，不变，否则取0
        s1=sum(map(sum,thr1))#背景图片总像素值
        u1=s1/n1#背景图片平均像素值

        g=w0*w1*(u0*u0-2*u0*u1+u1*u1)#综合计算 前景u0和背景u1的方差g，代码是g公式化简后的结果

        if g>my_g:
            my_g=g
            my_th=mythreshold

        mythreshold +=1
    print("最优阈值："+str(my_th))

    my_ret, my_th = cv2.threshold(img, my_th, 255, cv2.THRESH_BINARY)

    #? ↑my_otsu算法部分
    return my_ret,my_th

def my_contours(img):
    imgx=img.copy()
    imgy=img.copy()

    m,n =imgx.shape
    # n = int(3)
    # j=int(3)
    #! 遍历行/列，遇见的第一个0保留，后面连续的0全部变成255，直到遇见255再把前一个变成255的0还原成0.
    for i in range(m):  #行
        # print(str(i)+"...\t...\t...\t")
        for j in range(n):  #列
            if(imgx[i][j]==0 and j!=n-1):      #? 开头少一条黑线。仅仅海洋地方少一条黑线
                # print("\t:"+str(j))
                # print("\t:"+str(img[i-1][j-1-1]))
                # print("\t:"+str(img[i-1][j-1]))
                # print("\t:"+str(img[i-1][j-1+1]))
                while(imgx[i][j+1] == 0 and j+1!=n-1):
                    imgx[i][j+1] = 255
                    # print("\t\t-"+str(j))
                    j+=1
                if(j!=n-1):             #! 解决了，一句话搞定~~解决了 末尾双重黑问题
                    imgx[i][j] = 0  #?这里出问题，最后双重黑
                if(j+1==n-1 and imgx[i][j+1] == 0):
                    imgx[i][j] = 255    #! 改了三天bug 把第一根线改回来了 并且把 最后根线处理好了（还是最初的‘笨’方法）
                    break
                    # print("\t\t-"+str(j))
                    # print("\t\t-"+str(img[i-1][j-1-1]))
                    # print("\t\t-"+str(img[i-1][j-1]))
                    # print("\t\t-"+str(img[i-1][j-1+1]))
            # print(j)
    # print(img)
    # cv_show(imgx,'img_x')
    # img_x=imgx
    #! ↑X方向完成了

    m,n =imgy.shape
    for j in range(n):  #行
        # print(str(j)+"...\t...\t...\t")
        for i in range(m):  #列
            if(imgy[i][j]==0 and i!=m-1):     #! 天撸了 and i!=m-1 这个也要加上啊，倒是第二个白色 最后单独一个是黑色的情况呀！！！
                # print("\t:"+str(i))
                while(imgy[i+1][j] == 0 and i+1!=m-1):
                    imgy[i+1][j] = 255
                    # print("\t\t-"+str(i))
                    i+=1
                if(i!=m-1):
                    imgy[i][j] = 0
                if(i+1==m-1 and imgy[i+1][j] == 0):
                    imgy[i][j] = 255
                    break
    # print(img)
    # cv_show(imgy,'img_y')
    # img_y=imgy
    #! ↑y方向完成了

    # img_xy = img_x+img_y    #! 两图合并
    img_xy = imgx+imgy
    # cv_show(img_xy,'done_ing')
    # print(img_xy)

    #! ↓合并后的图像处理一下像素值 并保存contours的坐标列表
    m,n =img_xy.shape
    c_x = []    # 保存contours的x坐标
    c_y = []    # 保存contours的y坐标
    for j in range(n):  #行
        for i in range(m):  #列
            if(img_xy[i][j]==254):
                img_xy[i][j] = 255
            else:
                img_xy[i][j] =0
                c_x.append(i)
                c_y.append(j)
    # print(img)
    # cv_show(img_xy,'img_xy')
    # print(c_x)
    # print(c_y)

    return img_xy,imgx,imgy,c_x,c_y

#? Q1 打开一幅图片
img = cv2.imread('./img/whale.png', 0)

#? Q2 自己写一个otsu算法 做阈值分割
# 我的otsu 阈值法
ret1, th1 = myotsu(img)
print("MyOtsu 得到的阈值：\t"+str(ret1))
'''
ret1, th1 = cv2.threshold(img, 100, 255, cv2.THRESH_BINARY)
cv2.threshold (src, thresh, maxval, type)
cv2.threshold (源图片, 阈值, 填充色, 阈值类型)
'''

# Otsu阈值法
ret2, th2 = cv2.threshold(img, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
'''
ret1, th1 = cv2.threshold(img, 100, 255, cv2.THRESH_BINARY)
cv2.threshold (src, thresh, maxval, type)
cv2.threshold (源图片, 阈值, 填充色, 阈值类型)
'''

images = [img, 0, th1, img, 0, th2]
titles = ['Original', 'Histogram', 'my_Otsu:'+str(ret1),
         'Original', 'Histogram', "Otsu's"+str(ret2),]

for i in range(2):
    # 绘制原图
    plt.subplot(2, 3, i * 3 + 1)
    plt.imshow(images[i * 3], 'gray')
    plt.title(titles[i * 3], fontsize=8)
    plt.xticks([]), plt.yticks([])

    # 绘制直方图plt.hist, ravel函数将数组降成一维
    plt.subplot(2, 3, i * 3 + 2)
    plt.hist(images[i * 3].ravel(), 256)
    plt.title(titles[i * 3 + 1], fontsize=8)
    plt.xticks([]), plt.yticks([])

    # 绘制阈值图
    plt.subplot(2, 3, i * 3 + 3)
    plt.imshow(images[i * 3 + 2], 'gray')
    plt.title(titles[i * 3 + 2], fontsize=8)
    plt.xticks([]), plt.yticks([])
plt.show()

#? Q3 用形态学开/闭运算处理消除噪声点
#! 开运算与闭运算
kernel = np.ones((5,5),np.uint8)
o_img = th1.copy()# 开：先腐蚀，再膨胀
opening = cv2.morphologyEx(o_img, cv2.MORPH_OPEN, kernel)

c_img = th1.copy()# 闭：先膨胀，再腐蚀
closing = cv2.morphologyEx(c_img, cv2.MORPH_CLOSE, kernel)

plt.figure()
plt.subplot(2,2,1)		# 将画板分为2行两列，本幅图位于第一个位置
plt.imshow(img,cmap="gray")
plt.axis('off')
plt.subplot(2,2,2)		# 将画板分为2行两列，本幅图位于第二个位置
plt.imshow(th1,cmap="gray")
plt.axis('off')
plt.subplot(2,2,3)		# 将画板分为2行两列，本幅图位于第3个位置
plt.imshow(opening,cmap="gray")
plt.axis('off')
plt.subplot(2,2,4)		# 将画板分为2行两列，本幅图位于第3个位置
plt.imshow(closing,cmap="gray")
plt.axis('off')
plt.show()

#? Q4 自己写计算物理轮廓的算法 来提取分割区域的轮廓
# img = cv2.imread('./img/whale_otsued.png',0)
img_xy,img_x,img_y,c_x,c_y=my_contours(opening)

plt.figure()
plt.subplot(2,2,1)
plt.imshow(img_x,cmap="gray")
plt.axis('off')
plt.subplot(2,2,2)
plt.imshow(img_y,cmap="gray")
plt.axis('off')
plt.subplot(2,2,3)
plt.imshow(img_xy,cmap="gray")
plt.axis('off')
plt.subplot(2,2,4)
plt.imshow(cv2.imread('./img/whale_otsued.png',0),cmap="gray")
plt.axis('off')
plt.show()

#!  --方法对比---    用sobel算子 筛选轮廓
img = cv2.imread('./img/whale_otsued.png',0)    #分开计算再相加
sobelx = cv2.Sobel(img,cv2.CV_64F,1,0,ksize=1)
sobelx = cv2.convertScaleAbs(sobelx)    #取绝对值
# cv_show(sobelx,'sobelx')
sobely = cv2.Sobel(img,cv2.CV_64F,0,1,ksize=1)
sobely = cv2.convertScaleAbs(sobely)
# cv_show(sobely,'sobely')
# 分别计算x和y，再求和
sobelxy = cv2.addWeighted(sobelx,0.5,sobely,0.5,0)
# cv_show(sobelxy,'sobelxy')
#! 反色处理一下
sobelxy = cv2.bitwise_not(sobelxy)#反色

#?   --方法对比---    使用findcontours和drawcontours在白色画布上画出轮廓
img = cv2.imread('./img/whale_otsued.png',0)
contours,hierarchy = cv2.findContours(img, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)
'''
cv2.findContours(img,mode,method)
mode:轮廓检索模式
RETR_EXTERNAL ：只检索最外面的轮廓；
RETR_LIST：检索所有的轮廓，并将其保存到一条链表当中；
RETR_CCOMP：检索所有的轮廓，并将他们组织为两层：顶层是各部分的外部边界，第二层是空洞的边界;
RETR_TREE：检索所有的轮廓，并重构嵌套轮廓的整个层次;        [最常用]
method:轮廓逼近方法
CHAIN_APPROX_NONE：以Freeman链码的方式输出轮廓，所有其他方法输出多边形（顶点的序列）。
CHAIN_APPROX_SIMPLE:压缩水平的、垂直的和斜的部分，也就是，函数只保留他们的终点部分。
'''
# draw_img = img.copy()
m,n=img.shape
#! 使用Numpy创建一张图片大小的白纸纸
img_255 = np.zeros((m,n,3), np.uint8)
#! 使用白色填充图片区域,默认为黑色
img_255.fill(255)

img_fd = cv2.drawContours(img_255, contours, -1, (0,0,0), 1)
#! drawContours画轮廓，img原图，contours轮廓是啥，
#! -1 画第几个轮廓?-1是全都画，(0, 0, 255)（BGR），2 线条宽度
# cv_show(img_fd,'img_fd')

#? --方法对比---    完全使用现有类库处理
# 为了更高的准确率，使用二值图像。
img = cv2.imread('./img/whale.png') #! 第一步 读数据
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)    #! 第二步 把数据转换成灰度图
ret, thresh = cv2.threshold(gray, 127, 255, cv2.THRESH_BINARY)  #! 第三步 把图像数据进行二值处理 （通过图像阈值 0 1）
contours,hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)
'''
传入一个二值处理后的图像
RETR_TREE：检索所有的轮廓，并重构嵌套轮廓的整个层次
以Freeman链码的方式输出轮廓，所有其他方法输出多边形（顶点的序列）
#? binary, contours, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)
#? ValueError: not enough values to unpack (expected 3, got 2)
#? 原因是用的是Opencv4.0，把返回值从三个改回两个了
第一个值binary就是刚才的二值图
#np.array(contours).shape
第二个值contours 保存的是轮廓信息，是list结构 需要用np.array转换一下
hierarchy是一个层级，结果保存在层级当中
返回值:
　　contours:一个列表，每一项都是一个轮廓， 不会存储轮廓所有的点，只存储能描述轮廓的点
　　hierarchy:一个ndarray, 元素数量和轮廓数量一样
'''
m,n=thresh.shape    #? 这里不能用 img.shape？？！！
img_255 = np.zeros((m,n,3), np.uint8)
img_255.fill(255)
img_fdo = cv2.drawContours(img_255, contours, -1, (0, 0, 0), 1) #! 绘制轮廓
'''
传入绘制图像，轮廓，轮廓索引，颜色模式，线条厚度
# 注意需要copy,要不原图会变。。。
drawContours画轮廓，draw_img原图，contours轮廓是啥，
-1 画第几个轮廓?-1是全都画，(0, 0, 255)（BGR），2 线条宽度
'''
# cv_show(img_fdo,'res')

plt.figure()
plt.subplot(2,3,1)
plt.imshow(cv2.imread('./img/whale.png',0),cmap="gray") #原图
plt.axis('off')
plt.subplot(2,3,2)
plt.imshow(cv2.imread('./img/whale_otsued.png',0),cmap="gray")  #自己写的otsu算法
plt.axis('off')
plt.subplot(2,3,3)
plt.imshow(img_xy,cmap="gray")  #自己写的轮廓算法
plt.axis('off')
plt.subplot(2,3,4)
plt.imshow(sobelxy,cmap="gray") #sobel算法
plt.axis('off')
plt.subplot(2,3,5)
plt.imshow(img_fd,cmap="gray")  #findcontours drawcontours
plt.axis('off')
plt.subplot(2,3,6)
plt.imshow(img_fdo,cmap="gray") #调用otsu 调用findcontours drawcontours
plt.axis('off')
plt.show()

#? Q5 提取到的轮廓 叠加在原图上进行显示

#? Q5A1 兜兜转转 最初的思路最简单最有效：
#! （用myfind的XY坐标列表，直接修改原图像素点）
img = img_xy.copy()
imgin1 = cv2.imread('./img/whale.png')[:,:,::-1] #? [:,:,::-1]
#![:,:,::-1] 【第一个通道，第二个通道，第三个通道】第三个通道-1意思是RGB变成BGR

sx,sy= img.shape
l =len(c_x)

x = c_x    # 保存contours的x坐标
y = c_y    # 保存contours的y坐标
for i in range(l):
    imgin1[x[i],y[i],0]=255
    imgin1[x[i],y[i],1]=0
    imgin1[x[i],y[i],2]=0

# plt.imshow(imgin1)
# plt.axis('off')
# plt.show()

#? Q5A2 最丑，可能是因为我xy列表格式和plot不容易匹配
#! 获取 轮廓x[...],y[...]坐标 用 pyplot画图画出来
plt.figure()

sx,sy= img.shape
l =len(c_x)

x = c_x    # 保存contours的x坐标
y = c_y    # 保存contours的y坐标
for i in range(l):  #行
    plt.scatter(y[i],x[i]) #画点    xy位置需要换一下
plt.show()
# imgin2 = cv2.imread('./img/whale.png')[:,:,::-1] #? [:,:,::-1]
# plt.figure()
# for i in range(l):  #行
#     plt.plot(y[0],x[0], 'red', linewidth=2) #画点
# # for n, contour in enumerate(contours2):
# # plt.plot(contour[:, 1], contour[:, 0], 'yellow', linewidth=2)

# plt.imshow(imgin2)
# plt.axis('off')

# for i in range(l):  #行
#     plt.plot(y[i],x[i], 'red', linewidth=2) #画点
# plt.show()

#? Q5A3 图片融合 按比例显示 会变模糊
#! 把得到的轮廓图 线条 变成彩色
img_out= img_xy.copy()
# img_out = (img+1)*255
# cv_show(img_out,'out')    #可以显示
img_out2 = np.concatenate([img_out[:,:,None],img_out[:,:,None],img_out[:,:,None]] , axis=2)
'''
None表示该维不进行切片，而是将该维整体作为数组元素处理
'''
print(img_out.shape)
# cv_show(img_out,'out')    #可以显示

img_out3 = np.concatenate([255*np.ones_like(img_out[:,:,None]),img_out[:,:,None],img_out[:,:,None]] , axis=2)   #? ones_like查一下再写注释
'''
ones_like返回一个用1填充的跟输入 形状和类型 一致的数组。
zeros_like返回一个用0填充的跟输入数组 形状和类型一样的数组
'''
#cv_show(img_out,'out')
#? 不能显示，好像是通道的问题 不能用CV调用，只能用plt显示
'''
plt.figure()
plt.subplot(1,3,1)
plt.imshow(img,cmap="gray") #传进来的轮廓图
plt.axis('off')
plt.subplot(1,3,2)
plt.imshow(img_out2,cmap="gray")  #单通道转换成 三通道
plt.axis('off')
plt.subplot(1,3,3)
plt.imshow(img_out3,cmap="gray")  #数学计算一下 拼出 红色通道
plt.axis('off')
plt.show()
'''

#? 叠加在原图上进行显示
#? Q5A3_sj:
img_in = cv2.imread('./img/whale.png')
b,g,r = cv2.split(img_in)
img_in = cv2.merge([r,g,b])
factor = 0.6
img_bl = np.asarray(factor*img_in + (1-factor)*img_out3, np.uint8)
# plt.imshow(img_bl)
# plt.axis('off')
# plt.show()

#? Q5A3_chang ：
from PIL import Image
import matplotlib.pyplot as plt
img_out = img_out3
img_out = Image.fromarray(np.uint8(img_out3))   #! 需要有这句话转换一下格式，不然后面的convert不能执行
img_out = img_out.convert('RGBA')
'''
b,g,r = cv2.split(img_out3)
cur_img = img.copy()
'''
img_in = cv2.imread('./img/whale.png')
img_in = Image.fromarray(np.uint8(img_in))
img_in = img_in.convert('RGBA')
img_blend = Image.blend(img_out,img_in,0.7)
'''
image1:第一张图片
image2:第二张图片，必须具有与第一张图片相同的模式和尺寸。
alpha：内插alpha因子。如果alpha为0.0，则返回第一张图像的副本。
    如果alpha为1.0，则返回第二张图像的副本。
    alpha值没有限制。如有必要，将结果裁剪以适合允许的输出范围。
# '''
# plt.imshow(img_blend)
# plt.axis('off')

# plt.show()

plt.figure()
plt.subplot(3,3,1)
plt.imshow(imgin1)
plt.axis('off')
plt.subplot(3,3,4)
plt.imshow(img,cmap="gray") #传进来的轮廓图
plt.axis('off')
plt.subplot(3,3,5)
plt.imshow(img_out2,cmap="gray")  #单通道转换成 三通道
plt.axis('off')
plt.subplot(3,3,6)
plt.imshow(img_out3,cmap="gray")  #数学计算一下 拼出 红色通道
plt.axis('off')
plt.subplot(3,3,7)
plt.imshow(img_bl)
plt.axis('off')
plt.subplot(3,3,8)
plt.imshow(img_blend)
plt.axis('off')
plt.show()


from flaky import flaky

import deepchem as dc
from deepchem.models.tensorgraph.layers import Reshape, Variable, SoftMax, GRU
import numpy as np
import tensorflow as tf
import unittest

class TestA3C(unittest.TestCase):

  @flaky
  def test_roulette(self):
    """Test training a policy for the roulette environment."""

    # This is modeled after the Roulette-v0 environment from OpenAI Gym.
    # The player can bet on any number from 0 to 36, or walk away (which ends the
    # game).  The average reward for any bet is slightly negative, so the best
    # strategy is to walk away.

    class RouletteEnvironment(dc.rl.Environment):

      def __init__(self):
        super(RouletteEnvironment, self).__init__([(1,)], 38)
        self._state = [np.array([0])]

      def step(self, action):
        if action == 37:
          self._terminated = True  # Walk away.
          return 0.0
        wheel = np.random.randint(37)
        if wheel == 0:
          if action == 0:
            return 35.0
          return -1.0
        if action != 0 and wheel % 2 == action % 2:
          return 1.0
        return -1.0

      def reset(self):
        self._terminated = False

    env = RouletteEnvironment()

    # This policy just learns a constant probability for each action, and a constant for the value.

    class TestPolicy(dc.rl.Policy):

      def create_layers(self, state, **kwargs):
        action = Variable(np.ones(env.n_actions))
        output = SoftMax(
            in_layers=[Reshape(in_layers=[action], shape=(-1, env.n_actions))])
        value = Variable([0.0])
        return {'action_prob': output, 'value': value}

    # Optimize it.

    a3c = dc.rl.A3C(
        env,
        TestPolicy(),
        max_rollout_length=20,
        optimizer=dc.models.tensorgraph.TFWrapper(
            tf.train.AdamOptimizer, learning_rate=0.001))
    a3c.fit(100000)

    # It should have learned that the expected value is very close to zero, and that the best
    # action is to walk away.

    action_prob, value = a3c.predict([[0]])
    assert -0.5 < value[0] < 0.5
    assert action_prob.argmax() == 37
    assert a3c.select_action([[0]], deterministic=True) == 37

    # Verify that we can create a new A3C object, reload the parameters from the first one, and
    # get the same result.

    new_a3c = dc.rl.A3C(env, TestPolicy(), model_dir=a3c._graph.model_dir)
    new_a3c.restore()
    action_prob2, value2 = new_a3c.predict([[0]])
    assert value2 == value

    # Do the same thing, only using the "restore" argument to fit().

    new_a3c = dc.rl.A3C(env, TestPolicy(), model_dir=a3c._graph.model_dir)
    new_a3c.fit(0, restore=True)
    action_prob2, value2 = new_a3c.predict([[0]])
    assert value2 == value

  def test_recurrent_states(self):
    """Test a policy that involves recurrent layers."""

    # The environment just has a constant state.

    class TestEnvironment(dc.rl.Environment):

      def __init__(self):
        super(TestEnvironment, self).__init__([(10,)], 10)
        self._state = [np.random.random(10)]

      def step(self, action):
        self._state = [np.random.random(10)]
        return 0.0

      def reset(self):
        pass

    # The policy includes a single recurrent layer.

    class TestPolicy(dc.rl.Policy):

      def create_layers(self, state, **kwargs):

        reshaped = Reshape(shape=(1, -1, 10), in_layers=state)
        gru = GRU(n_hidden=10, batch_size=1, in_layers=reshaped)
        output = SoftMax(
            in_layers=[Reshape(in_layers=[gru], shape=(-1, env.n_actions))])
        value = Variable([0.0])
        return {'action_prob': output, 'value': value}

    # We don't care about actually optimizing it, so just run a few rollouts to make
    # sure fit() doesn't crash, then check the behavior of the GRU state.

    env = TestEnvironment()
    a3c = dc.rl.A3C(env, TestPolicy())
    a3c.fit(100)
    # On the first call, the initial state should be all zeros.
    prob1, value1 = a3c.predict(
        env.state, use_saved_states=True, save_states=False)
    # It should still be zeros since we didn't save it last time.
    prob2, value2 = a3c.predict(
        env.state, use_saved_states=True, save_states=True)
    # It should be different now.
    prob3, value3 = a3c.predict(
        env.state, use_saved_states=True, save_states=False)
    # This should be the same as the previous one.
    prob4, value4 = a3c.predict(
        env.state, use_saved_states=True, save_states=False)
    # Now we reset it, so we should get the same result as initially.
    prob5, value5 = a3c.predict(
        env.state, use_saved_states=False, save_states=True)
    assert np.array_equal(prob1, prob2)
    assert np.array_equal(prob1, prob5)
    assert np.array_equal(prob3, prob4)
    assert not np.array_equal(prob2, prob3)


#!/usr/bin/env python3
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

import argparse
import json
import logging
import os
import shutil
import subprocess
import sys
import tempfile
import traceback
from pathlib import Path
from typing import Optional, Any, List

from pyre_extensions import none_throws, safe_json

try:
    from ..facebook.shim import configuration
except Exception:
    # pyre-ignore
    from . import configuration

import pyredex

LOG: logging.Logger = logging.getLogger(__name__)

class ClientError(Exception):
    pass

def _path_exists(path: str) -> str:
    path = os.path.expanduser(path)
    if not os.path.exists(path):
        raise argparse.ArgumentTypeError(f"Path `{path}` does not exist.")
    return os.path.realpath(path)

def _directory_exists(path: str) -> str:
    path = os.path.expanduser(path)
    if not os.path.isdir(path):
        raise argparse.ArgumentTypeError(f"Path `{path}` is not a directory.")
    path = os.path.realpath(path)
    if path and path[-1] != "/":
        path = path + "/"
    return path

def _separated_paths_exist(paths: Optional[str]) -> Optional[str]:
    if paths is None:
        return None

    elements = paths.split(";")
    return ";".join([_path_exists(element) for element in elements])

def _check_executable(path: Path) -> Path:
    if not (path.exists() and os.access(path, os.X_OK)):
        raise ClientError(f"Invalid binary `{path}`.")
    return path

def _system_jar_configuration_path(input: str) -> str:
    if input.endswith(".json"):
        path = _path_exists(input)
        with open(path) as file:
            try:
                paths = safe_json.load(file, List[str])
                return ";".join(paths)
            except safe_json.InvalidJson:
                raise argparse.ArgumentTypeError(
                    f"`{path} must contain a list of strings"
                )

    # Validation deferred to backend if we pass `;` separated list of paths
    # because they are allowed to not exist.
    return input

class ExtractJexException(ClientError):
    pass

def _extract_jex_file_if_exists(path: Path, target: str, build_directory: Path) -> Path:
    jex_extract_directory: Path = Path(build_directory) / "jex"

    def run_unzip_command(command: List[str]) -> Path:
        output = subprocess.run(command, stderr=subprocess.PIPE)
        if output.returncode != 0:
            stderr = output.stderr.decode()
            if (
                command[0] == "unzip"
                and output.returncode == 1
                and "extra bytes at beginning or within zipfile" in stderr
            ):
                # This warning is fine to ignore, because *.jar files have been properly extracted
                LOG.warning(f"`unzip` warning: {stderr}")
            else:
                command_string = " ".join(command)
                raise ExtractJexException(
                    f"Unable to extract binary file `{path}` with command `{command_string}`:\n{stderr}"
                )

        jar_file_path = jex_extract_directory / (
            target.rsplit(":", maxsplit=1)[1] + ".jar"
        )
        if jar_file_path.exists():
            return jar_file_path
        else:
            raise ClientError(
                f"Could not find jar file `{path}` in `{jex_extract_directory}`."
            )

    # If the target is java_binary, then the output is a JEX file
    if path.suffix != ".jex":
        return path

    # Try to extract *.jex files with various unzip tools, until one of them succeeds. See D22579374 for why doing this
    try:
        return run_unzip_command(
            ["unsquashfs", "-d", str(jex_extract_directory), "-o", "4096", str(path)]
        )
    except ExtractJexException:
        LOG.warning(f"Running `unsquashfs` on file `{path}` failed.")
        LOG.warning(f"Trying to extract file `{path}` with `unzip`.")
        return run_unzip_command(["unzip", "-d", str(jex_extract_directory), str(path)])

def _build_target(target: str, *, mode: Optional[str] = None) -> Path:
    LOG.info(f"Building `{target}`%s...", f" with `{mode}`" if mode else "")

    # If a target starts with fbcode, then it needs to be built from fbcode instead of from fbsource
    if ":" not in target:
        LOG.warning(
            f"Target `{target}` is an alias. Please expand it if it is a fbcode target. Otherwise buck build will fail."
        )
    fbcode_target_prefix = "fbcode//"
    is_fbcode_target = target.startswith(fbcode_target_prefix)
    if is_fbcode_target:
        target = target[len(fbcode_target_prefix) :]

    command = ["buck", "build", "--show-json-output"]
    if mode:
        command.append(str(mode))
    command.append(target)
    current_working_directory = Path(os.getcwd())
    working_directory = (
        current_working_directory / "fbcode"
        if is_fbcode_target
        else current_working_directory
    )
    output = subprocess.run(command, stdout=subprocess.PIPE, cwd=working_directory)
    if output.returncode != 0:
        raise ClientError(f"Error while building buck target `{target}`, aborting.")

    try:
        response = json.loads(output.stdout)
    except json.JSONDecodeError:
        response = {}

    if len(response) != 1 or len(next(iter(response.values()))) == 0:
        raise ClientError(f"Unexpected buck output:\n{output.stdout.decode()}")

    return working_directory / next(iter(response.values()))

def _build_executable_target(target: str, *, mode: Optional[str] = None) -> Path:
    return _check_executable(_build_target(target, mode=mode))

def _get_analysis_binary(arguments: argparse.Namespace) -> Path:
    from_arguments = arguments.binary
    if from_arguments:
        # Use the user-provided binary.
        return _check_executable(Path(from_arguments))

    buck_target = configuration.BINARY_BUCK_TARGET
    if buck_target:
        # Build the mariana-trench binary from buck (facebook-only).
        return _build_executable_target(
            buck_target,
            mode=arguments.build,
        )

    path_command = configuration.BINARY_PATH_COMMAND
    if path_command:
        # Find the mariana-trench binary in the path (open-source).
        command = shutil.which(path_command)
        if command is None:
            raise ClientError(f"Could not find `{path_command}` in PATH.")
        return Path(command)

    raise ClientError("Could not find the analyzer binary.")

def _desugar_jar_file(jar_path: Path) -> Path:
    LOG.info(f"Desugaring `{jar_path}`...")
    desugar_tool = _build_target(none_throws(configuration.DESUGAR_BUCK_TARGET))
    desugared_jar_file = jar_path.parent / (jar_path.stem + "-desugared.jar")
    output = subprocess.run(
        [
            "java",
            f"-Dlog4j.configurationFile={configuration.DESUGAR_LOG_CONFIGURATION_PATH}",
            "-jar",
            desugar_tool,
            os.fspath(jar_path),
            os.fspath(desugared_jar_file),
        ]
    )
    if output.returncode != 0:
        raise ClientError("Error while desugaring jar file, aborting.")

    LOG.info(f"Desugared jar file: `{desugared_jar_file}`.")
    return desugared_jar_file

def _build_apk_from_jar(jar_path: Path) -> Path:
    _, dex_file = tempfile.mkstemp(suffix=".jar")

    LOG.info(f"Running d8 on `{jar_path}`...")
    output = subprocess.run(
        [
            "/opt/android/sdk_D23134735/build-tools/29.0.2/d8",
            "-JXmx8G",
            jar_path,
            "--output",
            dex_file,
            "--lib",
            "/opt/android/sdk_D23134735/platforms/android-29/android.jar",
            "--min-api",
            "25",  # mininum api 25 corresponds to dex 37
        ]
    )
    if output.returncode != 0:
        raise ClientError("Error while running d8, aborting.")

    return Path(dex_file)

class VersionAction(argparse.Action):
    def __call__(self, parser: argparse.ArgumentParser, *_: Any) -> None:
        from . import package

        print(f"{package.name} {package.version}")
        print("Copyright (c) Facebook, Inc. and its affiliates.")
        parser.exit()

def main() -> None:
    logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
    build_directory = Path(tempfile.mkdtemp())
    try:
        parser = argparse.ArgumentParser(
            description="A security-focused static analyzer targeting Android."
        )
        parser.add_argument(
            "--version",
            action=VersionAction,
            nargs=0,
            help="Print the version and exit",
        )

        target_arguments = parser.add_argument_group("Target arguments")
        target_arguments.add_argument(
            "--apk-path",
            type=_path_exists,
            help="The APK to analyze.",
        )
        if configuration.FACEBOOK_SHIM:
            target_arguments.add_argument(
                "--java-target",
                type=str,
                help="The java buck target to analyze. If the target is `java_library`, append `-javaX` were X is the java version (e.g. 11).",
            )
            target_arguments.add_argument(
                "--java-mode",
                type=str,
                help="The buck mode for building the java target.",
            )

        output_arguments = parser.add_argument_group("Output arguments")
        output_arguments.add_argument(
            "--output-directory",
            type=_directory_exists,
            default=".",
            help="The directory to store results in.",
        )
        output_arguments.add_argument(
            "--generated-models-directory",
            type=_directory_exists,
            help="Save generated models to this directory.",
        )

        binary_arguments = parser.add_argument_group("Analysis binary arguments")
        binary_arguments.add_argument(
            "--binary", type=str, help="The Mariana Trench binary."
        )
        if configuration.FACEBOOK_SHIM:
            binary_arguments.add_argument(
                "--build",
                type=str,
                default=none_throws(configuration.BINARY_BUCK_BUILD_MODE),
                metavar="BUILD_MODE",
                help="The Mariana Trench binary buck build mode.",
            )

        configuration_arguments = parser.add_argument_group("Configuration arguments")
        configuration_arguments.add_argument(
            "--system-jar-configuration-path",
            type=_system_jar_configuration_path,
            default=os.fspath(configuration.get_path("default_system_jar_paths.json")),
            help="A JSON configuration file with a list of paths to the system jars "
            + "or a `;` separated list of jars.",
        )
        configuration_arguments.add_argument(
            "--rules-paths",
            type=str,
            default=os.fspath(configuration.get_path("rules.json")),
            help="A `;`-separated list of rules files and directories containing rules files.",
        )
        configuration_arguments.add_argument(
            "--repository-root-directory",
            type=_directory_exists,
            default=".",
            help="The root of the repository. Resulting paths will be relative to this.",
        )
        configuration_arguments.add_argument(
            "--source-root-directory",
            type=_directory_exists,
            default=".",
            help="The root where source files for the APK can be found.",
        )
        configuration_arguments.add_argument(
            "--source-exclude-directories",
            type=str,
            default=None,
            help="A `;`-separated list of directories that should be excluded from indexed source files.",
        )
        configuration_arguments.add_argument(
            "--proguard-configuration-paths",
            type=_separated_paths_exist,
            default=None,
            help="A `;`-separated list of ProGuard configurations, which can be used for global inference and to ignore unreachable objects.",
        )
        configuration_arguments.add_argument(
            "--remove-unreachable-code",
            action="store_true",
            help="Prune unreachable code based on entry points specified in proguard configuration.",
        )
        configuration_arguments.add_argument(
            "--lifecycles-paths",
            type=_separated_paths_exist,
            default=None,
            help="A `;`-separated list of files and directories containing lifecycle definitions.",
        )
        configuration_arguments.add_argument(
            "--model-generator-configuration-paths",
            type=_separated_paths_exist,
            default=os.fspath(configuration.get_path("default_generator_config.json")),
            help="""A `;`-separated list of paths specifying JSON configuration files. Each file is a list of paths
            to JSON model generators relative to the configuration file or names of CPP model generators.""",
        )
        configuration_arguments.add_argument(
            "--model-generator-search-paths",
            type=_separated_paths_exist,
            default=";".join(
                os.fspath(path)
                for path in configuration.get_default_generator_search_paths()
            ),
            help="A `;`-separated list of paths where we look up JSON model generators.",
        )
        configuration_arguments.add_argument(
            "--maximum-source-sink-distance",
            type=int,
            default=configuration.DEFAULT_MAXIMUM_SOURCE_SINK_DISTANCE,
            help="Limits the distance of sources and sinks from a trace entry point.",
        )

        analysis_arguments = parser.add_argument_group("Analysis arguments")
        analysis_arguments.add_argument(
            "--sequential",
            action="store_true",
            help="Run the analysis sequentially, one a single thread.",
        )
        analysis_arguments.add_argument(
            "--skip-source-indexing",
            action="store_true",
            help="Skip indexing source files.",
        )
        analysis_arguments.add_argument(
            "--skip-model-generation",
            action="store_true",
            help="Skip model generation.",
        )
        analysis_arguments.add_argument(
            "--disable-parameter-type-overrides",
            action="store_true",
            help="Disable analyzing methods with specific parameter type information.",
        )
        analysis_arguments.add_argument(
            "--maximum-method-analysis-time",
            type=int,
            help="Specify number of seconds as a bound. If the analysis of a method takes longer than this then make the method obscure (default taint-in-taint-out).",
        )

        debug_arguments = parser.add_argument_group("Debugging arguments")
        debug_arguments.add_argument(
            "--verbosity",
            type=int,
            default=1,
            metavar="[1-5]",
            help="The logging verbosity.",
        )
        debug_arguments.add_argument(
            "--gdb", action="store_true", help="Run the analyzer inside gdb."
        )
        debug_arguments.add_argument(
            "--lldb", action="store_true", help="Run the analyzer inside lldb."
        )
        debug_arguments.add_argument(
            "--log-method",
            action="append",
            metavar="PATTERN",
            help="Enable logging for the given methods.",
        )
        debug_arguments.add_argument(
            "--dump-class-hierarchies",
            action="store_true",
            help="Dump the class hierarchies in `class_hierarchies.json`.",
        )
        debug_arguments.add_argument(
            "--dump-overrides",
            action="store_true",
            help="Dump the override graph in `overrides.json`.",
        )
        debug_arguments.add_argument(
            "--dump-call-graph",
            action="store_true",
            help="Dump the call graph in `call_graph.json`.",
        )
        debug_arguments.add_argument(
            "--dump-dependencies",
            action="store_true",
            help="Dump the dependency graph in `dependencies.json`.",
        )
        debug_arguments.add_argument(
            "--dump-methods",
            action="store_true",
            help="Dump a list of the method signatures in `methods.json`.",
        )

        arguments: argparse.Namespace = parser.parse_args()

        if (
            configuration.FACEBOOK_SHIM
            and arguments.java_target is not None
            and arguments.apk_path is not None
        ):
            parser.error(
                "The analysis target can either be a java target (--java-target)"
                + " or an apk file (--apk-path), but not both."
            )
        if (
            configuration.FACEBOOK_SHIM
            and arguments.java_target is None
            and arguments.apk_path is None
        ):
            parser.error(
                "The analysis target should either be a java target (--java-target)"
                + " or an apk file (--apk-path)."
            )
        if not configuration.FACEBOOK_SHIM and arguments.apk_path is None:
            parser.error("The argument --apk-path is required.")

        # Build the vanilla java project.
        if configuration.FACEBOOK_SHIM and arguments.java_target:
            jar_file = _extract_jex_file_if_exists(
                _build_target(arguments.java_target, mode=arguments.java_mode),
                arguments.java_target,
                build_directory,
            )
            desugared_jar_file = _desugar_jar_file(jar_file)
            arguments.apk_path = os.fspath(_build_apk_from_jar(desugared_jar_file))

        # Build the mariana trench binary if necessary.
        binary = _get_analysis_binary(arguments)

        LOG.info(f"Extracting `{arguments.apk_path}`...")
        apk_directory = tempfile.mkdtemp(suffix="_apk")
        dex_directory = tempfile.mkdtemp(suffix="_dex")
        pyredex.utils.unzip_apk(arguments.apk_path, apk_directory)
        dex_mode = pyredex.unpacker.detect_secondary_dex_mode(apk_directory)
        dex_mode.unpackage(apk_directory, dex_directory)
        LOG.info(f"Extracted APK into `{apk_directory}` and DEX into `{dex_directory}`")

        options = [
            "--system-jar-paths",
            arguments.system_jar_configuration_path,
            "--apk-directory",
            apk_directory,
            "--dex-directory",
            dex_directory,
            "--rules-paths",
            arguments.rules_paths,
            "--repository-root-directory",
            arguments.repository_root_directory,
            "--source-root-directory",
            arguments.source_root_directory,
            "--apk-path",
            arguments.apk_path,
            "--output-directory",
            arguments.output_directory,
            "--maximum-source-sink-distance",
            str(arguments.maximum_source_sink_distance),
            "--model-generator-configuration-paths",
            arguments.model_generator_configuration_paths,
        ]

        if arguments.model_generator_search_paths is not None:
            options.append("--model-generator-search-paths")
            options.append(arguments.model_generator_search_paths)

        if arguments.proguard_configuration_paths:
            options.append("--proguard-configuration-paths")
            options.append(arguments.proguard_configuration_paths)

        if arguments.lifecycles_paths:
            options.append("--lifecycles-paths")
            options.append(arguments.lifecycles_paths)

        if arguments.source_exclude_directories:
            options.append("--source-exclude-directories")
            options.append(arguments.source_exclude_directories)

        if arguments.generated_models_directory:
            options.append("--generated-models-directory")
            options.append(arguments.generated_models_directory)

        if arguments.sequential:
            options.append("--sequential")
        if arguments.skip_source_indexing:
            options.append("--skip-source-indexing")
        if arguments.skip_model_generation:
            options.append("--skip-model-generation")
        if arguments.disable_parameter_type_overrides:
            options.append("--disable-parameter-type-overrides")
        if arguments.remove_unreachable_code:
            options.append("--remove-unreachable-code")
        if arguments.maximum_method_analysis_time is not None:
            options.append("--maximum-method-analysis-time")
            options.append(str(arguments.maximum_method_analysis_time))

        trace_settings = [f"MARIANA_TRENCH:{arguments.verbosity}"]
        if "TRACE" in os.environ:
            trace_settings.insert(0, os.environ["TRACE"])
        os.environ["TRACE"] = ",".join(trace_settings)

        if arguments.log_method:
            for method in arguments.log_method:
                options.append("--log-method=%s" % method.strip())
        if arguments.dump_class_hierarchies:
            options.append("--dump-class-hierarchies")
        if arguments.dump_overrides:
            options.append("--dump-overrides")
        if arguments.dump_call_graph:
            options.append("--dump-call-graph")
        if arguments.dump_dependencies:
            options.append("--dump-dependencies")
        if arguments.dump_methods:
            options.append("--dump-methods")

        command = [os.fspath(binary.resolve())] + options
        if arguments.gdb:
            command = ["gdb", "--args"] + command
        elif arguments.lldb:
            command = ["lldb", "--"] + command
        LOG.info(f"Running Mariana Trench: {' '.join(command)}")
        output = subprocess.run(command)
        if output.returncode != 0:
            LOG.fatal(f"Analysis binary exited with exit code {output.returncode}.")
            sys.exit(output.returncode)
    except (ClientError, configuration.Error) as error:
        LOG.fatal(error.args[0])
        sys.exit(1)
    except Exception:
        LOG.fatal(f"Unexpected error:\n{traceback.format_exc()}")
        sys.exit(1)
    finally:
        try:
            shutil.rmtree(build_directory)
        except IOError:
            pass  # Swallow.

if __name__ == "__main__":
    main()


import os
import sys
import random

from pyramid.config import Configurator
from pyramid.events import subscriber, BeforeRender
from pyramid.httpexceptions import HTTPNotFound

PROJECT_PATH = os.path.realpath(os.path.dirname(__file__))
PROJECT_BASE_PATH = os.sep.join(PROJECT_PATH.split(os.sep)[:-1])
MEDIA_VERSION = random.randint(1, 1000000)

# include paths
sys.path.insert(0, PROJECT_PATH)
sys.path.insert(0, os.path.join(PROJECT_PATH, 'apps'))

from session.mongo import MongoSessionFactoryConfig

# from auth.security import auth_callback
def auth_callback(identity, request):
    return ['group:admins']

# logging
import logging
logger = logging.getLogger(__name__)

def notfound(request):
    return HTTPNotFound('Page not found.')

@subscriber(BeforeRender)
def add_global(event):
    """
    add template context procesors
    """
    event['DEVELOP'] = event['request'].registry.settings.get('DEVELOP', False)
    event['STATIC_URL'] = event['request'].registry.settings.get('static_url', '/static/')
    event['MEDIA_VERSION'] = MEDIA_VERSION

def main(global_config, **settings):
    """ This function returns a Pyramid WSGI application.
    """
    config = Configurator(
        settings=settings,
        # session_factory=session_factory,
        # root_factory='main.resources.RootFactory',
        # authorization_policy=authorization_policy,
        # authentication_policy=authentication_policy,
    )
    config.include('pyramid_jinja2')
    config.include('cornice')

    # templates
    config.add_renderer('.html', 'pyramid_jinja2.renderer_factory')
    config.add_renderer('.jinja2', 'pyramid_jinja2.renderer_factory')
    config.add_jinja2_search_path('templates')

    config.add_notfound_view(notfound, append_slash=True)
    config.add_static_view('static', 'static', cache_max_age=3600)
    config.add_subscriber(add_global, BeforeRender)

    config.commit()  # need to commit because cornice HTTPForbidden conflict
    config.include('main')
    return config.make_wsgi_app()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
from textblob import TextBlob
import sys
#text = '''
#The titular threat of The Blob has always struck me as the ultimate movie
#monster: an insatiably hungry, amoeba-like mass able to penetrate
#virtually any safeguard, capable of--as a doomed doctor chillingly
#describes it--"assimilating flesh on contact.
#Snide comparisons to gelatin be damned, it's a concept with the most
#devastating of potential consequences, not unlike the grey goo scenario
#proposed by technological theorists fearful of
#artificial intelligence run rampant.
#'''

def main():
    text = sys.argv[1]
    blob = TextBlob(text)
    blob.tags           # [('The', 'DT'), ('titular', 'JJ'),
                        #  ('threat', 'NN'), ('of', 'IN'), ...]

    mynewtext = blob.noun_phrases
    #print(mynewtext)    # WordList(['titular threat', 'blob',
                        #            'ultimate movie monster',
                        #            'amoeba-like mass', ...])
    output = ""
    for word in mynewtext:
        output += word + " "
    print(output)
    return output
    #blob.translate(to="es")  # 'La amenaza titular de The Blob...'

if __name__ == "__main__":
   main()


# -*- coding: utf-8 -*-
"""
Created on Mon Apr 16 17:37:11 2018
@author: Charles Edmond TanYZ
Simple app to:
1. Log in to Limesurvey and get your key to perform operations
2. Find a select a csv to change emails
3. Change the emails
"""
#####
# for the csv, first column will be the "key", second column will be the "value"
#####

import Tkinter as tk
import urllib
import urllib2
import json
import sys
import csv
import os
import tkFileDialog

###############################################################################
###############################################################################

def get_session_key(user, pw):
    req = urllib2.Request(url='https://survey.inresearch.com.sg/index.php/admin/remotecontrol',\
                          data='{\"method\":\"get_session_key\",\"params\":[\"'+user+'\",\"'+pw+'\"],\"id\":1}')
    req.add_header('content-type', 'application/json')
    req.add_header('connection', 'Keep-Alive')
    try:
        f = urllib2.urlopen(req)
        myretun = f.read()
        #print myretun
        j=json.loads(myretun)
        return j['result']
    except :
        e = sys.exc_info()[0]
        print ( "
Error: %s
" % e )

def release_session_key(relkey):
    req = urllib2.Request(url='https://survey.inresearch.com.sg/index.php/admin/remotecontrol',\
                          data='{\"method\":\"release_session_key\",\"params\":[\"'+relkey+'\"],\"id\":1}')
    req.add_header('content-type', 'application/json')
    req.add_header('connection', 'Keep-Alive')
    try:
        f = urllib2.urlopen(req)
        myretun = f.read()
        #print myretun
        j=json.loads(myretun)
        return j['result']
    except :
        e = sys.exc_info()[0]
        print ( "
Error: %s
" % e )

def get_participant_email(skey, sid, tid):
    req = urllib2.Request(url='https://survey.inresearch.com.sg/index.php/admin/remotecontrol',\
                      data='{\"method\":\"get_participant_properties\",\
                              "params\":\
                              [\
                              "'+skey+'\",\
                              "'+sid+'\",\
                              "'+tid+'\",[\
                              "email\"]\
                              ],\
                              "id\": 1}')
    req.add_header('content-type', 'application/json')
    req.add_header('connection', 'Keep-Alive')
    try:
        f = urllib2.urlopen(req)
        myretun = f.read()
        #print myretun
        j=json.loads(myretun)
        return j['result']
    except:
        e = sys.exc_info()[0]
        print ( "
Error: %s
" % e )

def set_participant_email(skey, sid, tid, newEmail):
    req = urllib2.Request(url='https://survey.inresearch.com.sg/index.php/admin/remotecontrol',\
                      data='{\"method\":\"set_participant_properties\",\
                              "params\":[\
                                         "'+skey+'\",\
                                         "'+sid+'\",\
                                         "'+tid+'\",\
                                         {\
                                         "email\":\"'+newEmail+'\",\
                                         "sent\":\"N\",\
                                         "remindersent\":\"N\",\
                                         "remindercount\":\"0\"\
                                         }\
                                         ],\
                              "id\": 1}')
    req.add_header('content-type', 'application/json')
    req.add_header('connection', 'Keep-Alive')
    try:
        f = urllib2.urlopen(req)
        myretun = f.read()
        #print myretun
        j=json.loads(myretun)
        return j['result']
    except:
        e = sys.exc_info()[0]
        print ( "
Error: %s
" % e )

def list_surveys(skey, user):
    req = urllib2.Request(url='https://survey.inresearch.com.sg/index.php/admin/remotecontrol',\
                          data='{\"method\":\"list_surveys\",\"params\":[\"'+skey+'\",\"'+user+'\"],\"id\":1}')
    req.add_header('content-type', 'application/json')
    req.add_header('connection', 'Keep-Alive')
    try:
        f = urllib2.urlopen(req)
        myretun = f.read()
        #print myretun
        j=json.loads(myretun)
        return j['result']
    except :
        e = sys.exc_info()[0]
        print ( "
Error: %s
" % e )

#hard coded to list only up to 18000 participants
def list_participants(skey, sid):
    req = urllib2.Request(url='https://survey.inresearch.com.sg/index.php/admin/remotecontrol',\
                          data='{\"method\":\"list_participants\",\
                                  "params\":[\"'+skey+'\",\"'+sid+'\",\"0\",\"10000\"],\"id\":1}')
    req.add_header('content-type', 'application/json')
    req.add_header('connection', 'Keep-Alive')
    try:
        f = urllib2.urlopen(req)
        myretun = f.read()
        #print myretun
        j=json.loads(myretun)
        return j['result']
    except :
        e = sys.exc_info()[0]
        print ( "
Error: %s
" % e )

###############################################################################
###############################################################################

# select working directory and update the label for it (below)
def select_directory():
    directory = tkFileDialog.askdirectory(title='Select folder datafile is located in...')
    try:
        os.chdir(directory)
        return(directory)
    except:
        print('Directory was not selected')
        pass

def update_directory(stringVar):
    directory = select_directory()
    try:
        stringVar.set('Selected Directory: ' + directory)
    except:
        print('Could not update directory')
        pass

# select the csv file and update the label for it(below)
def select_datafile():
    datafile = os.path.basename(tkFileDialog.askopenfilename(title='Select CSV',initialdir=os.getcwd()))
    if datafile == '':
       datafile = "No file selected"
    return datafile

def update_datafile(dataVar, stringVar):
    datafile = select_datafile()
    try:
        stringVar.set('Selected CSV: ' + datafile)
        dataVar.set(datafile)
    except:
        print('Could not update to selected CSV')
        pass

#to refresh survey list upon log in (see below)
def refresh_survey_list(menuButton, menuVar, skey, user):
    menuVar.set('')
    menuButton['menu'].delete(0, 'end')
    surveyDic = get_survey_dic(skey, user)
    new_choices = list(surveyDic)
    for choice in new_choices:
        menuButton['menu'].add_command(label=choice, command=tk._setit(menuVar, surveyDic[choice]))

# 'log in'
def update_session_key(user, pw, keyString, resultVar, menuButton, menuVar):
    key = get_session_key(user, pw)
    if type(key) == dict:
        resultVar.set(key['status'])
    else:
        try:
            keyString.set(key)
            resultVar.set('Log in success!')
            refresh_survey_list(menuButton, menuVar, key, user)
            menuVar.set('No Survey Selected')
            print(key)
        except:
            resultVar.set('Log in failure')
            print('Could not set key to variable')
            pass

def getDic(inFile, statusVar):
    dic ={}
    if inFile.endswith('.csv'):
        with open(inFile, 'r') as readcsv:
            reader = csv.reader(readcsv)
            for row in reader:
                try:
                    dic[row[0]] = row[1]
                except IndexError:
                    dic[row[0]] = ''
        return dic
    else:
        return None

def change_emails_tid(master, skey, sid, statusVar, inFile):
    dic = getDic(inFile, statusVar)
    if dic == None:
        statusVar.set('You did not select a csv')
    else:
        timerCounter = 1.00
        tokenList = list_participants(skey,sid)
        for token in tokenList:
            try:
                tokenID = token['tid']
            except:
                continue
            if tokenID in dic:
                set_participant_email(skey, sid, token['tid'], dic[tokenID])
                print('Token ID: ' + token['tid'])
                print(str(tokenID) + '\'s email changed to: ' + dic[tokenID])
            currentTimer = int(timerCounter/float(len(tokenList))*100)
            if currentTimer%5== 0:
                statusVar.set(str(currentTimer)+'%')
                master.update_idletasks()
            timerCounter += 1.00

def change_emails_email(master, skey, sid, statusVar, inFile):
    dic = getDic(inFile, statusVar)
    if dic == None:
        statusVar.set('You did not select a csv')
    else:
        timerCounter = 1.00
        tokenList = list_participants(skey,sid)
        for token in tokenList:
            try:
                tokenEmail = token['participant_info']['email']
            except:
                continue
            if tokenEmail in dic:
                set_participant_email(skey, sid, token['tid'], dic[tokenEmail])
                print('Token ID: ' + token['tid'])
                print(str(tokenEmail) + ' changed to: ' + dic[tokenEmail])
            currentTimer = int(timerCounter/float(len(tokenList))*100)
            if currentTimer%5== 0:
                statusVar.set(str(currentTimer)+'%')
                master.update_idletasks()
            timerCounter += 1.00

def change_emails_token(master, skey, sid, statusVar, inFile):
    dic = getDic(inFile, statusVar)
    if dic == None:
        statusVar.set('You did not select a csv')
    else:
        timerCounter = 1.00
        tokenList = list_participants(skey,sid)
        for token in tokenList:
            try:
                tokenToken = token['token']
            except:
                continue
            if tokenToken in dic:
                set_participant_email(skey, sid, token['tid'], dic[tokenToken])
                print('Token ID: ' + token['tid'])
                print(str(tokenToken) + '\'s email changed to: ' + dic[tokenToken])
            currentTimer = int(timerCounter/float(len(tokenList))*100)
            if currentTimer%5== 0:
                statusVar.set(str(currentTimer)+'%')
                master.update_idletasks()
            timerCounter += 1.00

def get_survey_dic(skey, user):
    try:
        surveyList = list_surveys(skey, user)
        surveyDic = {}
        for survey in surveyList:
            surveyDic[survey['surveyls_title']] = survey['sid']
        return(surveyDic)
    except:
        print('could not get survey list from LS')
        return['No survey selected']
###############################################################################
###############################################################################

'''
changeEmails(dic)
print(release_session_key(mykey))'''

class MainApplication(tk.Frame):
    def __init__(self, master, *args, **kwargs):
        tk.Frame.__init__(self, master, *args, **kwargs)
        self.master = master
        self.init_window()

    def init_window(self):
        self.master.title('Email changer')
        self.variables = Variables(self.master)
        self.buttons = Buttons(self.master, self.variables)
        self.body = Body(self.master, self.buttons, self.variables)

class Body(tk.Frame):
    def __init__(self, master, buttons, variables):

        ##### LABELS #####

        userLabel = tk.Label(master, text='Username: ')
        userLabel.grid(row=0, column=0)
        passLabel = tk.Label(master, text='Password: ')
        passLabel.grid(row=1, column=0)
        logInStatusLabel = tk.Label(master, textvariable=variables.logInStatusVar)
        logInStatusLabel.grid(row=1, column=2)
        dirLabel = tk.Label(master, textvariable = variables.dirVar)
        dirLabel.grid(row=2, column=1)
        csvLabel = tk.Label(master, textvariable = variables.csvTextVar)
        csvLabel.grid(row=3, column=1)
        statusLabel = tk.Label(master, textvariable= variables.statusVar)
        statusLabel.grid(row=5, column=1)

class Buttons(tk.Frame):
    def __init__(self, master, variables):
        userEntry = tk.Entry(master, textvariable=variables.userVar)
        userEntry.grid(row=0, column=1)
        passEntry = tk.Entry(master, textvariable=variables.passVar, show='*')
        passEntry.grid(row=1, column=1)

        changeDirButton = tk.Button(master, text='Select Directory', command=
                                    lambda: update_directory(variables.dirVar))
        changeDirButton.grid(row=2, column=0, sticky=tk.W+tk.E)
        selectCSVButton = tk.Button(master, text='Select CSV', command=
                                    lambda: update_datafile(variables.csvVar,variables.csvTextVar))
        selectCSVButton.grid(row=3, column=0, sticky=tk.W+tk.E)
        selectSurveyMenu = tk.OptionMenu(master, variables.surveyListVar, *['test','test'])
        selectSurveyMenu.grid(row=4,column=0)
        getKeyButton = tk.Button(master, text='Log In', command=
                                 lambda: update_session_key(userEntry.get(),passEntry.get(),variables.keyVar, variables.logInStatusVar, selectSurveyMenu,variables.surveyListVar))
        getKeyButton.grid(row=0,column=2,sticky=tk.W+tk.E)

        #################### just testing ########################
        changeEmailsIDButton = tk.Button(master, text='Change Emails (token ID)', command=
                                       lambda: change_emails_tid(master,variables.keyVar.get(),variables.surveyListVar.get(),variables.statusVar, variables.csvVar.get()))
        changeEmailsIDButton.grid(row=3,column=2, sticky=tk.W+tk.E)
        changeEmailsTokenButton = tk.Button(master, text='Change Emails (token)', command=
                                       lambda: change_emails_token(master,variables.keyVar.get(),variables.surveyListVar.get(),variables.statusVar, variables.csvVar.get()))
        changeEmailsTokenButton.grid(row=4,column=2, sticky=tk.W+tk.E)
        changeEmailsEmailButton = tk.Button(master, text='Change Emails (email)', command=
                                       lambda: change_emails_email(master,variables.keyVar.get(),variables.surveyListVar.get(),variables.statusVar, variables.csvVar.get()))
        changeEmailsEmailButton.grid(row=5,column=2, sticky=tk.W+tk.E)

class Variables(tk.Frame):
    def __init__(self, master):
        self.userVar = tk.StringVar()
        self.passVar = tk.StringVar()
        self.logInStatusVar = tk.StringVar()
        self.keyVar = tk.StringVar()
        self.dirVar = tk.StringVar()
        self.dirVar.set(os.getcwd())
        self.csvVar = tk.StringVar()
        self.csvTextVar = tk.StringVar()
        self.csvTextVar.set('No CSV selected')
        self.surveyListVar = tk.Variable()
        self.surveyListVar.set('Select Survey')
        self.statusVar = tk.StringVar()

if __name__ == "__main__":
    root = tk.Tk()
    MainApplication(root)
    root.mainloop()


from logging import getLogger
import traceback

from Acquisition import aq_base
from Missing import MV
from DateTime import DateTime
from Products.ZCatalog.Lazy import LazyMap
from Products.PluginIndexes.common import safe_callable
from Products.CMFCore.utils import _getAuthenticatedUser
from Products.CMFCore.permissions import AccessInactivePortalContent
from Products.CMFCore.utils import _checkPermission
from zope.component import getUtility
from zope.component import queryMultiAdapter
from zope.component import ComponentLookupError

from plone.registry.interfaces import IRegistry
from plone.indexer.interfaces import IIndexableObject

from pyes.exceptions import IndexMissingException
from pyes import ES
from pyes.exceptions import (IndexAlreadyExistsException,
                             NotFoundException)

from collective.elasticsearch.brain import BrainFactory
from collective.elasticsearch.query import QueryAssembler
from collective.elasticsearch.interfaces import (
    IElasticSettings, DISABLE_MODE, DUAL_MODE)
from collective.elasticsearch.utils import getUID
from collective.elasticsearch.indexes import getIndex
from collective.elasticsearch.ejson import dumps
from collective.elasticsearch import td

logger = getLogger(__name__)
info = logger.info
warn = logger.warn

CONVERTED_ATTR = '_elasticconverted'

class PatchCaller(object):
    """
    Very odd I have to do this. If I don't,
    I get very pecular errors trying to call
    the original methods
    """

    def __init__(self, patched_object):
        self._patched_object = patched_object

    def __getattr__(self, name):
        """
        assuming original attribute has "__old_" prefix
        """
        if name[0] == '_':
            return self.__dict__[name]
        _type = type(aq_base(self._patched_object))
        func = getattr(_type, '__old_' + name)
        # "bind" it
        def bound_func(*args, **kwargs):
            return func(self._patched_object, *args, **kwargs)
        return bound_func

class ResultWrapper(object):
    """
    To cache a group of results that rolls.
    optimized for sequentially access
    """

    def __init__(self, rl, count=None, cache_size=1000, bulk_size=400):
        self.rl = rl
        if count is None:
            count = self.rl.count()
        if cache_size > count:
            cache_size = count
        if bulk_size > count:
            bulk_size = count
        if bulk_size > cache_size:
            cache_size = min(bulk_size * 2, count)
        self.cache_size = cache_size
        self.bulk_size = bulk_size
        self.iloc = lbound = 0
        # fill up the cache to start...
        self.cache = rl[lbound:self.cache_size]

    def __getitem__(self, val):
        lbound = self.iloc
        rbound = lbound + self.cache_size
        if isinstance(val, slice):
            if lbound <= val.start and rbound >= val.end:
                val.start = val.start - lbound
                val.end = val.end - rbound
                return self.cache[val]
            else:
                start = val.start
                end = val.end
        else:
            if lbound <= val and rbound > val:
                return self.cache[val - self.iloc]
            else:
                start = end = val
        # grab a group, trimming off any that we need to...
        if start > (rbound - 1) or end > (rbound - 1):
            # in this case, we're adding to the end
            # chop off front
            self.cache = self.cache[self.bulk_size:]
            self.iloc += self.bulk_size
            additional = self.rl[rbound:rbound + self.bulk_size]
            if len(additional) == 0:
                raise IndexError
            # add to end
            self.cache.extend(additional)
        elif self.iloc > 0:
            # in this case, we're adding to front
            end = start
            start = min(self.iloc - self.bulk, 0)
            rcache = self.cache[end:]
            self.cache = self.rl[start:end]
            if len(self.cache) == 0:
                raise IndexError
            self.cache.extend(rcache)
        else:
            raise Exception("Error finding data")
        return self[val]

    def __iter__(self):
        return self

class ElasticSearch(object):

    trns_mapping = {
        'data': {
            'type': 'string',
            'index': 'not_analyzed',
            'store': True
        },
        'transaction_id': {
            'type': 'integer'
        },
        'order': {
            'type': 'integer'
        },
        'action': {
            'type': 'string',
            'index': 'not_analyzed',
            'store': True
        },
        'uid': {
            'type': 'string',
            'index': 'not_analyzed',
            'store': True
        }
    }

    def __init__(self, catalogtool):
        self.catalogtool = catalogtool
        self.catalog = catalogtool._catalog
        self.patched = PatchCaller(self.catalogtool)

        try:
            registry = getUtility(IRegistry)
            try:
                self.registry = registry.forInterface(IElasticSettings)
            except:
                self.registry = None
        except ComponentLookupError:
            self.registry = None

        self.tdata = td.get()

    @property
    def catalog_converted(self):
        return getattr(self.catalogtool, CONVERTED_ATTR, False)

    @property
    def mode(self):
        if not self.catalog_converted:
            return DISABLE_MODE
        if self.registry is None:
            return DISABLE_MODE
        return self.registry.mode

    @property
    def bulk_size(self):
        try:
            return self.registry.bulk_size
        except:
            return 400

    @property
    def max_retries(self):
        try:
            return self.registry.max_retries
        except:
            return 3

    @property
    def timeout(self):
        try:
            return self.registry.timeout
        except:
            return 30.0

    @property
    def conn(self):
        if self.tdata.conn is None:
            self.tdata.conn = ES(self.registry.connection_string,
                bulk_size=self.bulk_size,
                max_retries=self.max_retries,
                timeout=self.timeout)
        return self.tdata.conn

    def query(self, query):
        qassembler = QueryAssembler(self.catalogtool)
        dquery, sort = qassembler.normalize(query)
        equery = qassembler(dquery)
        result = self.conn.search(equery, self.catalogsid, self.catalogtype,
            sort=sort, fields="_metadata")
        count = result.count()
        result = ResultWrapper(result, count=count)
        factory = BrainFactory(self.catalog)
        return LazyMap(factory, result, count)

    def catalog_object(self, obj, uid=None, idxs=[],
                       update_metadata=1, pghandler=None):
        mode = self.mode
        if mode in (DISABLE_MODE, DUAL_MODE):
            result = self.patched.catalog_object(
                obj, uid, idxs, update_metadata, pghandler)
            if mode == DISABLE_MODE:
                return result
        wrapped_object = None
        if not IIndexableObject.providedBy(obj):
            # This is the CMF 2.2 compatible approach, which should be used
            # going forward
            wrapper = queryMultiAdapter((obj, self.catalogtool), IIndexableObject)
            if wrapper is not None:
                wrapped_object = wrapper
            else:
                wrapped_object = obj
        else:
            wrapped_object = obj
        conn = self.conn
        catalog = self.catalog
        if idxs == []:
            idxs = catalog.indexes.keys()
        index_data = {}
        for index_name in idxs:
            index = getIndex(catalog, index_name)
            if index is not None:
                value = index.get_value(wrapped_object)
                if value in (None, 'None'):
                    # yes, we'll index null data...
                    value = None
                index_data[index_name] = value
        if update_metadata:
            metadata = {}
            for meta_name in catalog.names:
                attr = getattr(wrapped_object, meta_name, MV)
                if (attr is not MV and safe_callable(attr)):
                    attr = attr()
                metadata[meta_name] = attr
            # XXX Also, always index path so we can use it with the brain
            # to make urls
            metadata['_path'] = wrapped_object.getPhysicalPath()
            index_data['_metadata'] = dumps(metadata)

        uid = getUID(obj)
        try:
            doc = conn.get(self.catalogsid, self.catalogtype, uid)
            self.registerInTransaction(uid, td.Actions.modify, doc)
        except NotFoundException:
            self.registerInTransaction(uid, td.Actions.add)
        conn.index(index_data, self.catalogsid, self.catalogtype, uid)
        if self.registry.auto_flush:
            conn.refresh()

    def registerInTransaction(self, uid, action, doc={}):
        if not self.tdata.registered:
            self.tdata.register(self)
        conn = self.conn
        data = {
            'transaction_id': self.tdata.tid,
            'data': dumps(doc),
            'order': self.tdata.counter,
            'action': action,
            'uid': uid
        }
        conn.index(data, self.catalogsid, self.trns_catalogtype)
        self.tdata.counter += 1

    def uncatalog_object(self, uid, obj=None, *args, **kwargs):
        mode = self.mode
        if mode in (DISABLE_MODE, DUAL_MODE):
            if self.catalog.uids.get(uid, None) is not None:
                result = self.patched.uncatalog_object(uid, *args, **kwargs)
            if mode == DISABLE_MODE:
                return result
        conn = self.conn

        uid = getUID(obj)
        try:
            doc = conn.get(self.catalogsid, self.catalogtype, uid)
            self.registerInTransaction(uid, td.Actions.delete, doc)
        except NotFoundException:
            pass
        try:
            conn.delete(self.catalogsid, self.catalogtype, uid)
        except NotFoundException:
            # already gone... Multiple calls?
            pass
        if self.registry.auto_flush:
            conn.refresh()

    def manage_catalogRebuild(self, REQUEST=None, RESPONSE=None):
        mode = self.mode
        if mode == DISABLE_MODE:
            return self.patched.manage_catalogRebuild(REQUEST, RESPONSE)

        self.recreateCatalog()

        return self.patched.manage_catalogRebuild(REQUEST, RESPONSE)

    def manage_catalogClear(self, REQUEST=None, RESPONSE=None, URL1=None):
        mode = self.mode
        if mode == DISABLE_MODE:
            return self.patched.manage_catalogClear(REQUEST, RESPONSE, URL1)

        self.recreateCatalog()

        if mode == DUAL_MODE:
            return self.patched.manage_catalogClear(REQUEST, RESPONSE, URL1)

    def refreshCatalog(self, clear=0, pghandler=None):
        mode = self.mode
        if mode == DISABLE_MODE:
            return self.patched.refreshCatalog(clear, pghandler)

        return self.patched.refreshCatalog(clear, pghandler)

    def recreateCatalog(self):
        conn = self.conn
        try:
            conn.delete_index(self.catalogsid)
        except IndexMissingException:
            pass
        self.convertToElastic()

    def searchResults(self, REQUEST=None, check_perms=False, **kw):
        mode = self.mode
        if mode == DISABLE_MODE:
            return self.patched.searchResults(REQUEST, **kw)
        if isinstance(REQUEST, dict):
            query = REQUEST.copy()
        else:
            query = {}
        query.update(kw)

        if check_perms:
            show_inactive = query.get('show_inactive', False)
            if isinstance(REQUEST, dict) and not show_inactive:
                show_inactive = 'show_inactive' in REQUEST

            user = _getAuthenticatedUser(self.catalogtool)
            query['allowedRolesAndUsers'] = self.catalogtool._listAllowedRolesAndUsers(user)

            if not show_inactive and not _checkPermission(
                    AccessInactivePortalContent, self.catalogtool):
                query['effectiveRange'] = DateTime()
        orig_query = query.copy()
        # info('Running query: %s' % repr(orig_query))
        try:
            return self.query(query)
        except:
            info("Error running Query: %s\n%s" %(
                repr(orig_query),
                traceback.format_exc()))
            if mode == DUAL_MODE:
                # fall back now...
                return self.patched.searchResults(REQUEST, **kw)
            else:
                return LazyMap(BrainFactory(self.catalog), [], 0)

    def convertToElastic(self):
        setattr(self.catalogtool, CONVERTED_ATTR, True)
        self.catalogtool._p_changed = True
        properties = {}
        for name in self.catalog.indexes.keys():
            index = getIndex(self.catalog, name)
            if index is not None:
                properties[name] = index.create_mapping(name)
            else:
                raise Exception("Can not locate index for %s" % (
                    name))

        # XXX then add an index specifically to hold metadata
        # We don't store any of the other index data
        # this will be json encoded
        properties['_metadata'] = {
            'type': 'string',
            'index': 'not_analyzed',
            'store': True
        }

        conn = self.conn
        try:
            conn.create_index(self.catalogsid)
        except IndexAlreadyExistsException:
            pass

        mapping = {'properties': properties}
        conn.indices.put_mapping(
            doc_type=self.catalogtype,
            mapping=mapping,
            indices=[self.catalogsid])
        conn.indices.put_mapping(
            doc_type=self.trns_catalogtype,
            mapping=self.trns_mapping,
            indices=[self.catalogsid])

    @property
    def catalogsid(self):
        return '-'.join(self.catalogtool.getPhysicalPath()[1:]).lower()

    @property
    def catalogtype(self):
        return self.catalogtool.getId().lower()

    @property
    def trns_catalogtype(self):
        return self.catalogtype + '_trns'


# -*- coding: utf-8 -*-
from django.db import models
from django_extensions.db.models import TimeStampedModel

from admin_app.core.models import Source

class Keyset(TimeStampedModel):
    title = models.CharField(
        max_length=255,
        verbose_name=u'название',
        )
    source = models.ForeignKey(
        Source,
        related_name='keysets',
        verbose_name=u'ресурс',
        )

    def __unicode__(self):
        return u'{0} {1}'.format(
            self.source.title,
            self.title
        )

    class Meta:
        unique_together = ('title', 'source')
        verbose_name = u'Набор ключей'
        verbose_name_plural = u'Наборы ключей'

class Mapping(TimeStampedModel):
    keyset = models.ForeignKey(
        Keyset,
        related_name='mappings',
        verbose_name=u'набор ключей',
    )
    key = models.CharField(
        max_length=255,
        verbose_name=u'ключ',
    )
    value = models.CharField(
        max_length=255,
        verbose_name=u'значение',
    )

    def __unicode__(self):
        return u'{0} {1} {2}'.format(
            self.keyset.title,
            self.key,
            self.value
        )

    class Meta:
        unique_together = ('keyset', 'key')
        verbose_name = u'Маппинг'
        verbose_name_plural = u'Маппинг'


# Import some useful libraries
import numpy as np
from sklearn.svm import SVC
from sklearn.svm import NuSVC
from sklearn.svm import LinearSVC
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV
from sklearn import preprocessing

# Import user files
import src.readData as rd
import src.featureExtraction as fe

def main():
	# ------------------------------------------------------------------------
	# READ DATA
	# ------------------------------------------------------------------------

	# Read train data from file
	img_train_data = rd.read_image_data("data/set_train/")
	if img_train_data == -1:
		print("ERROR: Could not read data!")
		return

	# Read associated target values from file
	train_targets = rd.read_file("data/targets.csv")
	if train_targets.all() == -1:
		print("ERROR: Could not read targets!")
		return

	# ------------------------------------------------------------------------
	# PREPROCESS AND FEATURE EXTRACTION
	# ------------------------------------------------------------------------

	# Extract features
	train_features = fe.subvoxels_heuristics(
		img_train_data,
		subvoxels_size_=np.array([5,5,5]),
		print_info_="train ")

	# Scale features to zero mean and unit variance
	train_scaler = preprocessing.StandardScaler().fit(train_features)
	train_features_scaled = train_scaler.transform(train_features)

	# # Get histogram vector features of training data
	train_feature_matrix = fe.slice_histogram(img_train_data, 30, 50)
	if train_feature_matrix.all() == -1:
		print("ERROR: Could not compute features!")
		return

	# # Postprocess features
	#(train_feature_matrix, low_std_indices) = postp.low_variance_filter(train_feature_matrix, threshold_=10)
	#print(train_feature_matrix.shape)

	# ------------------------------------------------------------------------
	# CLASSIFICATION MODELS
	# ------------------------------------------------------------------------

	# Declare classifier list
	classifier = [
		SVC()]

	# Make parameter lists
	C = np.arange(-1.0,4.0)
	C = 10 ** C
	gamma = np.arange(-3.0,4.0)
	gamma = 10 ** gamma
	kernel = ["rbf", "linear"]
	coef0 = np.arange(0.0,1.0)
	degree = np.arange(3,7)

	# Generate dicts
	param_SVC = {
		"C":C,
		"kernel":kernel,
		"degree":degree,
		"gamma":gamma,
		"coef0":coef0,
		"probability":[True],
		"class_weight":["balanced"]}

	# Save dicts in list
	param_grid = [
		param_SVC]

	# ------------------------------------------------------------------------
	# CROSSVALIDATION
	# ------------------------------------------------------------------------

	print("\nCross validation")

	# Validate each classification model on subsets
	clf = []

	for cl in classifier:
		index = classifier.index(cl)

		clf.append(GridSearchCV(cl,
			param_grid[index],
			scoring="neg_log_loss",
			cv=10))

		clf[index].fit(train_features_scaled, train_targets)

	# ------------------------------------------------------------------------
	# MODEL SELECTION
	# ------------------------------------------------------------------------

	print("\nModel selection")

	# Select model according to highest score (lowest error)
	best_params = []
	best_score = []
	best_estimator = []

	for cl in clf:
		index = clf.index(cl)

		best_score.append(cl.best_score_)
		best_params.append(cl.best_params_)
		best_estimator.append(cl.best_estimator_)

	best_model_index = best_score.index(max(best_score))
	print(best_params)
	print(best_score)

	# ------------------------------------------------------------------------
	# CLASSIFICATION
	# ------------------------------------------------------------------------

	# Read train data from file
	img_test_data = rd.read_image_data("data/set_test/")
	if img_test_data == -1:
		print("ERROR: Could not read data!")
		return

	# Get features
	test_features = fe.subvoxels_heuristics(
		img_test_data,
		subvoxels_size_=np.array([5,5,5]),
		print_info_="test ")

	# Preprocess features
	train_scaler = preprocessing.StandardScaler().fit(test_features)
	test_features_scaled = train_scaler.transform(test_features)

	# Predict with best model parameters
	prediction_test_targets = best_estimator[best_model_index].predict_proba(test_features_scaled)[:,1]

	# Dump prediction to file
	id_array = np.arange(1,len(prediction_test_targets)+1)
	submission_array = np.column_stack((id_array, prediction_test_targets))

	np.savetxt("final_sub.csv", submission_array, delimiter=",", header="ID,Prediction", comments="", fmt="%d,%f")

	return


import random
from enum import Enum

running = True

class rpsGame:
	def __init__(self):
		self.userChoice = items.NULL
		self.aiChoice = items.NULL
		self.winner = entity.NULL

	def setUserChoice(self, choice):
		self.userChoice = items(int(choice))

	def getUserChoice(self):
		return items(self.userChoice).name

	def generateAiChoice(self):
		self.aiChoice = items(random.randint(1, 3))

	def getAiChoice(self):
		return items(self.aiChoice).name

	def calculateWinner(self):
		if self.userChoice == items.ROCK:
			if self.aiChoice == items.ROCK: self.winner = entity.NONE
			if self.aiChoice == items.PAPER: self.winner = entity.AI
			if self.aiChoice == items.SCISSORS: self.winner = entity.PLAYER
		if self.userChoice == items.PAPER:
			if self.aiChoice == items.ROCK: self.winner = entity.PLAYER
			if self.aiChoice == items.PAPER: self.winner = entity.NONE
			if self.aiChoice == items.SCISSORS: self.winner = entity.AI
		if self.userChoice == items.SCISSORS:
			if self.aiChoice == items.ROCK: self.winner = entity.AI
			if self.aiChoice == items.PAPER: self.winner = entity.PLAYER
			if self.aiChoice == items.SCISSORS: self.winner = entity.NONE

	def getWinner(self):
		if self.winner == entity.NONE:
			winner = "Draw!"
		elif self.winner == entity.PLAYER:
			winner = "Player Wins!"
		elif self.winner == entity.AI:
			winner = "AI Wins!"
		return winner

	def compareWinner(self):
		if self.winner == entity.PLAYER:
			comp = " > "
		elif self.winner == entity.AI:
			comp = " < "
		else:
			comp = " = "
		return comp

class items(Enum):
	NULL = 0
	ROCK = 1
	PAPER = 2
	SCISSORS = 3

class entity(Enum):
	NULL = 0
	PLAYER = 1
	AI = 2
	NONE = 3

def main():
	# Initialize rpsGame class
	game = rpsGame()

	print("Rock Paper Scissors!")
	print("=======================")
	while running:
		print("")
		print("1 - Rock")
		print("2 - Paper")
		print("3 - Scissors")
		game.setUserChoice(input(">"))
		print("")
		print("AI is thinking...")
		game.generateAiChoice()
		print("")
		game.calculateWinner()
		print(game.getUserChoice(), game.compareWinner(), game.getAiChoice())
		print(game.getWinner())
	#end

if __name__ == "__main__":
	main()


import sys
from PyQt5.QtWidgets import (QMainWindow, QApplication)
from PyQt5.QtCore import pyqtSignal, QObject

class Commute(QObject):
    appSig = pyqtSignal()

class Example(QMainWindow):
    def __init__(self):
        super().__init__()
        self.initUI()

    def initUI(self):
        self.c = Commute()
        self.c.appSig.connect(self.print)

        self.setGeometry(300, 300, 400, 200)
        self.setWindowTitle('发射信号')
        self.show()

    def mousePressEvent(self, e):
        self.c.appSig.emit()

    def print(self):
        print("Clicked!")

if __name__ == "__main__":
    app = QApplication(sys.argv)
    ex = Example()
    sys.exit(app.exec_())


from bottle import route, run, template, static_file, get, post, request
from echo import add_message, get_messages, clear_messages

@get('/')
def get_chat():
    """ get_chat() runs when you load the page """
    messages = get_messages()
    return template('chat', name='', messages=messages)

@post('/message')
def post_message():
    """ post_message() will run when you press the chat button """
    name = request.forms.get('name')
    message = request.forms.get('message')

    messages = add_message(name, message)
    return template('chat', name=name, messages=messages)

@get('/clear')
def get_chat():
    messages = clear_messages()
    return template('chat', name='', messages=messages)

@route('/static/<filepath:path>')
def server_static(filepath):
    return static_file(filepath, root='./static')

run(host='localhost', port=8080, debug=True, reloader=True)


import pygame
from Hero_OOP import Hero
from Goblin_OOP import Goblin
from Monster_OOP import Monster

# from math import fabs
# from random import randint

pygame.init()

screen = {
"height": 512, #y
"width": 480 #x
}
keys_up = {
"right": 275,
"left": 276,
"up": 273,
"down": 274
}
keys_down = {
"right": False,
"left": False,
"up": False,
"down": False
}
powerup = {
'active':True,
'tick_gotten':0
}

game_paused = False

directions = ['N','S','E','W', 'NE','NW','SE','SW']

screen_size = (screen["height"], screen["width"])
pygame_screen = pygame.display.set_mode(screen_size)
pygame.display.set_caption("Welcome to Tech Narnia ~ CEO Showdown")
background_image = pygame.image.load('./images/narnia_pygame.jpg')
hero_image = pygame.image.load('./images/steve_jobs2.jpg')
hero_image_scaled = pygame.transform.scale(hero_image, (50,50))
goblin_image = pygame.image.load('./images/bill_gates2.png')
monster_image = pygame.image.load('./images/monster_pygame1.png')
# monster2_image =

hero = Hero("Steve",screen,100,100,10,0,True)
goblin = Goblin("Bill",screen,200,200,5,5,"N")
monster = Monster("Spiked Ball",screen,300,300,5,5,"N")
# monster2 = Monster("Lightning Strike",screen, 250,250,4,4, "S")
	# hero = { #Steve
	# "x": 100,
	# "y": 100,
	# "speed": 10,
	# "wins": 0,
	# "is_alive": True
	# }

	# goblin = { #Bill
	# 	"x": 200,
	# 	"y":200,
	# 	"speed_x": 5,
	# 	"speed_y":5,
	# 	"direction": "N"
	# }

	# monster = { #blue spiked ball
	# 	"x": 300,
	# 	"y": 300,
	# 	"speed_x": 5,
	# 	"speed_y": 5,
	# 	"direction":"N"
	# }

	# monster2 = {
	# "x": 250,
	# "y": 250,
	# "speed_x": 4,
	# "speed_y": 4,
	# "direction": "S"

	# }

pygame_screen.blit(background_image, [0,0])
#draw the hero wins on the screen.
font = pygame.font.Font(None, 25)
hp_text = font.render("HP %d" % (500), True, (255,255,255))
exp_text = font.render("EXP %d" % (0), True, (255,255,255))
pygame_screen.blit(hp_text, [40,40])
#draw the hero
pygame_screen.blit(hero_image_scaled, [hero['x'],hero['y']])
pygame_screen.blit(goblin_image, [goblin['x'],goblin['y']])
pygame_screen.blit(monster_image, [monster['x'],monster['y']])
#flip the screen and start ove4
pygame.display.flip()


from flask_restful import Resource, reqparse, Api
from flask import Flask, make_response
from flask_cors import CORS
from module.aladinV2 import Aladin
from module.yes24V2 import Yes24
# import module.aladin as Aladin
#import module.yes24 as Yes24
import json

class Search(Resource):
    def get(self):
        try:
            parser = reqparse.RequestParser()
            parser.add_argument('word', required=True,
                                type=str, help='Please enter word')
            parser.add_argument('mode', required=True,
                                type=int, help='Please enter mode')
            args = parser.parse_args()
            result = ''
            if args['mode'] == 0:
                aladin = Aladin(args['word'])
                result = aladin.result()
            elif args['mode'] == 1:
                yes24 = Yes24(args['word'])
                result = yes24.result()
            """
            elif args['mode'] == "common":
            """
            resp = make_response(json.dumps(result, ensure_ascii=False))
            return resp
        except Exception as e:
            return {'error': str(e)}

app = Flask('api_offbookstore')
CORS(app)
app.config['JSON_AS_ASCII'] = False
api = Api(app)
api.add_resource(Search, '/search')

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=7000, debug=True)


from selenium import webdriver
import time
import math
import os

link = "http://suninjuly.github.io/file_input.html"

try:
    browser = webdriver.Chrome(executable_path="C:\webdrivers\chromedriver_win32\chromedriver_win32 (1)\chromedriver.exe")
    browser.get(link)

    f_name_el = browser.find_element_by_name("firstname")
    f_name_el.send_keys("First")

    l_name_el = browser.find_element_by_name("lastname")
    l_name_el.send_keys("Last")

    f_name_el = browser.find_element_by_name("email")
    f_name_el.send_keys("Email")

    att_file_el = browser.find_element_by_id("file")
    current_dir = os.path.abspath(os.path.dirname(__file__))     # получаем путь к директории текущего исполняемого файла
    file_path = os.path.join(current_dir, 'HEllo.txt')           # добавляем к этому пути имя файла
    att_file_el.send_keys(file_path)

    butt_el = browser.find_element_by_class_name("btn.btn-primary")
    butt_el.click()

finally:
    # успеваем скопировать код за 30 секунд
    time.sleep(10)
    # закрываем браузер после всех манипуляций
    browser.quit()

# не забываем оставить пустую строку в конце файла


from autobahn.twisted.wamp import ApplicationSession
from autobahn.wamp import auth, types
from autobahn.wamp.exception import ApplicationError
from twisted.internet.defer import inlineCallbacks, returnValue

from autobahn.twisted.util import sleep

class Authenticator(ApplicationSession):

    @inlineCallbacks
    def onJoin(self, details):
        dummy_auth = {
            'martha.stewart@gmail.com': {
                'username': 'Martha113',
                'secret': 'bakery',
                'role': 'academic',
            },
            'christopher': {
                'username': 'christopher1984',
                'secret': 'lee',
                'role': 'academic;recruiter;pro',
            },
            'john': {'secret': 'locke', 'role': 'backend'},
            'joe.bloggs@gmail.com': {'secret': u'test', 'role': u'backend', 'data':{}}
        }
        def authenticate(realm, authid, details):
            print("test")
            print(realm)
            print(authid)
            print(details)
#            test = db.query(User).filter_by(
#                email=authid.lower()).first()
#            if test != None:
#                pass

            if authid == 'file_bot_server':
                print('{} connected!'.format(authid))
                return {'secret': 'bot_secret', 'role': 'file_bot_server'}
            if authid == 'file_bot_client':
                print('{} connected!'.format(authid))
                return {'secret': 'bot_secret', 'role': 'file_bot_client'}
            if authid in dummy_auth.keys():
                print("Valid authid")
                print(type(dummy_auth[authid]['secret']))
                print(type(dummy_auth[authid]['role']))
                return {'secret': dummy_auth[authid]['secret'],
                    'role': dummy_auth[authid]['role'],
                    'data':dummy_auth[authid]['data']}
            print("Invalid authid")
            raise ApplicationError("com.example.no_such_user", "could not authenticate session - no such user {}".format(authid))
        try:
            yield self.register(authenticate, 'com.centrality.authenticate')
            print("successfully registered")
        except Exception as e:
            print(e)
        print("registered authentication")

Academic Website (Requires Web Server)/authenticator.py