import cv2, copy
import numpy as np
def frameToImage(depth, color, ir):
    i_depth = depth.asarray() / 4500.
    i_color = cv2.resize(color.asarray(), (int(1920 / 3), int(1080 / 3)))
    i_ir = ir.asarray() / 65535.
    return i_depth, i_color, i_ir

def depthToPointCloudWithPos(device, depth):
    points = []
    for i in range(depth.height):
        for j in range(depth.width):
            pos = list(device.registration.getPointXYZ(depth, i, j))
            if not np.isnan(pos[0]):
                points.append([pos[0], pos[1], pos[2], i, j])
    return points

def depthToPointCloud(device, depth):
    points = []
    for i in range(depth.height):
        for j in range(depth.width):
            pos = list(device.registration.getPointXYZ(depth, i, j))
            if not np.isnan(pos[0]):
                points.append(pos)
    return points

def depthToPointCloudWithDownSample(device, depth, mask, d):
    points = []
    for i in range(depth.height):
        for j in range(depth.width):
            if mask[i][j] == True and (i + j) % d == 0:
                try:
                    state = True
                    for ii in range(5):
                        for jj in range(5):
                            state = state and mask[i + ii - 2][j + jj - 2]
                    if state:
                        pos = list(device.registration.getPointXYZ(depth, i, j))
                        if (not np.isnan(pos[0])) and (pos[0] ** 2 + pos[1] ** 2 + pos[2] ** 2 <= 2.2):
                            points.append(pos)
                except:
                    pass

    return points

def depthToPointCloudWithDownSampleAndPlane(device, depth, mask, d, plane):
    points = []
    tmp_scale = np.sqrt(plane[0] ** 2 + plane[1] ** 2 + 1)
    for i in range(depth.height):
        for j in range(depth.width):
            if mask[i][j] == True and (i + j) % d == 0:
                try:
                    state = True
                    for ii in range(11):
                        for jj in range(11):
                            state = state and mask[i + ii - 5][j + jj - 5]
                    if state:
                        pos = list(device.registration.getPointXYZ(depth, i, j))
                        if (not np.isnan(pos[0])) and (pos[0] ** 2 + pos[1] ** 2 + pos[2] ** 2 <= 2.2):
                            dist = (pos[0] * plane[0] + pos[1] * plane[1] - (pos[2] - plane[2])) / tmp_scale
                            if abs(dist) > 0.05:
                                points.append(pos)
                except:
                    pass

    return points

def depthToPointCloudWithMask(device, depth, mask):
    points = []
    for i in range(depth.height):
        for j in range(depth.width):
            if mask[i][j] == True:
                pos = list(device.registration.getPointXYZ(depth, i, j))
                if (not np.isnan(pos[0])) and (pos[0] ** 2 + pos[1] ** 2 + pos[2] ** 2 <= 2.2):
                    points.append(pos)
    return points

def depthToPoint(device, depth, x, y):
    min_dist = 2 ** 31
    feature_pos = None
    for i in range(3):
        for j in range(3):
            try:
                pos = list(device.registration.getPointXYZ(depth, x + i, y + j))
                if (j ** 2 + i ** 2 < min_dist) and (not np.isnan(pos[0])):
                    feature_pos = copy.copy(pos)
            except:
                pass
    return feature_pos

def grayToThreeChannels(gray):
    tem = copy.copy(gray)
    tem = np.array([tem])
    tem = np.concatenate((tem, tem, tem), axis = 0)
    return np.moveaxis(tem, 0, -1)


import pygame
from random import randint
pygame.mixer.init()
pygame.mixer.music.load("on000003.mp3")
pygame.mixer.music.play()
x= 410#265 555
y= 130
posx=260
posy=800
posya=800
posyb=800
velocidade=20

velocidade_foguetes= 15
fundo= pygame.image.load("fundo.PNG")
foguete= pygame.image.load("fog.PNG")
foguete2= pygame.image.load("fog2.PNG") #vermelho
foguete3= pygame.image.load("fog3.PNG") #azul
foguete4= pygame.image.load("fog4.PNG") #amarelo

janela = pygame.display.set_mode((888,500))
pygame.display.set_caption("JOGO DO TEMPO EM CRIACAO")

janela_aberta = True
while janela_aberta:
    pygame.time.delay(50)

    for event in pygame.event.get():
        if event.type==pygame.QUIT:
            janela_aberta= False
    comandos = pygame.key.get_pressed()

    if comandos[pygame.K_RIGHT] and x<=555:
        x += velocidade
    if comandos[pygame.K_LEFT] and x>=265:
        x -= velocidade

    if posy<=-180 and posya <=-180 and posyb <=-180:
        posy= randint(800,2000)
        posya=randint(800,2000)
        posyb=randint(800,2000)

    posy -=velocidade_foguetes+8
    posya -=velocidade_foguetes +10
    posyb -=velocidade_foguetes +5

    janela.blit(fundo, (0,0))
    janela.blit(foguete, (x,y))
    janela.blit(foguete2,(posx, posy))
    janela.blit(foguete3,(posx+300, posya))
    janela.blit(foguete4,(posx+150, posyb))

    pygame.display.update()
pygame.quit()


'''
This module addresses problems from 5.1.2.
'''

import datetime

def main():

    # The user is prompted to enter date and time with specific format.
    ui = input('Enter a datetime with the following format: YYYY-mm-dd  hh:mm:ss:ffffff  ')

    # Here, user info is output to screen.
    d = datetime.datetime.strptime(ui, '%Y-%m-%d %H:%M:%S:%f')
    print(d)

if __name__ == '__main__':
    main()


import numpy as np
import pandas as pd
import sklearn
import argparse
import sys

import kmapper as km
import sklearn.manifold as skm
import sklearn.decomposition as skd
import sklearn.neighbors as skn

import scipy.spatial.distance as ssd

from a_patent_data import PatentData
from a_analyzer import Analyzer
from a_utilities import color_averager, is_empty_data

import mappertools.outputs.text_dump as tdump

from analysis_merged_pca import get_labels_and_data

def get_common_parser():
    common_parser = argparse.ArgumentParser(add_help = False)
    common_parser.add_argument("--verbose", "-v", action="store_true", help="verbose.")

    # Data choice
    group_data_choice = common_parser.add_argument_group("Data choice")
    group_data_choice.add_argument("--data", help="data choice: 0 (folder 180901_csv) or 1 (folder 200110_csv) (default: 1).", type=int, default=1, choices=[0,1])
    group_data_choice.add_argument("--mode", "-m", help="mode choice: 0 or 1 or 2 (default: 0; both modes: 2).", type=int, default=0, choices=[0,1,2])

    group_data_choice.add_argument("--from_year", "-f", help="starting year to do analysis.", type=int,default=1976)
    group_data_choice.add_argument("--to_year", "-g", help="ending year (inclusive) to do analysis.", type=int,default=2005)

    # Processing
    group_processing = common_parser.add_argument_group("Processing")
    group_processing.add_argument("--keep_zeros", "-z", action="store_true", help="preserve zeros columns in data. Do not use. Otherwise, drop zero columns.")
    group_processing.add_argument("--cos_trans", "-c", action="store_true", help="use cosine distances to transform data.")

    # output choices
    group_output = common_parser.add_argument_group("Output options")
    group_output.add_argument("--clustermap", action="store_true", help="Do clustermap.")
    group_output.add_argument("--no_dump_raw", action="store_true", help="Skip dumping raw data.")
    # group_output.add_argument("--interactive", action="store_true", help="interactive plot of lens.")
    group_output.add_argument("--kclusters", help="number(s) of k-Medoids and k-Means to compute and append to cyjs output. Note that k-Means ignores --metric and always computes with Euclidean distances.", type=int, nargs="+", default=None)

    group_output.add_argument("--char_limit", help="limit chars for firms and patent class names", type=int, default=None)
    # group_output.add_argument("--no_mapper", action="store_true", help="Skip Mapper computation entirely.")
    group_output.add_argument("--output", "-o", help="output base folder.", type=str)

    return common_parser

def get_parser():
    parser = argparse.ArgumentParser(description="Program for performing Mapper analysis on patent data.")
    parser.set_defaults(verbosity=0, procedure=None)
    subparsers = parser.add_subparsers(help="choose mode of operation:")
    common_parser = get_common_parser()

    # ACCUMULATE
    accum_parser = subparsers.add_parser("accumulate",
                                         help="accumulate all chosen years. Operation order: cos-dist-transform (if enabled), then accumulate, then log (if enabled)",
                                         parents=[common_parser])
    accum_parser.set_defaults(procedure="accumulate")

    # MERGE
    merge_parser = subparsers.add_parser("merge",
                                         help="merge all chosen years",
                                         parents=[common_parser])
    merge_parser.set_defaults(procedure="merge")

    # MERGE-ACCUMULATE
    common_parser_copy = get_common_parser()
    matching_groups = (g for g in common_parser_copy._action_groups if g.title == 'Data choice')
    group = next(matching_groups, None) or common_parser_copy
    group.add_argument("--window", "-w", type=int, help="window size (default=5)",default=5)
    group.add_argument("--shift", "-s", type=int, help="window shift (default=5)",default=5)

    merge_accum_parser = subparsers.add_parser("ma",
                                               help="accumulate over window, then merge across shifted windows.",
                                               parents=[common_parser_copy])
    merge_accum_parser.set_defaults(procedure="merge_accumulate")

    return parser

def do_final_year_analysis(proc, data):
    ans_columns = ["yfinal--yfinal_neighbor", "yfinal--yfinal_neighborname", "yfinal--yfinal_popmean"]
    ans = pd.DataFrame(index=proc.labels.unique_members, columns=ans_columns)

    # extract final year
    final_year = max(proc.labels.years_data)
    final_year_bool = (proc.labels.years_data == final_year)
    final_year_data = data[final_year_bool]
    final_year_firms = proc.labels.intemporal_index[final_year_bool]

    # final year distances
    final_year_distances = ssd.squareform(ssd.pdist(final_year_data,metric="cosine"))

    # output distances
    main_folder = proc.get_main_folder()
    name = "{:s}_{:s}_finalyeardistances.csv".format(proc.labels.transforms_name, proc.labels.data_name)
    output_fname = main_folder.joinpath(name)
    pd.DataFrame(final_year_distances, index=final_year_firms, columns=final_year_firms).to_csv(str(output_fname))

    # final year pop mean
    final_year_popmean = np.mean(final_year_data, axis=0)

    # final year nn
    nbrs = skn.NearestNeighbors(n_neighbors=2, algorithm='auto', metric="precomputed").fit(final_year_distances)
    final_year_nn_distances, final_year_nn_indices = nbrs.kneighbors(n_neighbors=1)

    for i, firm in enumerate(final_year_firms):
        ans.loc[firm, "yfinal--yfinal_neighbor"] = final_year_nn_distances[i][0]
        ans.loc[firm, "yfinal--yfinal_neighborname"] = final_year_firms[final_year_nn_indices[i][0]]

        ans.loc[firm, "yfinal--yfinal_popmean"] = ssd.cosine(final_year_data.iloc[i,:], final_year_popmean)

    return ans

def do_year_mean_analysis(proc, data):
    ans_columns = ["ymean--ymean_neighbor", "ymean--ymean_neighborname", "ymean--ymean_popmean"]
    ans = pd.DataFrame(index=proc.labels.unique_members, columns=ans_columns)

    # create means for each firm
    firm_mean_data = pd.DataFrame(columns = data.columns)
    for firm in proc.labels.unique_members:
        firm_bool = proc.labels.intemporal_index == firm
        firm_data = np.mean(data[firm_bool])
        firm_mean_data.loc[firm] = firm_data
    # remove all-zero rows (firms):
    firm_mean_data.dropna(axis=0, how="all", inplace=True)

    # output year means
    main_folder = proc.get_main_folder()
    name = "{:s}_{:s}_firmmeans.csv".format(proc.labels.transforms_name, proc.labels.data_name)
    output_fname = main_folder.joinpath(name)
    firm_mean_data.to_csv(str(output_fname))

    # yearmean distances
    firm_mean_distances = ssd.squareform(ssd.pdist(firm_mean_data,metric="cosine"))

    # output distances
    name = "{:s}_{:s}_firmmeansdistances.csv".format(proc.labels.transforms_name, proc.labels.data_name)
    output_fname = main_folder.joinpath(name)
    pd.DataFrame(firm_mean_distances, index=firm_mean_data.index, columns=firm_mean_data.index).to_csv(str(output_fname))

    # firm mean pop mean
    firm_mean_popmean = np.mean(firm_mean_data, axis=0)

    # firm mean nn
    nbrs = skn.NearestNeighbors(n_neighbors=2, algorithm='auto', metric="precomputed").fit(firm_mean_distances)
    firm_mean_nn_distances, firm_mean_nn_indices = nbrs.kneighbors(n_neighbors=1)

    for i, firm in enumerate(firm_mean_data.index):
        ans.loc[firm, "ymean--ymean_neighbor"] = firm_mean_nn_distances[i][0]
        ans.loc[firm, "ymean--ymean_neighborname"] = firm_mean_data.index[firm_mean_nn_indices[i][0]]

        ans.loc[firm, "ymean--ymean_popmean"] = ssd.cosine(firm_mean_data.loc[firm,:], firm_mean_popmean)

    return ans

def do_jaffe_measures(args, labels, data, verbosity):
    if is_empty_data(data, args.from_year, args.to_year): return

    if args.sum_to_one:
        summed = data.sum(axis=1)
        assert np.allclose(summed.loc[summed!=0],1)

    # prepare mapper data and lens
    if args.mds == True:
        lens_name = "mds{}d".format(args.dimension)
    else:
        lens_name = "pca{}d".format(args.dimension)

    proc = Analyzer(data, labels=labels,
                          lens=None, lens_name=lens_name, metric=args.metric,
                          verbose=verbosity)

    if args.mds:
        if proc.metric == "precomputed":
            dists = proc.distance_matrix
        else:
            X = ssd.pdist(data, metric=args.metric)
            dists = ssd.squareform(X)
        proc.lens = skm.MDS(n_components=args.dimension, dissimilarity="precomputed").fit_transform(dists)
    else:
        proc.lens = skd.PCA(n_components=args.dimension).fit_transform(data)

    # initialize analyzer
    proc.initialize(base_path=args.output)

    # Other outputs
    if args.clustermap: proc.do_clustermap()
    if not args.no_dump_raw: proc.dump_data()
    if True: proc.plot_lens(np.array(labels.rgb_colors)/255., show=False)
    if args.kclusters:
        kClusters = proc.dataframe_kClusters(args.kclusters, dump_summary=True, dump_aggregates=True)

    ans = do_final_year_analysis(proc, data)
    ans = ans.join(do_year_mean_analysis(proc,data), how="outer")
    print(ans)

    main_folder = proc.get_main_folder()
    name = "{:s}_{:s}_jaffemeasures.csv".format(proc.labels.transforms_name, proc.labels.data_name)
    output_fname = main_folder.joinpath(name)
    ans.to_csv(str(output_fname))

def main(raw_args):
    parser = get_parser()
    args = parser.parse_args(raw_args)

    if args.procedure == None:
        parser.print_help()
        exit()

    # fix choices for Jaffe measure
    args.transpose = False
    args.log = False
    args.sum_to_one = True
    args.metric = "cosine"

    labels, data = get_labels_and_data(args)

    args.mds = False
    args.dimension = 2

    do_jaffe_measures(args, labels, data, verbosity=(2 if args.verbose else 0))

if __name__ == "__main__":
    main(sys.argv[1:])


#!/usr/bin/env python3

"""
NB THIS IS CURRENTLY WRONG
Author: Ian
Purpose: Coding challenge as per https://www.hackerrank.com/challenges/magic-square-forming/problem
Input: A 3*3 matrix of ints ranged 1 to 9 inclusive
Output: int, the minimum error correction to make the matrix a magic square
"""

import math
import pdb

def formingMagicSquare(s):
	"""Input: 3x3 matix of ints range 1 through 9
	Output: positive int"""
	# 1. Choose magic_num
	cols = []
	for ind,row in enumerate(s):
		col = [x[ind] for x in s]
		cols.append(col)
	fwd_diag = [s[x][x] for x in range(0, len(s))]
	rev_diag = [s[::-1][x][x] for x in range(0, len(s))]
	row_sums = [sum(x) for x in s]
	col_sums = [sum(x) for x in cols]
	diag_sums = [sum(x) for x in [rev_diag, fwd_diag]]
	mean = sum(row_sums + col_sums + diag_sums)/8
	magic_num = int(round(mean))

	# 2. Func to find MEC for given magic_num
	# collate all the triplets that are < magicnum
	# find the intersections of these, add one
	# repeat until none
	# rows = {}
	# cols = {}
	# for i in range(0,len(s)):
	# 	rows[i] = [(i,x) for x in range(0,len(s))]
	# 	cols[i] = [(x,i) for x in range(0,len(s))]
	rows = []
	cols = []
	for i in range(0,len(s)):
		rows.append([(i,x) for x in range(0,len(s))])
		cols.append([(x,i) for x in range(0,len(s))])
	diag_one = [(0,0), (1,1), (2,2)]
	diag_two = [(0,2), (1,1), (2,0)]
	all_triples = rows + cols + [diag_one] + [diag_two]

	# iterate through triples, find those less than magic
	less_than = [x for x in all_triples if sum_from_mat(x,s) < magic_num]
	more_than = [x for x in all_triples if sum_from_mat(x,s) > magic_num]
	score = 0
	while any(less_than + more_than):
		if less_than:
			intersect, move_on = find_intersect(less_than, fwd_diag, rev_diag)
			if move_on:
				lessers = less_than[:]
				less_than = []
			else:
				s[intersect[0]][intersect[1]] += 1
				score += 1
				less_than = [x for x in all_triples if sum_from_mat(x,s) < magic_num]
		else:
			intersect, move_on = find_intersect(more_than, fwd_diag, rev_diag)
			if move_on:
				morers = more_than[:]
				more_than = []
			else:
				s[intersect[0]][intersect[1]] -= 1
				score += 1
				more_than = [x for x in all_triples if sum_from_mat(x,s) > magic_num]

	# For each entry in morers and for each in the lessers, add 2 to score
	pdb.set_trace()

	# pdb.set_trace()
	# rows = {i: [(0,x) for x in range(0,len(s))], for i in range(0,len(s))}
	# row_a = [(0,x) for x in range(0,len(s))]
	# row_b = [(1,x) for x in range(0,len(s))]
	# row_c = [(2,x) for x in range(0,len(s))]
	# for

	# 3. Repeat for magic nums within possible realm of beating ans
	return score

def sum_from_mat(index_list, s):
	nums = [s[x[0]][x[1]] for x in index_list]
	return sum(nums)

def find_intersect(triples, fwd_diag, rev_diag):
	""" Find the position most common among a list of triple"""
	spilled_positions = [item for sublist in triples for item in sublist]
	intersects = sorted(spilled_positions, key=spilled_positions.count, reverse=True)
	intersect = intersects[0]
	counts = intersects.count(intersect)
	pdb.set_trace()
	# Alert if the intersect would change a working triple
	move_on = False
	if counts >= 3:
		move_on = False
	elif counts == 2 and ((intersect in fwd_diag) or (intersect in rev_diag)):
		move_on = False
	else:
		move_on = True
	return intersect, move_on
	# pdb.set_trace()
# def mec_matrix_to_magic_square (mat, magic_num):

if __name__ == "__main__":
	mat = [[4, 9, 2], [3, 5, 7], [8, 1, 5]] # ans 1
	mat = [[4, 8, 2], [4, 5, 7], [6, 1, 6]]  # ans 4
	ans = formingMagicSquare(mat)
	print(ans)

# def formingMagicSquare(s):
# 	# First get the average sum of the rows and cols
# 	cols = []
# 	for i in range(0, len(s)):
# 		cols.append([x[i] for x in s])
# 	rows = s
# 	rows_n_cols = rows + cols

# 	sums = [sum(x) for x in rows_n_cols]
# 	mean_sum = sum(sums) / len(sums)
# 	mean_sum = int(round(mean_sum))
# 	magic_num = mean_sum

# 	# Assuming magic_num is the mean, but need to check alternatives too
# 	current_ans = dist_to_magic_square(rows, cols, magic_num)

# 	# Make list of possible magic nums - all ints within float mean +/- min ans
# 	possible_magic_nums = []
# 	rows_sums = [sum(x) for x in rows] # redundantly written below but who's checking
# 	cols_sums =	[sum(x) for x in cols]
# 	lower_comparator = math.floor(sum(min(rows_sums, cols_sums)) / 3)
# 	higher_comparator = math.ceil(sum(max(rows_sums, cols_sums)) / 3)
# 	possible_magic_nums = list(range((lower_comparator - current_ans), (higher_comparator + current_ans) + 1))
# 	possible_magic_nums.remove(magic_num)

# 	for magic_number in possible_magic_nums:
# 		tmp_ans = dist_to_magic_square(rows, cols, magic_number)
# 		if tmp_ans < current_ans:
# 			current_ans = tmp_ans
# 	return current_ans

# def dist_to_magic_square(rows, cols, magic_num):
# 	# Hypoth - cost of single change is 1 if synergy, else 2
# 	rows_sums = [sum(x) for x in rows]
# 	cols_sums =	[sum(x) for x in cols]
# 	row_deltas = [(magic_num - x) for x in rows_sums]
# 	col_deltas = [(magic_num - x) for x in cols_sums]
# 	row_deltas_abs = [abs(magic_num - x) for x in rows_sums]
# 	col_deltas_abs = [abs(magic_num - x) for x in cols_sums]
# 	rows_total_delta = sum(row_deltas_abs)
# 	cols_total_delta = sum(col_deltas_abs)

# 	cost = 0
# 	neg_row_deltas = [x for x in row_deltas if x < 0]
# 	neg_col_deltas = [x for x in col_deltas if x < 0]
# 	pos_row_deltas = [x for x in row_deltas if x > 0]
# 	pos_col_deltas = [x for x in col_deltas if x > 0]
# 	sum_row_neg = sum(neg_row_deltas)
# 	sum_col_neg = sum(neg_col_deltas)
# 	sum_row_pos = sum(pos_row_deltas)
# 	sum_col_pos = sum(pos_col_deltas)

# 	if rows_total_delta >= cols_total_delta:
# 		adders = 2 * abs(sum_row_neg)
# 		adders -= min(abs(sum_row_neg), abs(sum_col_neg))
# 		cost += adders

# 		subtractors = 2 * (abs(sum_row_pos))
# 		subtractors -= min(abs(sum_row_pos), abs(sum_col_pos))
# 		cost += subtractors
# 	else:
# 		adders = 2 * abs(sum_col_neg)
# 		adders -= min(abs(sum_col_neg), abs(sum_row_neg))
# 		cost += adders

# 		subtractors = 2 * (abs(sum_col_pos))
# 		subtractors -= min(abs(sum_col_pos), abs(sum_row_pos))
# 		cost += subtractors

# 	return cost


from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import os
import csv

chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')
browser = None

def main():
    for i in range(7,10):
        getPageData(i)

def getPageData(pageindex):
    global browser
    browser = webdriver.Chrome(options=chrome_options)
    # browser = webdriver.Chrome()
    url = f'https://hub.docker.com/search?type=image&page={pageindex}'
    print(url)
    browser.get(url)
    now_handle = browser.current_window_handle
    browser.implicitly_wait(20)  # 智能等待20秒
    itemElements = browser.find_elements_by_xpath(r"//div[@id='searchResults']/div/a")

    imageList = []

    for itemElement in itemElements:
        browser.switch_to.window(now_handle)  # 切换到原来的窗口
        nameElement = itemElement.find_element_by_xpath(r"div/div[3]/div/div/div[1]")
        nameStr = nameElement.text #image名称
        # //div[@id='searchResults']/div/a/div/div[2]/div[3]/div[2]/span[1]
        try:
            starElement = itemElement.find_element_by_xpath(r"div/div[2]/div[3]/div[2]/span[1]")

        except:
            starElement = itemElement.find_element_by_xpath(r"div/div[2]/div[3]/div/span[1]")
        starStr = starElement.text  # image star
        dockerPullCmd,versionList = dealitemElement(itemElement,nameStr) # image pull command , version dockerfile link list
        imageDict = {"nameStr":nameStr,"starStr":starStr,"dockerPullCmd":dockerPullCmd,"versionList":versionList}
        print(imageDict)
        imageList.append(imageDict)
    SaveOnePageDataToCsv(imageList)

    browser.quit()

def SaveOnePageDataToCsv(imageList):
    for imageDict in imageList:
        printCsvImage("docker_hub",imageDict)

def printCsvTitle(filename):
    path = "./csv/"
    isExists=os.path.exists(path)
    if not isExists:
        os.makedirs(path)
    with open("./csv/"+filename+".csv",'w',newline='',encoding='utf8') as f:
        row_title = ['name','star','pull command','version','dockerfilelink']
        write=csv.writer(f)
        write.writerow(row_title)

def printCsvImage(filename,imageDict):
    # 不存在路径，创建路径
    path = "./csv/"
    isExists = os.path.exists(path)
    if not isExists:
        os.makedirs(path)
    # 不存在文件则创建文件，添加文件头
    path = "./csv/" + filename + ".csv"
    isExists = os.path.exists(path)
    if not isExists:
        printCsvTitle(filename)

    if imageDict['versionList'] == None:
        with open("./csv/" + filename + ".csv", 'a', newline='', encoding='utf8') as f:
            row = [imageDict['nameStr'], imageDict['starStr'], imageDict['dockerPullCmd'],"",""]
            write = csv.writer(f)
            write.writerow(row)
    else:
        for version in imageDict['versionList']:
            with open("./csv/" + filename + ".csv", 'a', newline='', encoding='utf8') as f:
                row = [imageDict['nameStr'], imageDict['starStr'], imageDict['dockerPullCmd'],version['version'],version['link']]
                write = csv.writer(f)
                write.writerow(row)

def dealitemElement(itemElement,nameStr):
    imagePageUrl = itemElement.get_attribute("href")
    js = f'window.open("{imagePageUrl}");'
    browser.execute_script(js)

    browser.implicitly_wait(20)  # 智能等待20秒

    originHandle = browser.current_window_handle
    handles = browser.window_handles
    imageHandle = None
    for handle in handles:
        if handle != originHandle:
            imageHandle = handle

    browser.switch_to.window(imageHandle)
    try:
        dockerPullCmd = browser.find_element_by_xpath(r"/html/body/div[1]/div[1]/div/div[2]/div/div/div/div[2]/div/div[1]/div[2]/div/div[2]/div/div[2]/input")\
            .get_attribute("value")
    except:
        dockerPullCmd = None
    try:
        versionUlElement = browser.find_element_by_xpath(
        r"/html/body/div[1]/div[1]/div/div[2]/div/div/div/div[3]/div/div[1]/div/div/div/div/div/ul[2]")
        versionList = getVersionAndDockerfileLink(versionUlElement)
    except:
        versionList = None
    browser.close()
    return dockerPullCmd,versionList

def getVersionAndDockerfileLink(versionUlElement):
    versionElements = versionUlElement.find_elements_by_xpath("li/a")
    versionList = []
    for versionElement in versionElements:
        dockerfileLinkStr = versionElement.get_attribute("href")
        version = versionElement.text
        versionList.append({"link" : dockerfileLinkStr, "version" : version})
    return versionList

if __name__ == '__main__':
    main()


#!/usr/bin/env python
#-*- coding:utf-8 -*-
# author:任向民
# datetime:2019/4/3 15:56
# software: PyCharm

TESTER_CYCLE = 20
GETTER_CYCLE = 20
TESTER_ENABLED = True
GETTER_ENABLED = True
API_ENABLED = True
API_HOST = '127.0.0.1'
API_PORT = 5555

from multiprocessing import Process
from ip_io import app
from save_run import Getter
from Test import Tester
import time

class Scheduler():
    def schedule_tester(self, cycle = TESTER_CYCLE):
        tester = Tester()
        while True:
            print('开始测试')
            tester.run()
            time.sleep(cycle)

    def schedule_getter(self, cycle=GETTER_CYCLE):

        getter = Getter()
        while True:
            print('开始抓取代理')
            getter.run()
            time.sleep(cycle)

    def schedule_api(self):
        app.run(API_HOST, API_PORT)

    def run(self):
        print('运行代理池')
        if TESTER_ENABLED:
            tester_process = Process(target=self.schedule_tester)
            tester_process.start()

        if GETTER_ENABLED:
            getter_process = Process(target=self.schedule_getter)
            getter_process.start()

        if API_ENABLED:
            api_process = Process(target=self.schedule_api)
            api_process.start()


import gym
import rl
from mountaincar_utils import *

import numpy as np

env_max_pos = 0.6
env_min_pos = -1.2
env_abs_max_speed = 0.07
env_goal_pos = 0.5

if __name__ == "__main__":

    episodes = 500
    episode_step_size = 1000
    batch_size = int(episode_step_size / 2)

    # environment setup
    env = gym.make('MountainCar-v0')
    env._max_episode_steps = episode_step_size + 1 # workaround for hardcoded 200 steps per episode wtf
    state_size = env.observation_space.shape[0]
    action_size = env.action_space.n

    # agent setup
    state = env.reset()
    state = np.reshape(state, [1, state_size])
    agent = rl.QLearningAgent(state_size, action_size, epsilon_decay=0.995)

    done = False

    for e in range(episodes):
        state = env.reset()

        for i in range(episode_step_size):
            #env.render()

            state = normalize_state(state,env_max_pos, env_min_pos, env_abs_max_speed)
            state = np.reshape(state, [1, state_size])

            action = agent.choose_action(state)

            next_state, _, done, _ = env.step(action)

            position = next_state[0]

            # print("Done: {}, reward: {}, step: {}".format(done, reward,i))
            # print("State: [position, velocity]: " + str(state))

            reward = cubic_approximation_reward_flat(position, done)

            agent.remember(state, action, reward, np.reshape(next_state, [1, state_size]), done)

            state = next_state

            if i % batch_size == 0 and i != 0:
                agent.learn_from_past(batch_size)

            if done:
                agent.learn_from_past(batch_size)
                break

        print("End of episode: {}/{}, epsilon: {}".format(e, episodes, round(agent.epsilon, 4)))

    save_model(agent)

    env.close()


from uuid import uuid4

from pylm.parts.core import Inbound, Outbound,\
    zmq_context, BypassInbound
from pylm.parts.messages_pb2 import PalmMessage
from http.server import HTTPServer, BaseHTTPRequestHandler
import zmq
import sys

class RepService(Inbound):
    """
    RepService binds to a given socket and returns something.
    """
    def __init__(self,
                 name,
                 listen_address,
                 broker_address="inproc://broker",
                 logger=None,
                 cache=None,
                 messages=sys.maxsize):
        """
        :param name: Name of the service
        :param listen_address: ZMQ socket address to bind to
        :param broker_address: ZMQ socket address of the broker
        :param logger: Logger instance
        :param messages: Maximum number of messages. Defaults to infinity
        :return:
        """
        super(RepService, self).__init__(
            name,
            listen_address,
            zmq.REP,
            reply=True,
            broker_address=broker_address,
            bind=True,
            logger=logger,
            cache=cache,
            messages=messages
        )

class PullService(Inbound):
    """
    PullService binds to a socket waits for messages from a push-pull queue.
    """
    def __init__(self,
                 name,
                 listen_address,
                 broker_address="inproc://broker",
                 logger=None,
                 cache=None,
                 messages=sys.maxsize):
        """
        :param name: Name of the service
        :param listen_address: ZMQ socket address to bind to
        :param broker_address: ZMQ socket address of the broker
        :param logger: Logger instance
        :param messages: Maximum number of messages. Defaults to infinity.
        :return:
        """
        super(PullService, self).__init__(
            name,
            listen_address=listen_address,
            socket_type=zmq.PULL,
            reply=False,
            broker_address=broker_address,
            bind=True,
            logger=logger,
            cache=cache,
            messages=messages
        )

class PushService(Outbound):
    """
    PullService binds to a socket waits for messages from a push-pull queue.
    """
    def __init__(self,
                 name,
                 listen_address,
                 broker_address="inproc://broker",
                 logger=None,
                 cache=None,
                 messages=sys.maxsize):
        """
        :param name: Name of the service
        :param listen_address: ZMQ socket address to bind to
        :param broker_address: ZMQ socket address of the broker
        :param logger: Logger instance
        :param messages: Maximum number of messages. Defaults to infinity.
        :return:
        """
        super(PushService, self).__init__(
            name,
            listen_address=listen_address,
            socket_type=zmq.PUSH,
            reply=False,
            broker_address=broker_address,
            bind=True,
            logger=logger,
            cache=cache,
            messages=messages
        )

class PubService(Outbound):
    """
    PullService binds to a socket waits for messages from a push-pull queue.

    :param name: Name of the service
    :param listen_address: ZMQ socket address to bind to
    :param broker_address: ZMQ socket address of the broker
    :param logger: Logger instance
    :param messages: Maximum number of messages. Defaults to infinity.
    :param pipelined: Defaults to False. Pipelined if publishes to a
        server, False if publishes to a client.
    :param server: Name of the server, necessary to pipeline messages.
    """
    def __init__(self,
                 name,
                 listen_address,
                 broker_address="inproc://broker",
                 logger=None,
                 cache=None,
                 messages=sys.maxsize,
                 pipelined=False,
                 server=None):
        super(PubService, self).__init__(
            name,
            listen_address=listen_address,
            socket_type=zmq.PUB,
            reply=False,
            broker_address=broker_address,
            bind=True,
            logger=logger,
            cache=cache,
            messages=messages
        )
        self.name = server
        self.pipelined = pipelined

    def handle_stream(self, message):
        """
        Handle the stream of messages.

        :param message: The message about to be sent to the next step in the
            cluster
        :return: topic (str) and message (PalmMessage)

        The default behaviour is the following. If you leave this function
        unchanged and pipeline is set to False, the topic is the ID of the
        client, which makes the message return to the client. If the pipeline
        parameter is set to True, the topic is set as the name of the server and
        the step of the message is incremented by one.

        You can alter this default behaviour by overriding this function.
        Take into account that the message is also available in this function,
        and you can change other parameters like the stage or the function.
        """
        if self.pipelined:
            # If the master is pipelined,
            topic = self.name
            message.stage += 1
        else:
            topic = message.client

        return topic, message

    def start(self):
        """
        Call this function to start the component
        """
        message = PalmMessage()

        self.listen_to.bind(self.listen_address)
        self.logger.info('{} successfully started'.format(self.name))

        for i in range(self.messages):
            self.logger.debug('{} blocked waiting for broker'.format(self.name))
            message_data = self.broker.recv()
            self.logger.debug('{} Got message from broker'.format(self.name))
            message_data = self._translate_from_broker(message_data)
            message.ParseFromString(message_data)

            for scattered in self.scatter(message):
                topic, scattered = self.handle_stream(scattered)
                self.listen_to.send_multipart([topic.encode('utf-8'),
                                               message.SerializeToString()])
                self.logger.debug('Component {} Sent message. Topic {}'.format(
                    self.name, topic))

                if self.reply:
                    feedback = self.listen_to.recv()
                    feedback = self._translate_to_broker(feedback)
                    self.handle_feedback(feedback)

            self.broker.send(self.reply_feedback())

        return self.name

class WorkerPushService(PushService):
    """
    This is a particular push service that does not modify the messages that
    the broker sends.
    """
    def _translate_from_broker(self, message_data):
        """
        See help of parent
        :param message_data:
        :return:
        """
        self.logger.debug('{} translating from broker'.format(self.name))
        return message_data

    def _translate_to_broker(self, message_data):
        """
        See help of parent
        :param message_data:
        :return:
        """
        return message_data

class WorkerPullService(PullService):
    """
    This is a particular pull service that does not modify the messages that
    the broker sends.
    """
    def _translate_to_broker(self, message_data):
        """
        See help of parent
        :param message_data:
        :return:
        """
        return message_data

    def _translate_from_broker(self, message_data):
        """
        See help of parent
        :param message_data:
        :return:
        """
        return message_data

class PushPullService(object):
    """
    Push-Pull Service to connect to workers
    """
    def __init__(self,
                 name,
                 push_address,
                 pull_address,
                 broker_address="inproc://broker",
                 logger=None,
                 cache=None,
                 messages=sys.maxsize):
        """
        :param name: Name of the component
        :param listen_address: ZMQ socket address to listen to
        :param socket_type: ZMQ inbound socket type
        :param reply: True if the listening socket blocks waiting a reply
        :param broker_address: ZMQ socket address for the broker
        :param bind: True if socket has to bind, instead of connect.
        :param logger: Logger instance
        :param cache: Cache for shared data in the server
        :param messages: Maximum number of inbound messages. Defaults to infinity.
        :return:
        """
        self.name = name.encode('utf-8')

        self.push = zmq_context.socket(zmq.PUSH)
        self.pull = zmq_context.socket(zmq.PULL)
        self.push_address = push_address
        self.pull_address = pull_address
        self.push.bind(push_address)
        self.pull.bind(pull_address)

        self.broker = zmq_context.socket(zmq.REQ)
        self.broker.identity = self.name
        self.broker.connect(broker_address)

        self.logger = logger
        self.cache = cache
        self.messages = messages

        self.last_message = b''

    def scatter(self, message_data):
        """
        To be overriden. Picks a message and returns a generator that multiplies the messages
        to the broker.
        :param message_data:
        :return:
        """
        yield message_data

    def handle_feedback(self, message_data):
        """
        To be overriden. Handles the feedback from the broker
        :param message_data:
        :return:
        """
        self.last_message = message_data

    def reply_feedback(self):
        """
        To be overriden. Returns the feedback if the component has to reply.
        :return:
        """
        return self.last_message

    def start(self):
        message = PalmMessage
        self.logger.info('{} Successfully started'.format(self.name))
        initial_message = PalmMessage()
        initial_message.pipeline = '0'
        initial_message.client = '0'
        initial_message.stage = 0
        initial_message.function = ''
        initial_message.payload = b'0'
        self.broker.send(initial_message.SerializeToString())

        for i in range(self.messages):
            self.logger.debug('{} blocked waiting for broker'.format(self.name))
            message_data = self.broker.recv()
            self.logger.debug('Got message {} from broker'.format(i))
            message.ParseFromString(message_data)

            for scattered in self.scatter(message):
                self.push.send(scattered)
                self.handle_feedback(self.pull.recv())

            self.broker.send(self.reply_feedback())

        return self.name

    def cleanup(self):
        self.push.close()
        self.pull.close()
        self.broker.close()

class RepBypassService(BypassInbound):
    """
    Generic connection that opens a Rep socket and bypasses the broker.
    """
    def __init__(self, name, listen_address, logger=None,
                 cache=None, messages=sys.maxsize):
        """
        :param name: Name of the connection
        :param listen_address: ZMQ socket address to listen to
        :param logger: Logger instance
        :param messages:
        :return:
        """
        super(RepBypassService, self).__init__(name, listen_address, zmq.REP,
                                               reply=True, bind=True,
                                               logger=logger, cache=cache,
                                               messages=messages)

class HttpService(Inbound):
    """
    Similar to PullService, but the connection offered is an HTTP server
    that deals with inbound messages.

    ACHTUNG: this thing is deliberately single threaded
    """
    def __init__(self,
                 name,
                 hostname,
                 port,
                 broker_address="inproc://broker",
                 logger=None,
                 cache=None):
        self.name = name.encode('utf-8')
        self.hostname = hostname
        self.port = port
        self.logger = logger
        self.broker = zmq_context.socket(zmq.REQ)
        self.broker.identity = self.name
        self.broker.connect(broker_address)
        self.cache = cache

    def _make_handler(self):
        """
        This is serious meta programming. Note that the handler reuses
        the socket that connects to the router. This is intentional and
        makes the handler strictly single threaded.

        :return: Returns a PalmHandler class, that is a subclass of
          BaseHttpRequestHandler
        """
        # Clarify the scope since self is masked by the returned class
        scatter = self.scatter
        _translate_to_broker = self._translate_to_broker
        broker = self.broker
        handle_feedback = self.handle_feedback
        reply_feedback = self.reply_feedback

        class PalmHandler(BaseHTTPRequestHandler):
            def do_POST(self):
                """
                Note that this http server always replies
                """
                message = PalmMessage()
                self.send_response(200)
                self.end_headers()
                message_data = self.rfile.read(
                    int(
                        self.headers.get('Content-Length')
                    )
                )

                message.ParseFromString(message_data)
                scattered_messages = scatter(message)

                if not scattered_messages:
                    self.wfile.write(b'0')

                else:
                    for scattered in scattered_messages:
                        scattered = _translate_to_broker(scattered)

                        if scattered:
                            broker.send(scattered.SerializeToString())
                            handle_feedback(broker.recv())

                    self.wfile.write(reply_feedback())

        return PalmHandler

    def debug(self):
        """
        Starts the component and serves the http server forever.
        """
        server = HTTPServer((self.hostname, self.port), self._make_handler())
        server.handle_request()

    def start(self):
        """
        Starts the component and serves the http server forever.
        """
        server = HTTPServer((self.hostname, self.port), self._make_handler())
        server.serve_forever()

class CacheService(RepBypassService):
    """
    Cache service for clients and workers
    """
    def recv(self):
        message_data = self.listen_to.recv()
        message = PalmMessage()
        message.ParseFromString(message_data)
        instruction = message.function.split('.')[1]

        if instruction == 'set':
            if message.cache:
                key = message.cache
            else:
                key = str(uuid4())

            self.logger.debug('Cache Service: Set key {}'.format(key))
            value = message.payload
            self.cache.set(key, value)
            return_value = key.encode('utf-8')

        elif instruction == 'get':
            key = message.payload.decode('utf-8')
            self.logger.debug('Cache Service: Get key {}'.format(key))
            value = self.cache.get(key)
            if not value:
                self.logger.error('key {} not present'.format(key))
                return_value = b''
            else:
                return_value = value

        elif instruction == 'delete':
            key = message.payload.decode('utf-8')
            self.logger.debug('Cache Service: Delete key {}'.format(key))
            self.cache.delete(key)
            return_value = key.encode('utf-8')

        else:
            self.logger.error(
                'Cache {}:Key not found in the database'.format(self.name)
            )
            return_value = b''

        if isinstance(return_value, str):
            self.listen_to.send_string(return_value)
        else:
            self.listen_to.send(return_value)


import os
from flask import Flask, flash, request, redirect, url_for, render_template
from werkzeug.utils import secure_filename
import colorgram
import webcolors

app = Flask(__name__)
UPLOAD_FOLDER = 'static/images/'
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
app.config['SECRET_KEY'] = os.environ.get('SECRET_KEY')

ALLOWED_EXTENSIONS = set(['png', 'jpg', 'jpeg', 'gif'])
MAX_COLOURS = 500

def allowed_file(filename):
    return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS

@app.route('/')
def home():
    return render_template('upload_image.html')

@app.route('/', methods=['POST'])
def upload_image():
    if 'file' not in request.files:
        flash('No file part')
        return redirect(request.url)
    file = request.files['file']
    if file.filename == '':
        flash('No image selected for uploading')
        return redirect(request.url)
    if file and allowed_file(file.filename):
        filename = secure_filename(file.filename)
        file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
        flash('Image successfully uploaded and displayed below')
        colors = image_colors(filename)
        return render_template('upload_image.html', filename=filename, colors=colors, color_count=0)
    else:
        flash('Allowed image types are -> png, jpg, jpeg, gif')
        return redirect(request.url)

@app.route('/display/<filename>')
def display_image(filename):
    return redirect(url_for('static', filename='images/' + filename), code=301)

def closest_colour(requested_colour):
    min_colours = {}
    for key, name in webcolors.CSS3_HEX_TO_NAMES.items():
        r_c, g_c, b_c = webcolors.hex_to_rgb(key)
        rd = (r_c - requested_colour[0]) ** 2
        gd = (g_c - requested_colour[1]) ** 2
        bd = (b_c - requested_colour[2]) ** 2
        min_colours[(rd + gd + bd)] = name
    return min_colours[min(min_colours.keys())]

def image_colors(filename):
    rgb_colors = colorgram.extract('static/images/'+filename, MAX_COLOURS)
    colors = {}
    for color in rgb_colors:
        rgb_color = (color.rgb.r, color.rgb.b, color.rgb.b)
        try:
            colors[webcolors.rgb_to_name(rgb_color, spec='css3')] = webcolors.rgb_to_hex(rgb_color)
        except ValueError:
            colors[closest_colour(rgb_color)] = webcolors.rgb_to_hex(rgb_color)
    # print(colors)
    return colors

if __name__ == "__main__":
    app.run(debug=True)


import cv2
import math
import numpy as np

def resize_image(res,img_path,img_name):
    """Resize images over 2500 px in both sides.

    Args:
        img_path: Path to drug image.
        img_name: Image file name.

    Returns:
        r_img: Resized image.
    """
    # Read image file
    img = cv2.imread('{}\{}'.format(img_path,img_name))

    # Resize image if both sides greater than selected resolution
    if (img.shape[1] > res) and (img.shape[0] > res):
        if img.shape[1] > img.shape[0]:
            diff = int(img.shape[1]/res)
        else:
            diff = int(img.shape[0]/res)

        dim = (int(img.shape[1]/diff), int(img.shape[0]/diff))
        print('Image dimensions: ', dim)
        img = cv2.resize(img, dim, interpolation = cv2.INTER_AREA)

    return(img)

def get_contours(img):
    """Get array of contours around drug pack.

    Args:
        img: Drug original or resized image.

    Returns:
        contours: Image contour arrays.
    """
    # Convert color to gray
    # cvt_color = cv2.cvtColor(img,cv2.COLOR_BGR2Lab)

    # Get edges
    # edged = cv2.Canny(cvt_color, 10, 550,L2gradient=True)
    edged = cv2.Canny(img, 10, 550,L2gradient=True)

    # Find and draw external countors
    contours, hierarchy = cv2.findContours(edged, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
    # cv2.drawContours(img, contours, -1, (0, 0, 255), 3)

    return(contours)

def draw_rectangle(img,contours):
    """Get array of rectangle around drug pack.

    Args:
        contours: Drug contour array.

    Returns:
        points: rectangle contour array.
    """
    # Get longest contour
    max_contour = max(contours, key=len)

    # Get minimum rectangle area and get its points
    rect = cv2.minAreaRect(max_contour)
    box = cv2.boxPoints(rect)
    box = np.int0(box)
    points = box.tolist()
    # cv2.drawContours(img,[box],0,(0,255,0),2)

    return(points)

def warp(img,points):
    """Warp current drug image.

    Args:
        img: Current drug image.
        points: Rectangle contour points.

    Returns:
        warp_image: final warped image.
    """
    # Getting hypotenuse for each rectangle size to provide new image dims
    x = int(math.sqrt(((abs(points[0][0]-points[1][0])**2))+((abs(points[0][1]-points[1][1])**2))))
    y = int(math.sqrt(((abs(points[1][0]-points[2][0])**2))+((abs(points[1][1]-points[2][1])**2))))

    pts1 = np.float32([points[2], points[3], points[1], points[0]])
    pts2 = np.float32([[0, 0], [x, 0], [0, y], [x, y]])

    # Apply Perspective Transform Algorithm
    matrix = cv2.getPerspectiveTransform(pts1, pts2)
    wrap_img = cv2.warpPerspective(img, matrix, (x, y))

    return (wrap_img)


from pyspark.sql import SparkSession
from pyspark.sql.types import *
from pyspark.sql import Row
from pyspark.sql.functions import *

if __name__ == '__main__':
    spark = SparkSession.builder.config("spark.driver.binAddress", "localhost").config("spark.ui.port", "4050").appName(
        "dataframe").master("local[*]").getOrCreate()

    dataschema1 = StructType([StructField(name="id", dataType=IntegerType()),
                              StructField("Firstname", StringType()),
                              StructField("Lastname", StringType()),
                              StructField("age", IntegerType()),
                              StructField("Gender", StringType()),
                              StructField("Dept_id", IntegerType()),
                              StructField("Salary", LongType())])

    csvwithschema = spark.read.csv(
        "C:\\Users\\Q\\PycharmProjects\\cloud_engineer_session1\\Dataframe_Intro\\input_withoutheader.csv",
        schema=dataschema1)
    # csvwithschema.printSchema()

    # Select Function

    csvwithschema.printSchema()
    # csvwithschema.show()
    # csvwithschema.cache()
    # csvwithschema.select(csvwithschema.id).show()
    # csvwithschema.select(csvwithschema.id,csvwithschema.age.alias("emp_age")).show()
    # csvwithschema.select(['id','age']).show()
    # csvwithschema.select(col("id"),col("age")).show()
    # csvwithschema.select(csvwithschema.columns[:3]).show()

    ##Select With Nested Data
    # nesteddf = spark.read.format("json").load(r"C:\Users\Q\PycharmProjects\cloud_engineer_session1\Dataframe_Intro\nesteddata.json")
    # nesteddf.printSchema()
    # nesteddf.show()
    # nesteddf.select("Address.*").show()
    # nesteddf.select((["Address.City","Address.State"])).show()

    # print(nesteddf.columns)

    ## WithColumn
    ##Existing Col value change
    # csvwithschema.withColumn("salary",col("salary")*10).show()
    # csvwithschema.withColumn("bonus-salary", col("salary") * 10).show()

    ##Change datatype of Exist Col
    # csvwithschema.withColumn("salary",col("salary").cast("string")).printSchema()

    ##Adding new column
    # csvwithschema.withColumn("state",lit("MH")).withColumn("country",lit("India")).show()

    ##withColumnRenamed

    # csvwithschema.withColumnRenamed("id","identifier").show()

    ##Filter
    # csvwithschema.filter(col("Gender")=="M").show()
    # csvwithschema.where((col("Gender")=="M") & (col("Salary")>="25000")).show()

    ##Distinct
    # csvwithschema.distinct().show()
    # dup= spark.read.csv(r"C:\Users\Q\PycharmProjects\cloud_engineer_session1\Dataframe_Intro\dupl.csv")
    # dup.show()
    # dup.distinct().show()

    ##Drop Duplicate ,Drop
    data = [Row(name="Ajay",Age=20),
            Row(name='Satish',Age=25),
            Row(name="Ajay",Age=20),
            Row(name="Ajay",Age=30)]
    # df1= spark.sparkContext.parallelize(data).toDF()
    # df1.show()
    # df1.printSchema()
    # df1.distinct().show()

    # df1.dropDuplicates(['name']).show()

    # df1.drop('age').show()

    ##Groupby() --Aggregate Function

    # csvwithschema.groupby("dept_id").avg("salary").show()
    # csvwithschema.groupby("dept_id").sum("salary").show()
    # csvwithschema.groupBy("dept_id").count().show()
    # csvwithschema.groupby("dept_id").max("salary").show()
    # csvwithschema.groupby("dept_id").mean("salary").show()
    #  csvwithschema.groupby("dept_id").agg(sum("salary").alias("sum_sal")).show()

    data1 = [Row(dept_no=11, dept_name='HR'),
             Row(dept_no=12, dept_name='ADMIN'),
             Row(dept_no=15, dept_name='IT')]
    deptdf = spark.sparkContext.parallelize(data1).toDF()
    # deptdf.printSchema()
    # deptdf.show()

    #Inner Join
    # csvwithschema.join(deptdf,on=csvwithschema.Dept_id == deptdf.dept_no,how='inner').select("id","Firstname","Lastname","age","Gender","Salary","dept_name").show()

    # cond= [csvwithschema.Dept_id == deptdf.dept_no]
    # csvwithschema.join(deptdf,cond,'inner').show()

    #Full Outer
    # csvwithschema.join(deptdf, cond, 'fullouter').show()

    #Left Join
    # csvwithschema.join(deptdf, cond, 'leftouter').show()

    # Right Join
    # csvwithschema.join(deptdf, cond, 'rightouter').show()

    # Anti-Left Join
    # csvwithschema.join(deptdf, cond, 'left_anti').show()

    # LeftSemi Join
    # csvwithschema.join(deptdf, cond, 'semi').show()

    ##Union

    # data2 = [Row(dept_no=11, dept_name='HR'),
    #          Row(dept_no=14, dept_name='Production'),
    #          Row(dept_no=13, dept_name='Sales')]
    # deptdf1 = spark.sparkContext.parallelize(data2).toDF()
    # # deptdf1.show()
    # # deptdf1.printSchema()
    #
    # deptdf.union(deptdf1).show()
    # deptdf.unionAll(deptdf1).show()
    # deptdf.union(deptdf1).distinct().show()


import cv2
import sys

face_cascade = cv2.CascadeClassifier('lbpcascade_frontalface.xml')
vc = cv2.VideoCapture('sample.mp4')

if vc.isOpened():
    rval , frame = vc.read()
else:
    rval = False
    roi = [0,0,0,0]

while rval:
    rval, frame = vc.read()

    # resize frame for speed.
    frame = cv2.resize(frame, (300,200))

    # face detection.
    faces = face_cascade.detectMultiScale(frame, 1.8, 2)
    nfaces = 0
    for (x,y,w,h) in faces:
        cv2.rectangle(frame,(x,y),(x+w,y+h),(0,0,255),2)
        nfaces = nfaces + 1
        roi = [x,y,w,h]

    # undetected face, show old on position.
    if nfaces == 0:
        cv2.rectangle(frame,(roi[0],roi[1]),(roi[0]+roi[2],roi[1]+roi[3]),(0,0,255),2)

    # show result
    cv2.imshow("Result",frame)
    cv2.waitKey(1);
    vc.release()


from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from sys import platform
import time
import shutil
import json
import os
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait

###########################################################################################
# Selects the correct chromedriver to run depending on the OS
###########################################################################################

if platform == "darwin":
    driver_path = "./chromedriver"
    # OS X
elif platform == "win32":
    driver_path = "./chromedriver.exe"
    # Windows...
else:
    driver_path = "ERROR"

print(platform)

###########################################
# GLOBAL DRIVER OPTIONS AND PATH
# Current options applied: Hides pop-up for browser
###########################################
chrome_options = Options()
chrome_options.headless = True
service = Service(executable_path=driver_path)

class diningHallWebScrapper:
    def __init__(self):
        self.isFirstInput = True  # Used to ensure that first JSON entry of a food entry does not contain a , at the start
        self.all_foods = {}  # Contains all the food items with their properties
        self.switch_for_dining_hall = {
            'College Nine/John R. Lewis Dining Hall': 'clgnine_johnlewis',
            'Cowell/Stevenson Dining Hall': 'cowell_steve',
            'Crown/Merrill Dining Hall': 'crown_merill',
            'Porter/Kresge Dining Hall': 'porter_kresge'
        }
        self.dining_halls = ['College Nine/John R. Lewis Dining Hall', 'Cowell/Stevenson Dining Hall',
                    'Crown/Merrill Dining Hall', 'Porter/Kresge Dining Hall']

        ###########################################################################################
        # Setup for the Webscrapper.
        # Sets start page to https://nutrition.sa.ucsc.edu/
        ###########################################################################################

        global service
        #self.driver = webdriver.Chrome(service=service, options=chrome_options) #Comment if (driver = ...) is uncommented
        self.driver = webdriver.Chrome(service=service) #Uncomment to see browser

        self.driver.get("https://nutrition.sa.ucsc.edu/")  # Starting website

    def getDiningHallUrl(self):
        return self.driver.current_url

    def startDiningHallScrapper(self):

        for curr_dining_hall in self.dining_halls:
            try:
                self.moveToNutritionPage(curr_dining_hall)
                self.webScrapAllMealsTimes(curr_dining_hall)
            except:
                print("Failed to extract " + curr_dining_hall + " dining hall")
                continue

        print("Total menu items scrapped: ", str(len(self.all_foods)))
        # driver.get("https://nutrition.sa.ucsc.edu/")
        self.endDiningHallScrapper()

    def endDiningHallScrapper(self):
        self.driver.quit()

    def createFoodResultsJson(self):
        try:
            self.removeCurrentJSON()
            self.convertToJSON(self.all_foods)
            self.moveToResultsFolder()
            return True
        except:
            return False

    def checkFoodResultsPath(self):
        if os.path.exists("./results/food_results.json"):
            return True
        return False

    def removeCurrentJSON(self):
        if self.checkFoodResultsPath():
            os.remove("./results/food_results.json")

    def moveToResultsFolder(self):
        shutil.move('./food_results.json', './results')

    # Converts the list of dictionaries for split_foods into a JSON file based on the dining hall
    def convertToJSON(self,all_foods):
        file_name = "./food_results.json"
        ofstrm = open(file_name, 'w+')
        size = len(all_foods)
        count = 0

        ofstrm.write('[\n')
        if self.isFirstInput:
            self.isFirstInput = False
        else:
            ofstrm.write(",")
        for key, value in all_foods.items():
            json.dump(value, ofstrm, indent=4)
            count += 1
            if count == size:
                ofstrm.write("\n")
            if count != size:
                ofstrm.write(",\n")
        ofstrm.write(']')
        ofstrm.close()

    def resetDiningHallPage(self):
        self.driver.get("https://nutrition.sa.ucsc.edu/")  # Resets the page incase url is different

    # Starting from the main dining hall page, the driver moves to the nutrition page for all the food of that dining hall
    def moveToNutritionPage(self,dining_hall):
        try:
            self.resetDiningHallPage()

            dining_hall_link = WebDriverWait(self.driver, timeout=3).until(
                lambda d: d.find_element(By.LINK_TEXT, dining_hall))
            dining_hall_link.click()
            return True
        except:
            return False

    def __getMealTimeLinks(self):
        # Setup: Obtaining len and name of meal times(Breakfast, Lunch, Dinner, Late_Night)
        nutrition_pages = WebDriverWait(self.driver, timeout=3).until(
            lambda d: d.find_elements(By.CLASS_NAME, "shortmenunutritive"))

        meal_time_elems = WebDriverWait(self.driver, timeout=3).until(
            lambda d: d.find_elements(By.CLASS_NAME, "shortmenumeals"))

        meal_times = []
        for meal_time in meal_time_elems:
            meal_times.append(meal_time.text)
        return meal_times

    # Calls each meal time for the specific dining hall and then calls getNutritionFacts() to start scrapping all the data
    def webScrapAllMealsTimes(self,dining_hall):
        try:
            # Setup: Obtaining len and name of meal times(Breakfast, Lunch, Dinner, Late_Night)
            meal_times = self.__getMealTimeLinks()
            total_meal_times = len(meal_times)

            # Webscrapper starts below this point using the data from setup:
            for curr_meal_type in range(total_meal_times):
                nutrition_pages = WebDriverWait(self.driver, timeout=3).until(
                    lambda d: d.find_elements(By.CLASS_NAME, "shortmenunutritive"))
                nutrition_pages[curr_meal_type].click()
                curr_meal_time = meal_times[curr_meal_type]
                if curr_meal_time == 'Late Night':  # Changed to Late_Night to avoid space conflicts
                    curr_meal_time = 'Late_Night'
                self.__getNutritionFacts(dining_hall, curr_meal_time)
            return True
        except:
            return False

    def __getFoodType(self, raw_food_type_text):
        REMOVE_FRONT_DASH = 3
        REMOVE_BACK_DASH = len(raw_food_type_text) - 3
        parsed_food_type = raw_food_type_text[REMOVE_FRONT_DASH:REMOVE_BACK_DASH]
        return parsed_food_type

    def __init_basic_food_obj(self,current_food_type,food_name):
        food_obj = {
            'food_type': current_food_type,
            'food_name': food_name,
            'dining_hall': {
                'clgnine_johnlewis': False,
                'cowell_steve': False,
                'crown_merill': False,
                'porter_kresge': False
            },
            'meal_time': {
                'Breakfast': False,
                'Lunch': False,
                'Dinner': False,
                'Late_Night': False
            }
        }
        self.all_foods[food_name] = food_obj
        return food_obj

    ###########################################################################################
    # Get functions for all nutrient/tag info about a specific food item
    # All get functions are called within singleNutrientFacts()
    ###########################################################################################

    # Used as an helper for the getter functions. Returns elem based on XPATH given
    def __searchNutritionFactBoxHTML(self,xpath_val):
        try:
            result_elem = WebDriverWait(self.driver, timeout=0.5).until(
                lambda d: d.find_element(By.XPATH, xpath_val))
            return result_elem.text.lstrip()
        except:
            return None

    def __getServing_size(self,curr_food_obj):
        curr_food_obj['serving_size'] = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[1]/td[1]/font[3]')

    def __getCalories(self,curr_food_obj):
        try:
            calories_text = WebDriverWait(self.driver, timeout=0.5).until(
                lambda d: d.find_element(By.XPATH, '/html/body/table[1]/tbody/tr/td/table/tbody/tr[1]/td[1]/font[4]'))

            calories_arr = calories_text.text.split(' ')

            NO_CALORIES_FOUND_BEYOND_2 = 2

            if len(calories_arr) < NO_CALORIES_FOUND_BEYOND_2:  # No Calories value
                curr_food_obj['calories'] = None
            else:
                curr_food_obj['calories'] = calories_arr[1]
        except:
            curr_food_obj['calories'] = None

    def __getTotalFat(self,curr_food_obj):
        curr_food_obj['total_fat'] = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[2]/td[1]/font[2]')

    def __getTotalFat_DV(self,curr_food_obj):
        result = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[2]/td[2]/font[1]/b')
        curr_food_obj['total_fat_DV'] = None if result == '' else result

    def __getSatFat(self,curr_food_obj):
        curr_food_obj['sat_fat'] = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[3]/td[1]/font[2]')

    def __getSatFat_DV(self,curr_food_obj):
        result = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[3]/td[2]/font[1]/b')
        curr_food_obj['sat_fat_DV'] = None if result == '' else result

    def __getTransFat(self,curr_food_obj):
        curr_food_obj['trans_fat'] = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[4]/td[1]/font[2]')

    def __getCholesterol(self,curr_food_obj):
        curr_food_obj['cholesterol'] = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[5]/td[1]/font[2]')

    def __getCholesterol_DV(self,curr_food_obj):
        result = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[5]/td[2]/font[1]/b')
        curr_food_obj['cholesterol_DV'] = None if result == '' else result

    def __getSodium(self,curr_food_obj):
        curr_food_obj['sodium'] = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[6]/td[1]/font[2]')

    def __getSodiumDV(self,curr_food_obj):
        result = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[6]/td[2]/font[1]/b')
        curr_food_obj['sodium_DV'] = None if result == '' else result

    def __getTotalCarbs(self,curr_food_obj):
        curr_food_obj['total_carbs'] = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[2]/td[3]/font[2]')

    def __getTotalCarbs_DV(self,curr_food_obj):
        result = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[2]/td[4]/font[1]/b')
        curr_food_obj['total_carbs_DV'] = None if result == '' else result

    def __getDietaryFiber(self,curr_food_obj):
        result = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[3]/td[3]/font[2]')
        curr_food_obj['dietary_fiber'] = '- - - g' if result is None else result

    def __getDietaryFiber_DV(self,curr_food_obj):
        result = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[3]/td[4]/font[1]/b')
        curr_food_obj['dietary_fiber_DV'] = None if result == '' else result

    def __getSugars(self,curr_food_obj):
        result = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[4]/td[3]/font[2]')
        curr_food_obj['sugars'] = '- - - g' if result is None else result

    def __getProtein(self,curr_food_obj):
        curr_food_obj['protein'] = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[5]/td[3]/font[2]')

    def __getIngredients(self,curr_food_obj):
        curr_food_obj['ingredients'] = self.__searchNutritionFactBoxHTML('/html/body/table[2]/tbody/tr/td/span[2]')

    def __getAllergens(self,curr_food_obj):
        result = self.__searchNutritionFactBoxHTML('/html/body/table[3]/tbody/tr/td/span[2]')
        curr_food_obj['allergens'] = None if result == '' else result

    def __getVitamin_D(self,curr_food_obj):
        result = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[7]/td/table/tbody/tr/td[1]/table/tbody/tr/td/font[2]')
        curr_food_obj['vitamin_d'] = None if result == '' else result

    def __getCalcium(self,curr_food_obj):
        result = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[7]/td/table/tbody/tr/td[2]/table/tbody/tr/td/li/font[2]')
        curr_food_obj['calcium'] = None if result == '' else result

    def __getIron(self,curr_food_obj):
        result = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[7]/td/table/tbody/tr/td[3]/table/tbody/tr/td/li/font[2]')
        curr_food_obj['iron'] = None if result == '' else result

    def __getPotassium(self,curr_food_obj):
        result = self.__searchNutritionFactBoxHTML(
            '/html/body/table[1]/tbody/tr/td/table/tbody/tr[7]/td/table/tbody/tr/td[4]/table/tbody/tr/td/li/font[2]')
        curr_food_obj['potassium'] = None if result == '' else result

    def __getFoodTags(self,curr_food_obj):
        tmp = {
            'Eggs': False,
            'Fish': False,
            'GlutenFree': False,
            'Dairy': False,
            'Peanuts': False,
            'Soy': False,
            'Vegan': False,
            'Vegetarian': False,
            'Pork': False,
            'Beef': False,
            'Halal': False,
            'Shellfish': False,
            'TreeNut': False

        }

        try:
            food_tag_elem = WebDriverWait(self.driver, timeout=0.5).until(
                lambda d: d.find_element(By.CLASS_NAME, 'labelwebcodesvalue'))

            all_tags = WebDriverWait(food_tag_elem, timeout=0.5).until(
                lambda d: d.find_elements(By.TAG_NAME, 'img'))

            curr_food_obj['food_tags'] = {

            }

            for curr_tag in all_tags:
                curr_tag_text = curr_tag.get_attribute('alt')
                if curr_tag_text == 'Egg/s':  # Egg/s changed to Eggs to avoid a possible URL path issue
                    tmp['Eggs'] = True
                elif curr_tag_text == 'Gluten Free' or curr_tag_text == 'Gluten-Free':
                    tmp['GlutenFree'] = True
                elif curr_tag_text == 'Tree Nut':
                    tmp['TreeNut'] = True
                else:
                    tmp[curr_tag_text] = True

            curr_food_obj['food_tags'] = tmp
        except:
            curr_food_obj['food_tags'] = tmp

    # Starting from the food item nutrition page, all nutrition facts are scrapped
    def singleNutritionFact(self,curr_food_obj):
        self.__getServing_size(curr_food_obj)
        self.__getCalories(curr_food_obj)
        self.__getTotalFat(curr_food_obj)
        self.__getSatFat(curr_food_obj)
        self.__getTransFat(curr_food_obj)
        self.__getCholesterol(curr_food_obj)
        self.__getSodium(curr_food_obj)
        self.__getTotalCarbs(curr_food_obj)
        self.__getDietaryFiber(curr_food_obj)
        self.__getSugars(curr_food_obj)
        self.__getProtein(curr_food_obj)
        self.__getIngredients(curr_food_obj)
        self.__getAllergens(curr_food_obj)
        self.__getVitamin_D(curr_food_obj)
        self.__getCalcium(curr_food_obj)
        self.__getIron(curr_food_obj)
        self.__getPotassium(curr_food_obj)
        self.__getFoodTags(curr_food_obj)
        self.__getTotalFat_DV(curr_food_obj)
        self.__getSatFat_DV(curr_food_obj)
        self.__getCholesterol_DV(curr_food_obj)
        self.__getSodiumDV(curr_food_obj)
        self.__getTotalCarbs_DV(curr_food_obj)
        self.__getDietaryFiber_DV(curr_food_obj)

    def __start_nutrient_fact_webscrapper(self, curr_foods):
        # Gets the nutrient facts for all food items that have just recently been added to all_foods{}.
        # Recent food names are in curr_foods[]
        for count in range(len(curr_foods)):
            link = WebDriverWait(self.driver, timeout=3).until(
                lambda d: d.find_element(By.LINK_TEXT, curr_foods[count]['food_name']))
            link.click()
            self.singleNutritionFact(curr_foods[count])
            self.driver.back()  # Returns to list of meal time page

    # Must call moveToNutritionPage() before calling this function.
    # Scraps all information(food genre, name, nutrition facts,etc) and writes it to a JSON file

    def __getNutritionFacts(self,dining_hall, meal_time):
        food_table = WebDriverWait(self.driver, timeout=3).until(lambda d: d.find_element(By.XPATH, '//table[@width="70%"]'))

        inner_table = WebDriverWait(food_table, timeout=3).until(lambda d: d.find_elements(By.TAG_NAME, 'table'))
        size = len(inner_table) - 1

        current_food_type = ''
        curr_foods = []
        for i in range(1, size, 2):
            try:
                try:  # Result is the food type. Parses the string and sets current_food_type
                    raw_food_type = inner_table[i].find_element(By.CLASS_NAME, "longmenucolmenucat")
                    current_food_type = self.__getFoodType(raw_food_type.text)

                except:  # Result is the food name. Checks if food_name is in all_foods{}.
                    # If not, create new obj with default parameters. Else update the object.
                    food_name = inner_table[i].find_element(By.CLASS_NAME, "longmenucoldispname")
                    food_obj = self.all_foods.get(food_name.text)
                    if food_obj is None:
                        food_obj = self.__init_basic_food_obj(current_food_type,food_name.text)
                        curr_foods.append(food_obj)
                    set_dining_hall = self.switch_for_dining_hall.get(dining_hall)
                    food_obj['dining_hall'][set_dining_hall] = True
                    food_obj['meal_time'][meal_time] = True
            except:
                print("FATAL ERROR: FOOD NAMES PARSED INCORRECTLY")

        self.__start_nutrient_fact_webscrapper(curr_foods)

        print("Webscrap " + dining_hall + ": " + meal_time + " Successful")
        self.driver.back()  # Returns back to dining hall's food list page


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Feb 24 13:57:10 2021

@author: akim
"""

import numpy as np
from NEK5000_func_lib import particleCoordsNew, fast_scandir, listfile, find_in_list_of_list, build_matrix
from os import listdir
import matplotlib.pyplot as plt
import itertools
from math import floor

dirpath = '../'
fold_name = 'fbala'
# dirpath = '/home/afabret/data/room_deposition/production_run/'
# fold_name = 'roomBackUp'

folders = fast_scandir(dirpath)
folders = [word for word in folders if fold_name in word]
folders.sort()

listOfFileList, allFileList = listfile(folders)

path1 = folders[0] + '/'

def pathTwo (ind):
    try:
        path2Ind = find_in_list_of_list(listOfFileList, allFileList[ind])
        path2 = folders[path2Ind[0]]
    except IndexError:
        path2Ind = find_in_list_of_list(listOfFileList, allFileList[-1])
        path2 = folders[path2Ind[0]]
    return path2 + '/'

step = 2
R = floor(len(allFileList)/step)
for i in range(R):
    print(i)
    file2 = 0 + i*step
    inpt =['s', 0, file2, 5, path1, pathTwo(file2), allFileList]
    t1, t2, B = build_matrix(*inpt)
    print(t2)
    # print(len(B), B.max())
    normalized = B/B.max()  # rescale to between 0 and 1
    corrected = np.power(normalized, 0.5) # try values between 0.5 and 2 as a start point
    # print(B)
    if inpt[0] == 'a':
        title = 'All particles' + ' t1 = ' + str(t1) +  ', t2 = ' + str(t2)
        tit = 'matrix_all' + '_'+ str(int(t1)) +  '_' + str(int(t2))
        heatmap = plt.pcolor(np.array(((0,1),(0,1))))
        # heatmap = plt.pcolor(B)
    elif inpt[0] == 's':
        title = 'Sphere'  + ' t1 = ' + str(t1) +  ', t2 = ' + str(t2)
        tit = 'matrix_sphere'  + '_'+ str(int(t1)) +  '_' + str(int(t2))
        heatmap = plt.pcolor(np.array(((0,30),(0,30))))
    fig, ax = plt.subplots(figsize=(7, 7))
    im = ax.imshow(corrected)
    ax.set_title(title)
    fig.tight_layout()
    plt.colorbar(heatmap)
    plt.savefig(tit + '.png', dpi=200)
    np.savetxt((tit + '.txt'), B)


import os
import sys
import site

# Add the site-packages of the chosen virtualenv to work with
site.addsitedir('/var/www/venv/lib/python2.7/site-packages')

INTERP = os.path.expanduser("/var/www/venv/bin/python")

if sys.executable != INTERP: os.execl(INTERP, INTERP, *sys.argv)

sys.path.insert(0,'/var/www/venv/bin')
sys.path.insert(0,'/var/www/venv/lib/python2.7/site-packages/django')
sys.path.insert(0,'/var/www/venv/lib/python2.7/site-packages')

# Add the app's directory to the PYTHONPATH
sys.path.append('/var/www/string_parser')

os.environ['DJANGO_SETTINGS_MODULE'] = 'string_parser.settings'

# Activate your virtual env
activate_env=os.path.expanduser("/var/www/venv/bin/activate_this.py")
execfile(activate_env, dict(__file__=activate_env))

from django.core.wsgi import get_wsgi_application
application = get_wsgi_application()


# External modules
from colorama import init, Fore
import time
import os
# Local modules
import crud

def clear():
    os.system("cls")

init(autoreset= True)

def validation(msg, a, b):
    while True:
        clear()
        try:
            logo()
            print("\n" + msg + "\n")
            opc = int(input("Ingrese una opcion: "))
            if opc >= a and opc <= b:
                return opc
            else:
                clear()
                print(Fore.RED + "[ X ] Oops!, Opcion invalida... Intente de nuevo.")
                time.sleep(2)
                clear()

        except ValueError:
            clear()
            print(Fore.RED + "[ X ] Oops!, Opcion invalida... Intente de nuevo.")
            time.sleep(2)
            clear()

def logo():
    print("\t\t" + "#"*65, end="\n\t\t#" + " "*63 + "#\n")
    print("\t\t#\t" + Fore.RED             + " /$$   /$$  /$$$$$$  /$$   /$$  /$$$$$$  /$$       " + Fore.RESET +"\t#")
    print("\t\t#\t" + Fore.BLUE            + "| $$  | $$ /$$__  $$| $$$ | $$ /$$__  $$| $$       " + Fore.RESET +"\t#")
    print("\t\t#\t" + Fore.YELLOW          + "| $$  | $$| $$  \ $$| $$$$| $$| $$  \ $$| $$       " + Fore.RESET +"\t#")
    print("\t\t#\t" + Fore.MAGENTA         + "| $$  | $$| $$$$$$$$| $$ $$ $$| $$$$$$$$| $$       " + Fore.RESET +"\t#")
    print("\t\t#\t" + Fore.CYAN            + "| $$  | $$| $$__  $$| $$  $$$$| $$__  $$| $$       " + Fore.RESET +"\t#")
    print("\t\t#\t" + Fore.GREEN           + "| $$  | $$| $$  | $$| $$\  $$$| $$  | $$| $$       " + Fore.RESET +"\t#")
    print("\t\t#\t" + Fore.WHITE           + "|  $$$$$$/| $$  | $$| $$ \  $$| $$  | $$| $$$$$$$$ " + Fore.RESET +"\t#")
    print("\t\t#\t" + Fore.LIGHTMAGENTA_EX + " \______/ |__/  |__/|__/  \__/|__/  |__/|________/ " + Fore.RESET +"\t#", end="\n\t\t#" + " "*63 + "#\n")
    print("\t\t" + "#"*65)

def menu():
    msg = "[ 1 ] Crear\n[ 2 ] Mostrar\n[ 3 ] Modificar\n[ 4 ] Eliminar\n[ 5 ] Salir"

    opc = validation(msg, 1, 5)

    if opc == 1:
        crear()
    elif opc == 2:
        mostrar()
    elif opc == 3:
        modificar()
    elif opc == 4:
        eliminar()
    else:
        exit()

    return opc

def salir_regresar(funcion):
    opc = int(input("\n[1] Regresar\t[2] Salir\n\nSeleccione una opcion: "))
    if opc == 1:
        funcion()
    else:
        exit()

def crear():
    msg = "[ 1 ] Registrar Alumno\n[ 2 ] Registrar Trabajador\n[ 3 ] Dar de alta un profesor\n[ 4 ] Regresar"
    opc = validation(msg, 1, 4)

    clear()
    if opc == 1:
        app.CrearAlumno()
        time.sleep(1)
    elif opc == 2:
        app.CrearTrabajador()
    elif opc == 3:
        app.crearProfesor()
    else:
        menu()

    salir_regresar(crear)

def mostrar():
    msg = "[ 1 ] Mostrar Alumnos\n[ 2 ] Mostrar todos los trabajadores\n[ 3 ] Mostrar Profesores\n[ 4 ] Mostrar Facultades\n[ 5 ] Regresar"
    opc = validation(msg, 1, 6)

    clear()
    if opc == 1:
        print("Mostrando tabla de Alumnos...\n\n")
        time.sleep(2)
        app.mostrarAlumnos()
    elif opc == 2:
        print("Mostrando tabla de Trabajadores...\n\n")
        time.sleep(2)
        app.mostrarTrabajadores()
    elif opc == 3:
        print("Mostrando tabla de Profesores...\n\n")
        time.sleep(2)
        app.mostrarProfesor()
    elif opc == 4:
        print("Mostrando tabla de Facultades...\n\n")
        time.sleep(2)
        app.mostrarFacultades()
    else:
        menu()

    salir_regresar(mostrar)

def modificar():
    msg = "[ 1 ] Modificar datos de alumno\n[ 2 ] Modificar datos de trabajador\n[ 3 ] Regresar"
    opc = validation(msg, 1, 4)

    if opc == 1:
        pass
    elif opc == 2:
        pass
    else:
        modificar()

    salir_regresar(modificar)

def eliminar():
    msg = "[ 1 ] Eliminar un alumno\n[ 2 ] Eliminar trabajador\n[ 4 ] Regresar"
    opc = validation(msg, 1, 4)

    if opc == 1:
        pass
    elif opc == 2:
        pass
    else:
        eliminar()

    salir_regresar(eliminar)

if __name__ == '__main__':

    # Inicializar Crud
    app = crud.db()

    menu()

    app.close_connection()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Remove all temporary files in a project's folder.

Scan the current directory for the file build_rules.py and look for
the function ``clean`` and call it.

Full documentation on the operation of [``cleanme`` is here](cleanme_man.md).

See Also:
    main, makeprojects.buildme, makeprojects.rebuildme

@package makeprojects.cleanme
"""

# pylint: disable=useless-object-inheritance
# pylint: disable=consider-using-f-string

from __future__ import absolute_import, print_function, unicode_literals

import os
import sys
import argparse
from operator import attrgetter
from burger import convert_to_array, import_py_script, norm_paths
from .config import BUILD_RULES_PY, save_default, _XCODEPROJECT_FILE, \
    _XCODEPROJ_MATCH
from .__init__ import __version__
from .util import get_build_rules, getattr_build_rules_list, was_processed, \
    fixup_args, clear_build_rules_cache, getattr_build_rules
from .build_objects import BuildError
from .modules import MODULES
from .python import create_clean_rules_objects, BuildPythonFile

########################################

def create_parser():
    """
    Create the parser to process the command line for buildme

    The returned object has these member variables

    - version boolean if version is requested
    - recursive boolean for directory recursion
    - verbose boolean for verbose output
    - preview boolean for previewing the clean process
    - generate_build_rules boolean create build rules and exit
    - rules_file string override build_rules.py
    - fatal boolean abort if error occurs in processing
    - directories string array of directories to process
    - files string array of project files to process
    - configurations string array of configurations to process
    - args string array of unknown parameters

    Returns:
        argparse.ArgumentParser() object
    """

    # Parse the command line
    parser = argparse.ArgumentParser(
        description="Remove project output files. "
        "Copyright by Rebecca Ann Heineman")

    parser.add_argument("--version", action="version",
                        version="%(prog)s " + __version__)

    parser.add_argument("-r", "-all", dest="recursive", action="store_true",
                        default=False, help="Perform a recursive clean")

    parser.add_argument("-n", "-preview", dest="preview", action="store_true",
                        default=False, help="Preview clean commands.")

    parser.add_argument("-v", "-verbose", dest="verbose", action="store_true",
                        default=False, help="Verbose output.")

    parser.add_argument("--generate-rules", dest="generate_build_rules",
                        action="store_true", default=False,
                        help="Generate a sample configuration file and exit.")

    parser.add_argument(
        "--rules-file",
        dest="rules_file",
        metavar="<file>",
        default=BUILD_RULES_PY,
        help="Specify a configuration file.")

    parser.add_argument("-q", dest="fatal", action="store_true",
                        default=False, help="Quit immediately on any error.")

    parser.add_argument("-f", dest="files", action="append",
                        metavar="<filename>",
                        help="Project file to process.")

    parser.add_argument("-d", dest="directories", action="append",
                        metavar="<directory>", default=[],
                        help="List of directories to clean.")

    parser.add_argument("-c", dest="configurations", action="append",
                        metavar="<configuration>",
                        help="Configuration to process.")

    parser.add_argument("args", nargs=argparse.REMAINDER,
                        help="project filenames")

    return parser

########################################

def add_clean_rules(projects, working_directory,
                    file_name, args, build_rules=None):
    """
    Add a build_rules.py to the clean list.

    Given a build_rules.py to parse, check for the function "clean"
    and use that for scanning for functions to call.

    All valid entries will be appended to the projects list.

    Args:
        projects: List of projects to clean.
        working_directory: Directory to perform actions on
        file_name: Pathname to the build_rules.py file.
        args: Args for determining verbosity for output.
        build_rules: Preloaded build_rules.py object.
    See Also:
        add_project
    """

    # Was the build_rules already loaded?
    if not build_rules:
        build_rules = import_py_script(file_name)

    dependencies = []

    # Was a build_rules.py file found?
    if build_rules:
        if args.verbose:
            print("Using configuration file {}".format(file_name))

        # Test for functions and append all that are found
        if working_directory is None:
            working_directory = os.path.dirname(file_name)
        parms = {
            "working_directory": working_directory}

        # Get the dependency list, if any
        dependencies = getattr_build_rules(
            build_rules, ("CLEANME_DEPENDENCIES", "DEPENDENCIES"), None)[0]

        if dependencies:
            # Ensure all paths are normalized
            dependencies = norm_paths(
                working_directory,
                convert_to_array(dependencies))
        else:
            dependencies = []

        # Add build object found in the build_rules.py file
        projects.extend(
            create_clean_rules_objects(
                file_name,
                build_rules,
                parms,
                args.verbose))
    return dependencies

########################################

def add_project(projects, processed, file_name, args):
    """
    Detect the project type and add it to the list.

    Args:
        projects: List of projects to build.
        processed: List of directories already processed.
        file_name: Pathname to the build_rules.py file.
        args: Args for determining verbosity for output.
    Returns:
        True if the file was buildable, False if not.
    """

    # pylint: disable=too-many-return-statements
    # pylint: disable=too-many-branches

    # Check if the file is accepted by a build module
    for module in MODULES:

        # Match the file?
        if module.match(file_name):

            # Test for recursion
            if was_processed(processed, file_name, args.verbose):
                return True

            # Create the build objects
            projects.extend(
                module.create_clean_object(
                    file_name,
                    configurations=args.configurations,
                    verbose=args.verbose))
            return True

    return False

########################################

def process_projects(results, projects, args):
    """
    Process a list of projects

    Sort the projects by priority and build all of them.
    """
    # Sort the list by priority (The third parameter is priority from 1-99)
    error = 0
    projects = sorted(projects, key=attrgetter("priority"))

    # If in preview mode, just show the generated build objects
    # and exit
    if args.preview:
        for project in projects:
            print(project)
        return False

    # Clean all the projects
    # Note, python objects are a special case, if any return a
    # non "None" error code, don't call any others

    python_none = False
    for project in projects:

        # Skip all python clean objects if a numeric error code
        # was already obtained
        if isinstance(project, BuildPythonFile) and python_none:
            continue

        berror = project.clean()
        error = 0
        if berror is not None:
            results.append(berror)
            error = berror.error

            # Check the error code, was it None?
            if error is not None:
                if isinstance(project, BuildPythonFile):
                    # Don't parse any more python "clean" scripts
                    python_none = True

        # Abort on error?
        if error and args.fatal:
            return True
    return False

########################################

def process_files(results, processed, files, args):
    """
    Process a list of files.
    """
    projects = []
    for item in files:
        base_name = os.path.basename(item)
        if base_name == args.rules_file:
            if not was_processed(processed, item, args.verbose):
                process_dependencies(
                    results, processed, add_clean_rules(
                        projects, None, item, args), args)
        elif not add_project(projects, processed, item, args):
            print("\"{}\" is not supported.".format(item))
            return True
    return process_projects(results, projects, args)

########################################

def process_directories(results, processed, directories, args):
    """
    Process a list of directories.

    Args:
        results: list object to append BuildError objects
        processed: List of directories already processed.
        directories: iterable list of directories to process
        args: parsed argument list for verbosity
    Returns:
        True if processing should abort, False if not.
    """

    # pylint: disable=too-many-branches
    # pylint: disable=too-many-nested-blocks

    # Process the directory list
    for working_directory in directories:

        # Was this directory already processed?
        if was_processed(processed, working_directory, args.verbose):
            # Technically not an error to abort processing, so skip
            continue

        # Only process directories
        if not os.path.isdir(working_directory):
            msg = "{} is not a directory.".format(working_directory)
            results.append(BuildError(10, working_directory, msg=msg))
            if args.fatal:
                return True
            continue

        # Are there build rules in this directory?
        build_rules_list = get_build_rules(
            working_directory, args.verbose, args.rules_file, "CLEANME")

        # Is recursion allowed?
        allow_recursion = not getattr_build_rules_list(
            build_rules_list, ("CLEANME_NO_RECURSE", "NO_RECURSE"), False)

        allow_files = getattr_build_rules_list(
            build_rules_list, ("CLEANME_PROCESS_PROJECT_FILES", "PROCESS_PROJECT_FILES"), True)

        # Pass one, create a list of all projects to build
        projects = []

        # Process all of the dependencies first, then this folder
        for build_rules in build_rules_list:
            process_dependencies(
                results,
                processed,
                add_clean_rules(
                    projects,
                    working_directory,
                    build_rules.__file__,
                    args,
                    build_rules),
                args)

        # Iterate over the directory to find all the other files
        if allow_files or (args.recursive and allow_recursion):
            for entry in os.listdir(working_directory):

                full_name = os.path.join(working_directory, entry)

                # If it's a directory, check for recursion
                if os.path.isdir(full_name):

                    # Special case for xcode, if it's a *.xcodeproj
                    if allow_files and _XCODEPROJ_MATCH.match(entry):

                        # Check if it's an xcode project file, if so, add it
                        if not add_project(projects, processed, os.path.join(
                                full_name, _XCODEPROJECT_FILE), args):
                            print(
                                ("\"{}\" is not supported on this platform.")
                                .format(full_name))
                            return True
                        continue

                    if args.recursive and allow_recursion:
                        # Process the directory first
                        if process_directories(
                                results, processed, [full_name],
                                args):
                            # Abort?
                            return True
                    continue

                # It's a file, process it, if possible
                # Don't double process the rules file
                if (args.rules_file != entry) and allow_files:
                    add_project(projects, processed, full_name, args)

        # The list is ready, process it in priority order
        # and then loop to the next directory to process
        temp = process_projects(results, projects, args)
        if temp:
            return temp
    return False

########################################

def process_dependencies(results, processed, dependencies, args):
    """
    Process a mixed string list of both directories and files.

    Iterate over the dependencies list and test each object if it's a directory,
    and if so, dispatch to the directory handler, otherwise, process as a file.

    Args:
        results: list object to append BuildError objects
        processed: List of directories already processed.
        dependencies: iterable list of files/directories to process
        args: parsed argument list for verbosity
    Returns:
        True if processing should abort, False if not.
    """

    # Test, in case it is None
    if dependencies:
        for item in dependencies:
            if os.path.isdir(item):
                error = process_directories(results, processed, (item,), args)
            elif os.path.isfile(item):
                error = process_files(results, processed, (item,), args)
            else:
                error = 0
            if error:
                return error
    return 0

########################################

def main(working_directory=None, args=None):
    """
    Command line shell for ``cleanme``.

    Entry point for the program ``cleanme``, this function
    will either get the parameters from sys.argv or the paramater ``args``.

    - ``--version``, show version.
    - ``-r``, Perform a recursive clean.
    - ``-v``, Verbose output.
    - ``-n``, Preview clean commands
    - ``--generate-rules``, Create build_rules.py and exit.
    - ``--rules-file``, Override the configruration file.
    - ``-q``, Quit after the first error
    - ``-f``, List of files to build.
    - ``-d``, List of directories to clean.
    - ``-c``, List of configurations to build
    - Additional terms are considered specific files or configurations to clean.

    Args:
        working_directory: Directory to operate on, or None for os.getcwd()
        args: Command line to use instead of sys.argv
    Returns:
        Zero on no error, non-zero on error
    """

    # Make sure working_directory is properly set
    if working_directory is None:
        working_directory = os.getcwd()

    # Create the parser
    parser = create_parser()

    # Parse everything
    args = parser.parse_args(args=args)

    # Output default configuration
    if args.generate_build_rules:
        if args.verbose:
            print(
                "Saving {}".format(
                    os.path.join(
                        working_directory,
                        args.rules_file)))
        return save_default(working_directory, destinationfile=args.rules_file)

    # Handle extra arguments
    fixup_args(args)

    # Get lists of files/directories to build
    files = args.files
    directories = args.directories

    # If there are no entries, use the working directory
    if not directories and not files:
        directories = [working_directory]

    # List of errors created during building
    results = []
    processed = set()

    # Try building all individual files first
    if not process_files(results, processed, files, args):

        # If successful, process all directories
        process_directories(results, processed, directories, args)

    # Was there a build error?
    error = 0
    for item in results:
        if item.error:
            print("Errors detected in clean.")
            error = item.error
            break
    else:
        if args.verbose:
            print("Clean is successful!")

    # Dump the error log if requested or an error
    if args.verbose or error:
        for item in results:
            if args.verbose or item.error:
                print(item)

    # In case clean is being called from a function,
    # clear the build_rules.py cache
    clear_build_rules_cache()
    return error

# If called as a function and not a class,
# call my main

if __name__ == "__main__":
    sys.exit(main())