try:
    import requests
except:
    from os import system
    system("pip install requests")
    import requests

from django.http import HttpResponse
from django.shortcuts import render

from .gets import *
from cart.get import u_cart
from reserve.get import *

# Create your views here.

def products_view(request, *args, **kwargs):
    thisCart = u_cart(request)

    productsRows, categories = getProductsCatrows()

    myContext = {
        "title": "Товары",
        "categories": categories
    }

    myContext["products"] = productsRows

    print(productsRows['овощи'][0].price)

    return render(request, "products.html", myContext)

def product_detail_view(request, *args, **kwargs):
    thisCart = u_cart(request)
    queryDict = request.GET
    requestedArticle = int(request.GET['article'])
    requestedObject = Product.objects.get(article=requestedArticle)
    requestedObject.amount -= getReservation(requestedArticle)

    myContext = {
        "title": f"Купить {requestedObject.title}",
        "pr": requestedObject
    }

    # requests.get(f"https://api.telegram.org/bot1225466990:AAHeSxZ66mt1sOD_0ojhUf4EpbxoVK06TAY/sendMessage?chat_id=@dchadm&text=Кто-то%20заказал:%20{requestedObject.title}")

    return render(request, "product_details.html", myContext)

def add_to_cart(request, *args, **kwargs):
    thisCart = u_cart(request)
    queryDict = request.GET

    if not (request.method == "GET") or not ('article' in queryDict):
        return HttpResponse("Иди куда подальще!")

    try:
        requestedArticle = int(request.GET['article'])
        requestedAmount = int (request.GET['amount'])
        requestedObject = Product.objects.get(article=requestedArticle)
        if requestedAmount > requestedObject.amount:
            return HttpResponse("Нет такого количества товаров!")
    except:
        return HttpResponse("Bad request!")

    newAmount = requestedObject.amount - requestedAmount
    whetherAvailable = True
    if newAmount == 0:
        whetherAvailable = False

    return HttpResponse(requestedObject.title)


#!/usr/bin/env python
# coding: utf-8

# In[1]:

class graph:
    def __init__(self):
        self.bigl=[]
    def addvertex (self,vert):
        self.bigl.append([vert,{}])
    def addedge (self,vertfro,vertto,weight=0):
        for i in range(len(self.bigl)):
            if self.bigl[i][0] ==vertfro:
                self.bigl[i][1][vertto]=weight
            elif self.bigl[i][0] ==vertto:
                self.bigl[i][1][vertfro]=weight
    def getgraph(self):
        return self.bigl

    def __iter__(self):
        return iter(self.bigl)

# In[ ]:

# In[29]:

class graph1:
    def __init__(self):
        self.bigl={}
    def addvertex (self,vert):
        self.bigl[vert]={}
    def addedge (self,vertfro,vertto,weight=0):
        for i in range(len(self.bigl)):
            self.bigl[vertfro][vertto]=weight
            self.bigl[vertto][vertfro]=weight

    def getgraph(self):
        return self.bigl

    def __iter__(self):
        return iter(self.bigl)

# In[ ]:

# In[30]:

g=graph1()

# In[31]:

for i in range(8):
    g.addvertex(i)

# In[32]:

g.addedge(2,3,9)

# In[33]:

g.addedge(4,3,90)

# In[34]:

g.addedge(1,2,10)

# In[35]:

g.addedge(4,5,8)

# In[36]:

g.addedge(5,6,70)

# In[37]:

g.addedge(6,7,5)

# In[38]:

g.addedge(0,1,4)

# In[39]:

g.addedge(1,7,5)

# In[43]:

g.getgraph()

# In[45]:

for gus in g:
    print (gus)

# In[26]:

import random

random.randint(0,10)

# In[54]:

a=[9,8,7]

# In[55]:

a.pop(0)

# In[56]:

a

# In[ ]:

# In[ ]:

# In[ ]:

# In[47]:

g.getgraph()[3]

# In[48]:

d={"a":3,"b":4}
for i in d.keys():
    print(i)

# In[ ]:

# In[46]:

len(g.getgraph())

# In[75]:

def n_bfs(graphs):
    x=random.randint(0,len(graphs.getgraph())-1)
    q=[]
    q.append(x)
    visited=[]
    while len(visited)<len(graphs.getgraph()):

        tovis=q[0]
        if tovis not in visited:
            dtc=graphs.getgraph()[tovis]
            for i in dtc.keys():
                if i not in q:
                    q.append(i)

            visited.append(tovis)
            q.pop(0)
        else:
            q.pop(0)

    return visited

# In[76]:

n_bfs(g)

# In[61]:

g.getgraph()[1]

# In[66]:

g.getgraph()

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[14]:

def bfs (graphs):
    queue=[]
    for item in graphs:
        if item[0] not in queue:
            queue.append(item[0])
        for kitem in item[1]:
            if kitem not in queue:
                queue.append(kitem)
    return (queue)

# In[ ]:

def dfs (graphs):
    queue=[]
    for item in graphs:
        if item[0] not in queue:
            queue.append(item[0])
        for kitem in item[1]:
            if kitem not in queue:
                queue.append(kitem)
    return (queue)

# In[15]:

bfs(g)

# In[60]:

import difflib

case_a = 'h'
case_b = 'hat'

output_list = [li for li in difflib.ndiff(case_a, case_b) if li[0] != ' ']

# In[61]:

output_list

# In[55]:

difflib.ndiff(case_a, case_b)

# In[62]:

from itertools import combinations

# In[ ]:

list(combinations(arr, r))

# In[ ]:

# In[67]:

def Word_ladder(beginword,endword,wordlist):
    f=graph()
    for word in wordlist:
        f.addvertex(word)
    l2=list(combinations(wordlist, 2))
    for l in l2:
        output_list = [li for li in difflib.ndiff(l[0], l[1]) if li[0] != ' ']
        if len(output_list)==2:
            f.addedge(l[0],l[1],1)
    print (f.getgraph())
    return (bfs(f))

# In[70]:

beginWord = "hit",
endWord = "cog",
wordList = ["hit","hot","dot","dog","lll","lot","log","cog","ooo","ppp"]

# In[71]:

Word_ladder(beginWord,endWord,wordList)

# In[ ]:


"""Helper function to get input from user side with video detection"""
from __future__ import annotations

import argparse

def parse_arguments() -> argparse[object]:
    """Function to parse the arguments"""
    ap = argparse.ArgumentParser()

    ap.add_argument("-i", "--input", type=str, required=True,
                    help="Path to optional input video file")

    ap.add_argument("-m", "--model", required=True,
                    help="Path to yolo model")

    args = vars(ap.parse_args())
    return args


from django.http import HttpResponse
from django.shortcuts import redirect, render
from django.http import HttpResponse

from myapp.forms import TaskForm
from .models import Task
# Create your views here.
def home(request):
    tasks =Task.objects.all()
    all_task = tasks.count()

    completed = Task.objects.filter(status=True)
    completed_task = completed.count()

    Incompleted = Task.objects.filter(status=False)
    Incompleted_task = Incompleted.count()

    form = TaskForm
    if request.method =='POST':
        form = TaskForm(request.POST)
        if form.is_valid():
            form.save()
        return redirect('/')

    context = {'tasks':tasks,'form':form, 'all_task':all_task,'completed':completed_task,'Incompleted':Incompleted_task}
    return render(request ,'home.html', context)

def deleteTask(request,pk):
    item = Task.objects.get(id=pk)
    if request.method=='POST':
        item.delete()
        return redirect('home')
    context={
        'item':item
    }
    return render(request,'delete.html')

def update_Task(request,pk):
    task = Task.objects.get(id=pk)

    form = TaskForm(instance=task)

    if request.method=='POST':
        form = TaskForm(request.POST,instance=task)
        form.save()
        return redirect('home')

    context ={
        'form':form
    }

    return render(request, 'update.html', context)


import torch
import torchvision
import torchvision.transforms as transforms
import torch.utils.data as Utils
from torch.autograd import Variable
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

import os
import cv2
import csv
import sys
import json
import math
import imageio
import numpy as np
import pandas as pd
import scipy.ndimage
from tqdm import tnrange
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
from IPython.display import clear_output

LATENT_DIM = 20
BATCH_SIZE = 32
MAX_EPOCHS = 5000
RESIZE = 256
BETA = 10
LR = 1e-4
NAME = "Pokemon"

if not os.path.isdir('sweep'):
    os.system('mkdir sweep')

if not os.path.isdir('data'):
    os.system('mkdir data')

if not os.path.isdir('checkpoints'):
    os.system('mkdir checkpoints')

def gen_data_list(name="Pokemon"):
    if not os.path.isfile(name+'List'):
        if os.path.exists('./'+name):
            with open(name+'List', 'w') as myfile:
                wr = csv.writer(myfile, quoting=csv.QUOTE_ALL)
                data = [os.path.abspath("./"+name+"/")+"/"+file for file in os.listdir("./"+name+"/")]
                wr.writerow(data)
            print("Generated list of", name, "image paths")
        else:
            print("Missing", name, "folder with images")
    else:
        print(name, "image list available")

class MultiSet(Utils.Dataset):
    """
        Dataloader for the model. Can easily add more datasets.
    """
    def __init__(self, name='Pokemon'):
        self.list = pd.read_csv(name+"List", header=None, delimiter=',').values[0]

    def __len__(self):
        return len(self.list)

    def __getitem__(self, index):
        data = mpimg.imread(self.list[index])
        data = cv2.resize(data, (RESIZE,RESIZE))/255
        return data

def load_checkpoint(filename, net, LR):
    '''
        Loading function for the model before and during training
        From a checkpoint file, it loads and returns all necessary data
        input: filename -> The name of the checkpoint file to be opened (.pth or .pt)
        output: net -> The saved model, including weights and biases
        output: epoch -> The epoch number at which the training was saved
        output: loss_save -> An array of all the saved batch losses during training
        output: optimizer -> The current state of the optimiser with its updated LR
    '''
    checkpoint = torch.load(filename) # load checkpoint data
    net.load_state_dict(checkpoint['state_dict'])
    epoch = checkpoint['epoch']
    losses = checkpoint['losses']
    bces = checkpoint['bces']
    kls = checkpoint['kls']
    cs = checkpoint['cs']
    optimizer = optim.Adam(net.parameters(), lr=LR, amsgrad=True)
    optimizer.load_state_dict(checkpoint['optimizer'])
    scheduler = SGDRScheduler(optimizer, min_lr=1e-5, max_lr=LR, cycle_length=500, current_step=cs)

    print("Loaded checkpoint:", filename)
    return net, epoch, losses, bces, kls, optimizer, scheduler

def generate_animation(path, label):
    images = []
    files = sorted(os.listdir(path))
    for file in files:
        if file[-4:] == '.png':
            images.append(mpimg.imread(path + file))
    imageio.mimsave(path + label + '_animation.gif', images, fps=15)

def multi_plot(images, model, ROW=4, COL=4):
    """
        To plot an array of images
        Need batch size of row*col and a screen
        input: batch of image arrays
    """
    try:
        f, axarr = plt.subplots(ROW, COL, figsize=(15, ROW*4))
        for row in range(ROW//2):
            for col in range(COL):
                image = images[col+((COL-1)*row),:,:,:].unsqueeze(0)
                axarr[2*row,col].imshow(image.squeeze().numpy())
                image = image.permute(0,3,1,2)
                x_out, _, _ = model(Variable(image.float().cuda()))
                x_out = x_out.permute(0,2,3,1)
                axarr[2*row+1,col].imshow(x_out.data.cpu().squeeze().numpy())
        plt.show()
    except:
        raise

def data_train(model, path, epoch):
    try:
        data = mpimg.imread(path)
        data = cv2.resize(data, (RESIZE,RESIZE))/255
        x_in = Variable(torch.FloatTensor(data).unsqueeze(0).permute(0,3,1,2).cuda())
        x_out, _, _ = model(x_in)
        im = np.floor(x_out.permute(0,2,3,1).data.cpu().squeeze().numpy()*255).astype(np.uint8)
        plt.imsave("data/img_{:04d}".format(epoch) + ".png", im)
        print("Saved checkpoint image")
    except:
        raise
        pass

### The following was taken from https://github.com/A-Jacobson/tacotron2

def adjust_lr(optimizer, lr):
    for param_group in optimizer.param_groups:
        param_group['lr'] = lr

class SGDRScheduler:
    """
        Implements STOCHASTIC GRADIENT DESCENT WITH WARM RESTARTS (SGDR)
        with cosine annealing from https://arxiv.org/pdf/1608.03983.pdf.
    """

    def __init__(self, optimizer, min_lr, max_lr, cycle_length, warmup_steps=5, current_step=0):
        self.optimizer = optimizer
        self.min_lr = min_lr
        self.max_lr = max_lr
        self.lr = optimizer.param_groups[0]['lr']
        self.cycle_length = cycle_length
        self.current_step = current_step
        self.warmup_steps = warmup_steps

    def calculate_lr(self):
        """
            Calculates new learning rate with cosine annealing
        """
        step = self.current_step % self.cycle_length  # get step in current cycle
        self.lr = self.min_lr + 0.5 * (self.max_lr - self.min_lr) * \
                  (1 + np.cos((step / self.cycle_length) * np.pi))

    def step(self):
        self.current_step += 1
        self.calculate_lr()
        if self.current_step in range(self.warmup_steps):
            self.lr /= 10.0  # take a few steps with a lower lr to "warmup"
        adjust_lr(self.optimizer, self.lr)


import functools
import os
import shutil
import sys

run_path = os.path.dirname(os.path.realpath(__file__))
sys.path.append(os.path.join(run_path, ".."))

import lib.SQLiteDB as Database
from lib.Objects            import Song, Genre, Charter
from lib.ObjectVerification import verify_song, verify_genre, verify_charter
from lib.Singleton          import Singleton
from etc.Settings           import Settings
from lib.TJA                import read_tja, parse_tja, set_tja_metadata, clean_path, write_tja

conf = Settings()

class DatabaseLayer(metaclass=Singleton):
    def __init__(self):
        self.songs    = Songs()
        self.genres   = Genres()
        self.charters = Charters()
        self.tjas     = TJAs()

class Songs():
    def __init__(self):
        self.db = Database.Database()

    def add(self, song):
        if not verify_song(song):
            sys.exit("Could not verify object!")
        if song.updated in (None, ''):  song.updated = song.added
        vars = (song.title_orig, song.title_eng, song.subtitle_orig,
                song.subtitle_eng, song.artist_orig, song.artist_eng,
                song.source_orig, song.source_eng, song.bpm, song.genre._id,
                song.charter._id, song.d_kantan, song.d_futsuu,
                song.d_muzukashii, song.d_oni, song.d_ura, song.vetted,
                song.comments, song.video_link, song.path, song.md5_orig,
                song.md5_eng, song.added, song.updated)
        return self.db.add_song(*vars)

    def update(self, song):
        if not verify_song(song):
            sys.exit("Could not verify object!")
        if not song._id:
            raise Exception("Song has no ID")

        vars = (song._id, song.title_orig, song.title_eng, song.subtitle_orig,
                song.subtitle_eng, song.artist_orig, song.artist_eng,
                song.source_orig, song.source_eng, song.bpm, song.genre._id,
                song.charter._id, song.d_kantan, song.d_futsuu,
                song.d_muzukashii, song.d_oni, song.d_ura, song.vetted,
                song.comments, song.video_link, song.path, song.md5_orig,
                song.md5_eng, song.added, song.updated)
        return self.db.update_song(*vars)

    def get_by_id(self, id):
        song = self.db.get_song_by_id(id)
        if len(song) == 0:
            return None
        s = song[0]
        g = DatabaseLayer().genres.get_by_id(s['Genre_ID'])
        c = DatabaseLayer().charters.get_by_id(s['Charter_ID'])
        return Song(s['ID'], s['Title_Orig'], s['Title_Eng'], s['Subtitle_Orig'],
                    s['Subtitle_Eng'], s['Artist_Orig'], s['Artist_Eng'],
                    s['Source_Orig'], s['Source_Eng'], s['BPM'], g, c,
                    s['D_Kantan'], s['D_Futsuu'], s['D_Muzukashii'],
                    s['D_Oni'], s['D_Ura'], bool(s['Vetted']), s['Comments'],
                    s['Video_Link'], s['Path'], s['MD5_Orig'], s['MD5_Eng'],
                    s['Added'], s['Updated'])

    def get_all(self):
        data = []
        for s in self.db.get_all_songs():
            g = DatabaseLayer().genres.get_by_id(s['Genre_ID'])
            c = DatabaseLayer().charters.get_by_id(s['Charter_ID'])
            x = Song(s['ID'], s['Title_Orig'], s['Title_Eng'], s['Subtitle_Orig'],
                     s['Subtitle_Eng'], s['Artist_Orig'], s['Artist_Eng'],
                     s['Source_Orig'], s['Source_Eng'], s['BPM'], g, c,
                     s['D_Kantan'], s['D_Futsuu'], s['D_Muzukashii'],
                     s['D_Oni'], s['D_Ura'], bool(s['Vetted']), s['Comments'],
                     s['Video_Link'], s['Path'], s['MD5_Orig'], s['MD5_Eng'],
                     s['Added'], s['Updated'])
            data.append(x)
        return data

    def generate_path(self, song):
        if not song._id:
            raise Execption("No ID set")
        if not song.title_orig:
            raise Exception("No title set")

        path = conf.file_db
        if not os.path.isabs(path):
            path = os.path.join(run_path, '..', path)
        path = os.path.join(path, str(song._id),
                            clean_path(song.title_orig)) + ".tja"
        return os.path.normpath(path)

class Genres():
    def __init__(self):
        self.db = Database.Database()

    def add(self, genre):
        if not verify_genre(genre):
            sys.exit("Could not verify object!")
        return self.db.add_genre(genre.name_jp, genre.name_eng, genre.genre)

    @functools.lru_cache(maxsize=None)
    def get_by_id(self, id):
        g = self.db.get_genre_by_id(id)
        if len(g) == 0:
            return None
        g = g[0]
        return Genre(g['ID'], g['Title_JP'], g['Title_EN'], g['Genre'])

    @functools.lru_cache(maxsize=None)
    def get_by_genre(self, genre):
        g = self.db.get_genre_by_genre(genre)
        if len(g) == 0:
            return None
        g = g[0]
        return Genre(g['ID'], g['Title_JP'], g['Title_EN'], g['Genre'])

    def get_all(self):
        data = []
        for g in self.db.get_all_genres():
            x = Genre(g['ID'], g['Title_JP'], g['Title_EN'], g['Genre'])
            data.append(x)
        return data

class Charters():
    def __init__(self):
        self.db = Database.Database()

    def add(self, charter):
        if not verify_charter(charter):
            sys.exit("Could not verify object!")
        return self.db.add_charter(charter.name, charter.image, charter.about,
                                   charter.staff )

    @functools.lru_cache(maxsize=None)
    def get_by_id(self, id):
        c = self.db.get_charter_by_id(id)
        if len(c) == 0:
            return None
        c = c[0]
        return Charter(c['ID'], c['Name'], c['Image'], c['About'], bool(c['Staff']))

    @functools.lru_cache(maxsize=None)
    def get_by_name(self, name):
        c = self.db.get_charter_by_name(name)
        if len(c) == 0:
            return None
        c = c[0]
        return Charter(c['ID'], c['Name'], c['Image'], c['About'], bool(c['Staff']))

    def get_all(self):
        data = []
        for c in self.db.get_all_charters():
            x = Charter(c['ID'], c['Name'], c['Image'], c['About'], bool(c['Staff']))
            data.append(x)
        return data

class TJAs():
    def store_tja(self, song, tja, song_path, movie_path=None):
        path = DatabaseLayer().songs.generate_path(song)[:-4]+'.ogg'
        tja = set_tja_metadata(tja, title=song.title_orig, sub=song.subtitle_orig,
                                    song=clean_path(song.title_orig)+'.ogg')
        if not os.path.exists(os.path.dirname(song.path)):
            os.makedirs(os.path.dirname(song.path))
        write_tja(tja, song.path)
        shutil.move(song_path, path)
        if movie_path:
            shutil.move(movie_path, os.path.join(os.path.dirname(path), os.path.basename(movie_path)))

    def get_tja(self, song):
        return open(song.path, "rb").read()

    def get_ogg(self, song):
        return open(song.path[:-3]+"ogg", "rb").read()

    def get_mov(self, song):
        meta = parse_tja(self.get_tja(song))
        if meta['movie'] in [None, '', False]:
            return None
        path = os.path.join(os.path.dirname(song.path), meta['movie'])
        return open(path, "rb").read()

    def get_info(self, song):
        def _score(i):
            if i == None:
                return '*'
            return str(i)
        def _alt(orig, alt):
            if orig != alt:
                return " (%s)\n"%alt
            return '\n'
        def _difficulty(song):
            scores = [song.d_kantan, song.d_futsuu, song.d_muzukashii, song.d_oni, song.d_ura]
            return '/'.join([_score(i) for i in scores])

        text =  "Title: " + song.title_orig + _alt(song.title_orig, song.title_eng)
        text += "Artist: " + song.artist_orig + _alt(song.artist_orig, song.artist_eng)
        text += "From: " + song.source_orig + _alt(song.source_orig, song.source_eng)
        text += "Charter: " + song.charter.name + "\n"
        text += "Difficulty: " + _difficulty(song) + "\n"
        text += "Genre: %s (%s)\n"%(song.genre.name_jp, song.genre.name_eng)
        text += "BPM: %s\n"%song.bpm
        text += "Last update: %s\n\n"%song.updated
        return text


from django.conf.urls import url
from django.contrib.auth.views import login
from . import views

app_name="klucz"
urlpatterns =[
    url(r'^$', views.login_view, name='logowanie'),
    url(r'potwierdzenie', views.potwierdzenie, name='potwierdzenie'),
    url(r'klucz', views.wybierz_kluczowe, name='wybierz_kluczowe'),
    url(r'^login/$', login, {'template_name': 'login2.html'}),

    url(r'^error/$', login, {'template_name': 'login_error.html'}),

]

#url(r'^$', views.wybierz_kluczowe, name='wybierz_kluczowe'),


# -*- coding: utf-8 -*-
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import NoAlertPresentException
import unittest, time, re

class SeleniumEntrenamiento(unittest.TestCase):
    def setUp(self):
        self.driver = webdriver.Firefox()
        self.driver.implicitly_wait(30)
        self.base_url = "http://127.0.0.1:8000/"
        self.verificationErrors = []
        self.accept_next_alert = True

    def test_selenium_entrenamiento(self):
        driver = self.driver
        driver.get(self.base_url + "/EigenFaces/trainer/")
        driver.find_element_by_id("procesarCentroide").click()
        driver.find_element_by_id("procesarCentroide").click()
        # ERROR: Caught exception [ERROR: Unsupported command [waitForFrameToLoad |  | ]]
        driver.find_element_by_name("file").click()
        # Warning: waitForTextPresent may require manual changes
        for i in range(60):
            try:
                if re.search(r"^[\s\S]*1-1\.pgm[\s\S]*$", driver.find_element_by_css_selector("BODY").text): break
            except: pass
            time.sleep(1)
        else: self.fail("time out")
        driver.find_element_by_id("procesarCentroide").click()
        # ERROR: Caught exception [ERROR: Unsupported command [waitForFrameToLoad |  | ]]
        driver.find_element_by_id("procesarCentroide").click()
        # ERROR: Caught exception [ERROR: Unsupported command [waitForFrameToLoad | Seleccione un archivo. | ]]

    def is_element_present(self, how, what):
        try: self.driver.find_element(by=how, value=what)
        except NoSuchElementException as e: return False
        return True

    def is_alert_present(self):
        try: self.driver.switch_to_alert()
        except NoAlertPresentException as e: return False
        return True

    def close_alert_and_get_its_text(self):
        try:
            alert = self.driver.switch_to_alert()
            alert_text = alert.text
            if self.accept_next_alert:
                alert.accept()
            else:
                alert.dismiss()
            return alert_text
        finally: self.accept_next_alert = True

    def tearDown(self):
        self.driver.quit()
        self.assertEqual([], self.verificationErrors)

if __name__ == "__main__":
    unittest.main()

Avance 3/Pruebas de Sistema/Pruebas Entrenamiento/SeleniumEntrenamiento.py


#!/usr/bin/env python3

import csv
import matplotlib.pyplot as plt
import matplotlib.ticker as plticker

from typing import Sequence, Tuple, List, Optional, NamedTuple, Union
from cycler import cycler

from misc import ProgressElement
from misc import get_file_name_stat, get_file_name_stat_img

from post_common import NetInfo

def load_results(file_path: str) -> List[ProgressElement]:
    with open(file_path, 'r') as f:
        reader = csv.DictReader(f)
        return list(reader)

def get_legend_long(net_info: Union[Tuple, NetInfo]) -> str:
    if not isinstance(net_info, NetInfo):
        net_info = NetInfo(*net_info)

    af_name_ffn = net_info.af_name

    if net_info.af_name_cnn is None:
        af_name_cnn = af_name_ffn
    else:
        af_name_cnn = net_info.af_name_cnn

    if net_info.net_type == "base":
        legend = "{} CNN, {} FFN".format(af_name_cnn, af_name_ffn)
    elif net_info.net_type == "ahaf":
        legend = "{}-like AHAF CNN, {}-like AHAF FFN".format(
            af_name_cnn, af_name_ffn
        )
    elif net_info.net_type == "fuzzy_ffn":
        legend = "{} CNN, {}-like Fuzzy FFN".format(
            af_name_cnn, af_name_ffn
        )
    else:
        raise ValueError("Network type is not supported")

    if net_info.fine_tuned:
        legend = legend + ", fine-tuned"

    return legend

def get_short_af_name(orig: str) -> str:
    if orig == "Tanh":
        return "tanh"
    elif orig == "Sigmoid":
        return "σ-fn"
    else:
        return orig

def get_legend_short(net_info: Union[Tuple, NetInfo]) -> str:
    if not isinstance(net_info, NetInfo):
        net_info = NetInfo(*net_info)

    af_name_ffn = net_info.af_name

    if net_info.af_name_cnn is None:
        af_name_cnn = af_name_ffn
    else:
        af_name_cnn = net_info.af_name_cnn

    af_name_cnn = get_short_af_name(af_name_cnn)
    af_name_ffn = get_short_af_name(af_name_ffn)

    if net_info.net_type == "base":
        legend = "Base, {}, {}".format(af_name_cnn, af_name_ffn)
    elif net_info.net_type == "ahaf":
        legend = "AHAF, {}, {}".format(
            af_name_cnn, af_name_ffn
        )
    elif net_info.net_type == "fuzzy_ffn":
        legend = "Fuzzy, {}, {}".format(
            af_name_cnn, af_name_ffn
        )
    else:
        raise ValueError("Network type is not supported")

    if net_info.fine_tuned:
        legend = legend + ", tuned"

    return legend

def analyze_network(net_info: Tuple):
    file_path = get_file_name_stat(*net_info)
    results = load_results(file_path)
    base_legend = get_legend_short(net_info)

    acc = []
    loss = []

    for r in results:
        acc.append(float(r["test_acc"]) * 100.0)
        loss.append(float(r["train_loss_mean"]))

    return base_legend, acc, loss

def plot_networks(fig, nets: Sequence[Union[Tuple, NetInfo]], bw=False):
    acc_legends = []
    loss_legends = []

    monochrome = (
            cycler('color', ['black', 'grey', 'dimgray'])
            * cycler('linestyle', ['--', ':', '-.', '-'])
            * cycler('marker', ['None'])
    )

    gs = plt.GridSpec(1, 2)

    acc_fig = fig.add_subplot(gs[0, 0])
    #acc_loc = plticker.LinearLocator(numticks=10)
    #acc_fig.yaxis.set_major_locator(acc_loc)
    acc_fig.set_xlabel('epoch')
    acc_fig.set_ylabel('test accuracy, %')
    acc_fig.grid()
    if bw:
        acc_fig.set_prop_cycle(monochrome)

    loss_fig = fig.add_subplot(gs[0, 1])
    #loss_loc = plticker.LinearLocator(numticks=10)
    #loss_fig.yaxis.set_major_locator(loss_loc)
    loss_fig.set_xlabel('epoch')
    loss_fig.set_ylabel('training loss')
    loss_fig.grid()
    if bw:
        loss_fig.set_prop_cycle(monochrome)

    for net in nets:
        try:
            base_legend, acc, loss = analyze_network(net)
        except Exception as e:
            print("Exception: {}, skipped".format(e))
            continue

        n_epochs = len(acc)
        end_ep = net.epoch
        start_ep = end_ep - n_epochs

        x = tuple(range(start_ep, end_ep))

        acc_legends.append(
            base_legend
        )
        loss_legends.append(
            base_legend
        )
        acc_fig.plot(x, acc)
        loss_fig.plot(x, loss)

    acc_fig.legend(acc_legends)
    loss_fig.legend(loss_legends)

def visualize(
        net_group: str, nets: Sequence[Union[Tuple, NetInfo]], base_title=None,
        bw: bool = False
):
    fig = plt.figure(tight_layout=True, figsize=(7, 3.5))
    if base_title is not None:
        title = "{}, test accuracy and training loss".format(base_title)
        fig.suptitle(title)

    plot_networks(fig, nets, bw)

    if nets:
        #plt.show()
        plt.savefig(get_file_name_stat_img(
            net_group, nets[0].epoch, nets[0].patched, nets[0].fine_tuned
        ))

def main():
    nets_vs_ahaf = [
        NetInfo("base", "ReLU", 100, patched=False, fine_tuned=False),
        NetInfo("ahaf", "ReLU", 100, patched=False, fine_tuned=False),
        NetInfo("ahaf", "ReLU", 150, patched=True, fine_tuned=True),
        NetInfo("base", "SiLU", 100, patched=False, fine_tuned=False),
        NetInfo("ahaf", "SiLU", 100, patched=False, fine_tuned=False),
        NetInfo("ahaf", "SiLU", 150, patched=True, fine_tuned=True),
    ]

    visualize("vs_ahaf", nets_vs_ahaf, "Base VS AHAF", bw=False)

    nets_vs_fuzzy = [
        NetInfo("base", "Tanh", 100, patched=False, fine_tuned=False),
        NetInfo("fuzzy_ffn", "Tanh", 100, patched=False, fine_tuned=False),
        NetInfo("fuzzy_ffn", "Tanh", 150, patched=True, fine_tuned=True),
        NetInfo("base", "Sigmoid", 100, patched=False, fine_tuned=False),
        NetInfo("fuzzy_ffn", "Sigmoid", 100, patched=False, fine_tuned=False),
        NetInfo("fuzzy_ffn", "Sigmoid", 150, patched=True, fine_tuned=True)
    ]

    visualize("vs_fuzzy", nets_vs_fuzzy, "Base VS Fuzzy", bw=False)

if __name__ == "__main__":
    main()


#!/usr/bin/env python
# encoding=utf-8
# maintainer: rgaudin

import re

from datetime import date, timedelta

from bolibana.models import Provider

def contact_for(identity):
    try:
        provider = Provider.objects.get(phone_number=identity)
    except Provider.DoesNotExist:
        provider = None

    return provider

def resp_error(message, action):
    message.respond(u"[ERREUR] Impossible de comprendre " \
                    u"le SMS pour %s" % action)
    return True

def resp_error_date(message):
    message.respond(u"[Date de visite] la date n'est pas valide.")
    return True

def conv_str_int(value):
    try:
        value = int(value)
    except:
        value = None
    return value

def resp_error_dob(message):
    message.respond(u"[ERREUR] la date de naissance n'est pas valide")
    return True

def resp_error_provider(message):
    message.respond(u"Aucun utilisateur ne possede ce numero de telephone")
    return True

def parse_age_dob(age_or_dob, only_date=False):
    """ parse argument as date or age. return date and bool if estimation """

    if re.match(r'^\d{8}$', age_or_dob):
        auto = False
        parsed_date = date(int(age_or_dob[0:4]), int(age_or_dob[4:6]), \
                           int(age_or_dob[6:8]))
    else:
        auto = True
        today = date.today()
        unit = age_or_dob[-1]
        value = int(age_or_dob[:-1])
        if unit.lower() == 'a':
            parsed_date = today - timedelta(365 * value) - timedelta(160)
        elif unit.lower() == 'm':
            parsed_date = today - timedelta(30 * value) - timedelta(15)
        else:
            raise ValueError(u"Age unit unknown: %s" % unit)

    if only_date:
        return parsed_date
    else:
        return (parsed_date, auto)

def date_is_old(reporting_date):

    if (date.today() - reporting_date).days > 30:
        raise ValueError(u"Le %s est passé il y a plus 30 jours" % reporting_date)


import cv2
import os
import numpy as np
import qrcode
import random

# 1) Считываем все фотографии из директории
dir_path = "images"
all_images = [os.path.join(dir_path, img) for img in os.listdir(dir_path) if img.endswith(('jpg', 'png'))]

# Создаем QR-код
data = "https://t.me/burmistrov_robotics"
box_size = 27  # Это начальное значение, вы можете изменить его позже для настройки разрешения

qr = qrcode.QRCode(
    version=1,
    error_correction=qrcode.constants.ERROR_CORRECT_L,
    box_size=box_size,
    border=1
)
qr.add_data(data)
qr.make(fit=True)

# Вычисляем размер модуля в зависимости от размера итогового изображения
final_image_side = 4050
# Отношение сторон итогового изображения к количеству модулей в QR коде
scale_factor = final_image_side / (len(qr.modules) * box_size)

# Теперь создаем изображение с учетом масштабного коэффициента
img_qr = qr.make_image(fill_color="black", back_color="white").resize((int(scale_factor * len(qr.modules) * box_size), int(scale_factor * len(qr.modules) * box_size)))

# Если вы хотите сохранить QR-код в файл для проверки:
img_qr.save("high_res_qr.png")

# Открываем файл с QR-кодом
img_qr = cv2.imread('high_res_qr.png')

# Задаем параметры
box_size_pixels = 150
modules_count = 27  # так как 27x27 квадратов

# Проходим по всем квадратам
for y in range(modules_count):
    for x in range(modules_count):
        # Извлекаем квадрат из изображения
        square = img_qr[y*box_size_pixels:(y+1)*box_size_pixels, x*box_size_pixels:(x+1)*box_size_pixels]
        # Вычисляем средний цвет квадрата
        module_color = np.mean(square)
        if module_color < 128:  # Если модуль черный
            # Выбираем случайное изображение
            random_img_path = random.choice(all_images)
            img = cv2.imread(random_img_path)
            brightness_offset = 0  # Можете выбрать любое другое значение
            img = cv2.subtract(img, np.ones_like(img) * brightness_offset)
            # Ограничиваем значения пикселей, чтобы они оставались в диапазоне [0, 255]
            img = np.clip(img, 0, 255)
            h, w, _ = img.shape
            min_edge = min(h, w)
            yoff = (h - min_edge) // 2
            xoff = (w - min_edge) // 2
            squared = img[yoff:yoff+min_edge, xoff:xoff+min_edge]
            squared_resized = cv2.resize(squared, (box_size_pixels, box_size_pixels))
            # Вставляем изображение вместо черного квадрата
            img_qr[y*box_size_pixels:(y+1)*box_size_pixels, x*box_size_pixels:(x+1)*box_size_pixels] = squared_resized

# Сохраняем результирующее изображение
cv2.imwrite('final_qr_collage.png', img_qr)

# Конвертировать изображение в градации серого
gray_qr = cv2.cvtColor(img_qr, cv2.COLOR_BGR2GRAY)

# Сохранить бинаризованное изображение
cv2.imwrite('final_qr_collage_bw.png', gray_qr)


import arrow
from decimal import Decimal  # If you want to use money this works better

# for floats 0.01 +0.02 != 0.03
import numpy as np
import numpy_financial as npf
from pathlib import Path
from scipy import stats
import shutil

import xlsxwriter
import importlib.metadata

# This is the controlling model which has all the constants in it
#  The various other modules then use this to do
# - capex
# - drug dosage
# - simulation run
class CoreModel:
    def __init__(self):
        self.num_years = 15
        # Defaults for spreadsheet
        self.wb = None
        self.ws = None  # For base case and mean
        self.ws_sd = None  # For Standard Deviation MC
        # Define formats in case not using ws and so not defined later
        self.bold = None
        self.money_fmt = None
        self.bold_money_fmt = None

        self.write_base = True
        try:
            self.version = importlib.metadata.version("pyxmc")
        except importlib.metadata.PackageNotFoundError:
            self.version = "V0.0.0 Alpha"

        self.xlsx_name = "pyxmc.xlsx"
        self.copy_xlsx_path = None

    def create_xlsx(self, filename=None):
        if filename:
            self.xlsx_name = filename
        self.wb = xlsxwriter.Workbook(f"{self.xlsx_name}")
        self.scenario = ""
        self.bold = self.wb.add_format({"bold": 1})
        self.highlight_fmt = self.wb.add_format({"bold": 1, "bg_color": "yellow"})
        self.money_fmt = self.wb.add_format({"num_format": "#,##0"})
        self.bold_money_fmt = self.wb.add_format({"bold": 1, "num_format": "#,##0"})
        # Used on first run to mark where
        self.row = 0

    def close_xlsx(self):
        self.wb.close()
        my_file = Path(self.xlsx_name)
        if self.copy_xlsx_path:
            to_file = Path(self.copy_xlsx_path)
            try:
                shutil.copy(my_file, to_file)
            except PermissionError:
                print("Excel probably open so can't copy over")

    def add_worksheet(self, tabname, colour="green"):
        ws = self.wb.add_worksheet(tabname)
        ws.set_tab_color(colour)
        ws.set_column(0, 0, 25)
        ws.set_column(
            1, self.num_years + 1, 10
        )  # Col B used for single valued data eg NPV
        ws.write_string(0, 0, self.version)
        self.add_years(ws, 1)
        if self.row is None or self.row < 3:
            self.row = 4  # Singleton to mark number of rows used by header
        return ws

    def add_years(self, ws, row):
        ws.write_string(row + 1, 0, "Years", self.bold)
        base_year = arrow.utcnow().shift(years=+1).year
        for i in range(self.num_years):
            ws.write_number(row, i + 2, i + base_year)
            ws.write_number(row + 1, i + 2, i, self.bold)


import matplotlib.pyplot as plt
import cv2

img_logo = cv2.imread("./logo.png")

b, g, r = cv2.split(img_logo)

img_new = cv2.merge([r, g, b])

# plt.subplot(121)
# plt.imshow(img_logo)
# plt.subplot(122)
# plt.imshow(img_new)
#
# plt.show()

cv2.imshow("bgr_image", img_logo)
cv2.imshow("rgb_image", img_new)
cv2.waitKey(0)
cv2.destroyAllWindows()


import torch

import torch.nn as nn

from embed_regularize import embedded_dropout
from locked_dropout import LockedDropout
from weight_drop import WeightDrop

class RNNModel(nn.Module):
    """Container module with an encoder, a recurrent module, and a decoder."""

    def __init__(self, rnn_type, vocab, ninp, nhid, nlayers, dropout=0.5, dropouth=0.5, dropouti=0.5, dropoute=0.1, wdrop=0, tie_weights=False, locked=True, padding_idx=None):
        super(RNNModel, self).__init__()

        self.lockdrop = LockedDropout()
        self.drop = self._lock_drop if locked else self._nonlock_drop

        self.encoder = nn.Embedding(vocab.size, ninp, padding_idx=padding_idx)
        assert rnn_type in ['LSTM', 'QRNN', 'GRU'], 'RNN type is not supported'
        if rnn_type == 'LSTM':
            self.rnns = [torch.nn.LSTM(ninp if l == 0 else nhid, nhid if l != nlayers - 1 else (ninp if tie_weights else nhid), 1, dropout=0, batch_first=True) for l in range(nlayers)]
            if wdrop:
                self.rnns = [WeightDrop(rnn, ['weight_hh_l0'], dropout=wdrop) for rnn in self.rnns]
        if rnn_type == 'GRU':
            self.rnns = [torch.nn.GRU(ninp if l == 0 else nhid, nhid if l != nlayers - 1 else ninp, 1, dropout=0, batch_first=True) for l in range(nlayers)]
            if wdrop:
                self.rnns = [WeightDrop(rnn, ['weight_hh_l0'], dropout=wdrop) for rnn in self.rnns]
        elif rnn_type == 'QRNN':
            from torchqrnn import QRNNLayer
            self.rnns = [QRNNLayer(input_size=ninp if l == 0 else nhid, hidden_size=nhid if l != nlayers - 1 else (ninp if tie_weights else nhid), save_prev_x=True, zoneout=0, window=2 if l == 0 else 1, output_gate=True) for l in range(nlayers)]
            for rnn in self.rnns:
                rnn.linear = WeightDrop(rnn.linear, ['weight'], dropout=wdrop)

        self.rnns = torch.nn.ModuleList(self.rnns)
        self.decoder = nn.Linear(nhid, vocab.size)

        # Optionally tie weights as in:
        # "Using the Output Embedding to Improve Language Models" (Press & Wolf 2016)
        # https://arxiv.org/abs/1608.05859
        # and
        # "Tying Word Vectors and Word Classifiers: A Loss Framework for Language Modeling" (Inan et al. 2016)
        # https://arxiv.org/abs/1611.01462
        if tie_weights:
            #if nhid != ninp:
            #    raise ValueError('When using the tied flag, nhid must be equal to emsize')
            self.decoder.weight = self.encoder.weight

        self.init_weights()

        self.rnn_type = rnn_type
        self.vocab = vocab
        self.ninp = ninp
        self.nhid = nhid
        self.nlayers = nlayers
        self.dropout = dropout
        self.dropouti = dropouti
        self.dropouth = dropouth
        self.dropoute = dropoute
        self.tie_weights = tie_weights

    def _lock_drop(self, w, ratio):
        return self.lockdrop(w, ratio)

    def _nonlock_drop(self, w, ratio):
        d = nn.Dropout(ratio)
        d(w)
        return w

    def reset(self):
        if self.rnn_type == 'QRNN': [r.reset() for r in self.rnns]

    def init_weights(self):
        initrange = 0.1
        self.encoder.weight.data.uniform_(-initrange, initrange)
        self.decoder.bias.data.fill_(0)
        self.decoder.weight.data.uniform_(-initrange, initrange)

    def forward(self, input, hidden=None, return_h=False, input_lengths=None):
        """
        Apply a multi-layer RNN to an input sequence.

        If input_lengths is given, variable-length inputs are assumed. The input to
        RNN is packed in this case. This method does not sort the input by length
        internally, so input_var should be sorted beforehand.
        """
        emb = embedded_dropout(
            self.encoder, input, dropout=self.dropoute if self.training else 0)

        #emb = self.lockdrop(emb, self.dropouti)
        emb = self.drop(emb, self.dropouti)
        raw_output = emb

        if not hidden:
            hidden = self.init_hidden(input.size(0)) # batch_first

        new_hidden = []
        #raw_output, hidden = self.rnn(emb, hidden)
        raw_outputs = []
        outputs = []
        # enumerate layers
        for l, rnn in enumerate(self.rnns):
            if input_lengths is not None:
                raw_output = nn.utils.rnn.pack_padded_sequence(
                    raw_output, input_lengths, batch_first=True)
            raw_output, new_h = rnn(raw_output, hidden[l])
            new_hidden.append(new_h)
            if input_lengths is not None:
                raw_output, _ = nn.utils.rnn.pad_packed_sequence(
                    raw_output, batch_first=True)
            raw_outputs.append(raw_output)
            if l != self.nlayers - 1:
                # self.hdrop(raw_output)
                raw_output = self.drop(raw_output, self.dropouth)
                # raw_output = self.lockdrop(raw_output, self.dropouth)
                outputs.append(raw_output)
        hidden = new_hidden

        # output = self.lockdrop(raw_output, self.dropout)
        output = self.drop(raw_output, self.dropout)
        outputs.append(output)

        result = output.contiguous().view(
            output.size(0)*output.size(1), output.size(2))
        if return_h:
            return result, hidden, raw_outputs, outputs
        return result, hidden

    def init_hidden(self, bsz):
        weight = next(self.parameters()).data
        if self.rnn_type == 'LSTM':
            return [(weight.new(1, bsz, self.nhid if l != self.nlayers - 1 else (self.ninp if self.tie_weights else self.nhid)).zero_(),
                     weight.new(1, bsz, self.nhid if l != self.nlayers - 1 else (self.ninp if self.tie_weights else self.nhid)).zero_())
                    for l in range(self.nlayers)]
        elif self.rnn_type == 'QRNN' or self.rnn_type == 'GRU':
            return [weight.new(1, bsz, self.nhid if l != self.nlayers - 1 else (self.ninp if self.tie_weights else self.nhid)).zero_()
                    for l in range(self.nlayers)]


import collections
import contextlib
import re

import mock
from oslo.vmware import exceptions as vexc
from oslo.vmware import pbm

from nova import context
from nova import exception
from nova.network import model as network_model
from nova.openstack.common import uuidutils
from nova import test
from nova.tests.unit import fake_instance
from nova.tests.unit.virt.vmwareapi import fake
from nova.tests.unit.virt.vmwareapi import stubs
from nova.virt.vmwareapi import driver
from nova.virt.vmwareapi import ds_util
from nova.virt.vmwareapi import vm_util

class partialObject(object):
    def __init__(self, path='fake-path'):
        self.path = path
        self.fault = fake.DataObject()

class VMwareVMUtilTestCase(test.NoDBTestCase):
    def setUp(self):
        super(VMwareVMUtilTestCase, self).setUp()
        fake.reset()
        stubs.set_stubs(self.stubs)
        vm_util.vm_refs_cache_reset()

    def _test_get_stats_from_cluster(self, connection_state="connected",
                                     maintenance_mode=False):
        ManagedObjectRefs = [fake.ManagedObjectReference("host1",
                                                         "HostSystem"),
                             fake.ManagedObjectReference("host2",
                                                         "HostSystem")]
        hosts = fake._convert_to_array_of_mor(ManagedObjectRefs)
        respool = fake.ManagedObjectReference("resgroup-11", "ResourcePool")
        prop_dict = {'host': hosts, 'resourcePool': respool}

        hardware = fake.DataObject()
        hardware.numCpuCores = 8
        hardware.numCpuThreads = 16
        hardware.vendor = "Intel"
        hardware.cpuModel = "Intel(R) Xeon(R)"

        runtime_host_1 = fake.DataObject()
        runtime_host_1.connectionState = "connected"
        runtime_host_1.inMaintenanceMode = False

        runtime_host_2 = fake.DataObject()
        runtime_host_2.connectionState = connection_state
        runtime_host_2.inMaintenanceMode = maintenance_mode

        prop_list_host_1 = [fake.Prop(name="hardware_summary", val=hardware),
                            fake.Prop(name="runtime_summary",
                                      val=runtime_host_1)]
        prop_list_host_2 = [fake.Prop(name="hardware_summary", val=hardware),
                            fake.Prop(name="runtime_summary",
                                      val=runtime_host_2)]

        fake_objects = fake.FakeRetrieveResult()
        fake_objects.add_object(fake.ObjectContent("prop_list_host1",
                                                   prop_list_host_1))
        fake_objects.add_object(fake.ObjectContent("prop_list_host1",
                                                   prop_list_host_2))

        respool_resource_usage = fake.DataObject()
        respool_resource_usage.maxUsage = 5368709120
        respool_resource_usage.overallUsage = 2147483648

        def fake_call_method(*args):
            if "get_dynamic_properties" in args:
                return prop_dict
            elif "get_properties_for_a_collection_of_objects" in args:
                return fake_objects
            else:
                return respool_resource_usage

        session = fake.FakeSession()
        with mock.patch.object(session, '_call_method', fake_call_method):
            result = vm_util.get_stats_from_cluster(session, "cluster1")
            mem_info = {}
            if connection_state == "connected" and not maintenance_mode:
                vcpus = 32
            else:
                vcpus = 16
            mem_info['total'] = 5120
            mem_info['free'] = 3072
            expected_stats = {'vcpus': vcpus, 'mem': mem_info}
            self.assertEqual(expected_stats, result)

    def test_get_stats_from_cluster_hosts_connected_and_active(self):
        self._test_get_stats_from_cluster()

    def test_get_stats_from_cluster_hosts_disconnected_and_active(self):
        self._test_get_stats_from_cluster(connection_state="disconnected")

    def test_get_stats_from_cluster_hosts_connected_and_maintenance(self):
        self._test_get_stats_from_cluster(maintenance_mode=True)

    def test_get_host_ref_no_hosts_in_cluster(self):
        self.assertRaises(exception.NoValidHost,
                          vm_util.get_host_ref,
                          fake.FakeObjectRetrievalSession(""), 'fake_cluster')

    def test_get_resize_spec(self):
        fake_instance = {'id': 7, 'name': 'fake!',
                         'uuid': 'bda5fb9e-b347-40e8-8256-42397848cb00',
                         'vcpus': 2, 'memory_mb': 2048}
        result = vm_util.get_vm_resize_spec(fake.FakeFactory(),
                                            fake_instance)
        expected = """{'memoryMB': 2048,
                       'numCPUs': 2,
                       'obj_name': 'ns0:VirtualMachineConfigSpec'}"""
        expected = re.sub(r'\s+', '', expected)
        result = re.sub(r'\s+', '', repr(result))
        self.assertEqual(expected, result)

    def test_get_cdrom_attach_config_spec(self):

        result = vm_util.get_cdrom_attach_config_spec(fake.FakeFactory(),
                                             fake.Datastore(),
                                             "/tmp/foo.iso",
                                             200, 0)
        expected = """{
    'deviceChange': [
        {
            'device': {
                'connectable': {
                    'allowGuestControl': False,
                    'startConnected': True,
                    'connected': True,
                    'obj_name': 'ns0: VirtualDeviceConnectInfo'
                },
                'backing': {
                    'datastore': {
                        "summary.maintenanceMode": "normal",
                        "summary.type": "VMFS",
                        "summary.accessible":true,
                        "summary.name": "fake-ds",
                        "summary.capacity": 1099511627776,
                        "summary.freeSpace": 536870912000,
                        "browser": ""
                    },
                    'fileName': '/tmp/foo.iso',
                    'obj_name': 'ns0: VirtualCdromIsoBackingInfo'
                },
                'controllerKey': 200,
                'unitNumber': 0,
                'key': -1,
                'obj_name': 'ns0: VirtualCdrom'
            },
            'operation': 'add',
            'obj_name': 'ns0: VirtualDeviceConfigSpec'
        }
    ],
    'obj_name': 'ns0: VirtualMachineConfigSpec'
}
"""

        expected = re.sub(r'\s+', '', expected)
        result = re.sub(r'\s+', '', repr(result))
        self.assertEqual(expected, result)

    def test_lsilogic_controller_spec(self):
        # Test controller spec returned for lsiLogic sas adapter type
        config_spec = vm_util.create_controller_spec(fake.FakeFactory(), -101,
                          adapter_type="lsiLogicsas")
        self.assertEqual("ns0:VirtualLsiLogicSASController",
                         config_spec.device.obj_name)

    def test_paravirtual_controller_spec(self):
        # Test controller spec returned for paraVirtual adapter type
        config_spec = vm_util.create_controller_spec(fake.FakeFactory(), -101,
                          adapter_type="paraVirtual")
        self.assertEqual("ns0:ParaVirtualSCSIController",
                         config_spec.device.obj_name)

    def _vmdk_path_and_adapter_type_devices(self, filename, parent=None):
        # Test the adapter_type returned for a lsiLogic sas controller
        controller_key = 1000
        disk = fake.VirtualDisk()
        disk.controllerKey = controller_key
        disk_backing = fake.VirtualDiskFlatVer2BackingInfo()
        disk_backing.fileName = filename
        disk.capacityInBytes = 1024
        if parent:
            disk_backing.parent = parent
        disk.backing = disk_backing
        controller = fake.VirtualLsiLogicSASController()
        controller.key = controller_key
        devices = [disk, controller]
        return devices

    def test_get_vmdk_path_and_adapter_type(self):
        filename = '[test_datastore] test_file.vmdk'
        devices = self._vmdk_path_and_adapter_type_devices(filename)
        session = fake.FakeSession()
        with mock.patch.object(session, '_call_method', return_value=devices):
            vmdk = vm_util.get_vmdk_info(session, None)
            self.assertEqual('lsiLogicsas', vmdk.adapter_type)
            self.assertEqual(filename, vmdk.path)

    def test_get_vmdk_path_and_adapter_type_with_match(self):
        n_filename = '[test_datastore] uuid/uuid.vmdk'
        devices = self._vmdk_path_and_adapter_type_devices(n_filename)
        session = fake.FakeSession()
        with mock.patch.object(session, '_call_method', return_value=devices):
            vmdk = vm_util.get_vmdk_info(session, None, uuid='uuid')
            self.assertEqual('lsiLogicsas', vmdk.adapter_type)
            self.assertEqual(n_filename, vmdk.path)

    def test_get_vmdk_path_and_adapter_type_with_nomatch(self):
        n_filename = '[test_datastore] diuu/diuu.vmdk'
        session = fake.FakeSession()
        devices = self._vmdk_path_and_adapter_type_devices(n_filename)
        with mock.patch.object(session, '_call_method', return_value=devices):
            vmdk = vm_util.get_vmdk_info(session, None, uuid='uuid')
            self.assertEqual('lsiLogicsas', vmdk.adapter_type)
            self.assertIsNone(vmdk.path)

    def test_get_vmdk_adapter_type(self):
        # Test for the adapter_type to be used in vmdk descriptor
        # Adapter type in vmdk descriptor is same for LSI-SAS, LSILogic
        # and ParaVirtual
        vmdk_adapter_type = vm_util.get_vmdk_adapter_type("lsiLogic")
        self.assertEqual("lsiLogic", vmdk_adapter_type)
        vmdk_adapter_type = vm_util.get_vmdk_adapter_type("lsiLogicsas")
        self.assertEqual("lsiLogic", vmdk_adapter_type)
        vmdk_adapter_type = vm_util.get_vmdk_adapter_type("paraVirtual")
        self.assertEqual("lsiLogic", vmdk_adapter_type)
        vmdk_adapter_type = vm_util.get_vmdk_adapter_type("dummyAdapter")
        self.assertEqual("dummyAdapter", vmdk_adapter_type)

    def test_find_allocated_slots(self):
        disk1 = fake.VirtualDisk(200, 0)
        disk2 = fake.VirtualDisk(200, 1)
        disk3 = fake.VirtualDisk(201, 1)
        ide0 = fake.VirtualIDEController(200)
        ide1 = fake.VirtualIDEController(201)
        scsi0 = fake.VirtualLsiLogicController(key=1000, scsiCtlrUnitNumber=7)
        devices = [disk1, disk2, disk3, ide0, ide1, scsi0]
        taken = vm_util._find_allocated_slots(devices)
        self.assertEqual([0, 1], sorted(taken[200]))
        self.assertEqual([1], taken[201])
        self.assertEqual([7], taken[1000])

    def test_allocate_controller_key_and_unit_number_ide_default(self):
        # Test that default IDE controllers are used when there is a free slot
        # on them
        disk1 = fake.VirtualDisk(200, 0)
        disk2 = fake.VirtualDisk(200, 1)
        ide0 = fake.VirtualIDEController(200)
        ide1 = fake.VirtualIDEController(201)
        devices = [disk1, disk2, ide0, ide1]
        (controller_key, unit_number,
         controller_spec) = vm_util.allocate_controller_key_and_unit_number(
                                                            None,
                                                            devices,
                                                            'ide')
        self.assertEqual(201, controller_key)
        self.assertEqual(0, unit_number)
        self.assertIsNone(controller_spec)

    def test_allocate_controller_key_and_unit_number_ide(self):
        # Test that a new controller is created when there is no free slot on
        # the default IDE controllers
        ide0 = fake.VirtualIDEController(200)
        ide1 = fake.VirtualIDEController(201)
        devices = [ide0, ide1]
        for controller_key in [200, 201]:
            for unit_number in [0, 1]:
                disk = fake.VirtualDisk(controller_key, unit_number)
                devices.append(disk)
        factory = fake.FakeFactory()
        (controller_key, unit_number,
         controller_spec) = vm_util.allocate_controller_key_and_unit_number(
                                                            factory,
                                                            devices,
                                                            'ide')
        self.assertEqual(-101, controller_key)
        self.assertEqual(0, unit_number)
        self.assertIsNotNone(controller_spec)

    def test_allocate_controller_key_and_unit_number_scsi(self):
        # Test that we allocate on existing SCSI controller if there is a free
        # slot on it
        devices = [fake.VirtualLsiLogicController(1000, scsiCtlrUnitNumber=7)]
        for unit_number in range(7):
            disk = fake.VirtualDisk(1000, unit_number)
            devices.append(disk)
        factory = fake.FakeFactory()
        (controller_key, unit_number,
         controller_spec) = vm_util.allocate_controller_key_and_unit_number(
                                                            factory,
                                                            devices,
                                                            'lsiLogic')
        self.assertEqual(1000, controller_key)
        self.assertEqual(8, unit_number)
        self.assertIsNone(controller_spec)

    def _test_get_vnc_config_spec(self, port):

        result = vm_util.get_vnc_config_spec(fake.FakeFactory(),
                                             port)
        return result

    def test_get_vnc_config_spec(self):
        result = self._test_get_vnc_config_spec(7)
        expected = """{'extraConfig': [
                          {'value': 'true',
                           'key': 'RemoteDisplay.vnc.enabled',
                           'obj_name': 'ns0:OptionValue'},
                          {'value': 7,
                           'key': 'RemoteDisplay.vnc.port',
                           'obj_name': 'ns0:OptionValue'}],
                       'obj_name': 'ns0:VirtualMachineConfigSpec'}"""
        expected = re.sub(r'\s+', '', expected)
        result = re.sub(r'\s+', '', repr(result))
        self.assertEqual(expected, result)

    def _create_fake_vms(self):
        fake_vms = fake.FakeRetrieveResult()
        OptionValue = collections.namedtuple('OptionValue', ['key', 'value'])
        for i in range(10):
            vm = fake.ManagedObject()
            opt_val = OptionValue(key='', value=5900 + i)
            vm.set(vm_util.VNC_CONFIG_KEY, opt_val)
            fake_vms.add_object(vm)
        return fake_vms

    def test_get_vnc_port(self):
        fake_vms = self._create_fake_vms()
        self.flags(vnc_port=5900, group='vmware')
        self.flags(vnc_port_total=10000, group='vmware')
        actual = vm_util.get_vnc_port(
            fake.FakeObjectRetrievalSession(fake_vms))
        self.assertEqual(actual, 5910)

    def test_get_vnc_port_exhausted(self):
        fake_vms = self._create_fake_vms()
        self.flags(vnc_port=5900, group='vmware')
        self.flags(vnc_port_total=10, group='vmware')
        self.assertRaises(exception.ConsolePortRangeExhausted,
                          vm_util.get_vnc_port,
                          fake.FakeObjectRetrievalSession(fake_vms))

    def test_get_all_cluster_refs_by_name_none(self):
        fake_objects = fake.FakeRetrieveResult()
        refs = vm_util.get_all_cluster_refs_by_name(
            fake.FakeObjectRetrievalSession(fake_objects), ['fake_cluster'])
        self.assertEqual({}, refs)

    def test_get_all_cluster_refs_by_name_exists(self):
        fake_objects = fake.FakeRetrieveResult()
        fake_objects.add_object(fake.ClusterComputeResource(name='cluster'))
        refs = vm_util.get_all_cluster_refs_by_name(
            fake.FakeObjectRetrievalSession(fake_objects), ['cluster'])
        self.assertEqual(1, len(refs))

    def test_get_all_cluster_refs_by_name_missing(self):
        fake_objects = fake.FakeRetrieveResult()
        fake_objects.add_object(partialObject(path='cluster'))
        refs = vm_util.get_all_cluster_refs_by_name(
            fake.FakeObjectRetrievalSession(fake_objects), ['cluster'])
        self.assertEqual({}, refs)

    def test_propset_dict_simple(self):
        ObjectContent = collections.namedtuple('ObjectContent', ['propSet'])
        DynamicProperty = collections.namedtuple('Property', ['name', 'val'])

        object = ObjectContent(propSet=[
                    DynamicProperty(name='foo', val="bar")])
        propdict = vm_util.propset_dict(object.propSet)
        self.assertEqual("bar", propdict['foo'])

    def test_propset_dict_complex(self):
        ObjectContent = collections.namedtuple('ObjectContent', ['propSet'])
        DynamicProperty = collections.namedtuple('Property', ['name', 'val'])
        MoRef = collections.namedtuple('Val', ['value'])

        object = ObjectContent(propSet=[
                    DynamicProperty(name='foo', val="bar"),
                    DynamicProperty(name='some.thing',
                                    val=MoRef(value='else')),
                    DynamicProperty(name='another.thing', val='value')])

        propdict = vm_util.propset_dict(object.propSet)
        self.assertEqual("bar", propdict['foo'])
        self.assertTrue(hasattr(propdict['some.thing'], 'value'))
        self.assertEqual("else", propdict['some.thing'].value)
        self.assertEqual("value", propdict['another.thing'])

    def _test_detach_virtual_disk_spec(self, destroy_disk=False):
        virtual_device_config = vm_util.detach_virtual_disk_spec(
                                                     fake.FakeFactory(),
                                                     'fake_device',
                                                     destroy_disk)
        self.assertEqual('remove', virtual_device_config.operation)
        self.assertEqual('fake_device', virtual_device_config.device)
        self.assertEqual('ns0:VirtualDeviceConfigSpec',
                         virtual_device_config.obj_name)
        if destroy_disk:
            self.assertEqual('destroy', virtual_device_config.fileOperation)
        else:
            self.assertFalse(hasattr(virtual_device_config, 'fileOperation'))

    def test_detach_virtual_disk_spec(self):
        self._test_detach_virtual_disk_spec(destroy_disk=False)

    def test_detach_virtual_disk_destroy_spec(self):
        self._test_detach_virtual_disk_spec(destroy_disk=True)

    def test_get_vm_create_spec(self):
        instance_uuid = uuidutils.generate_uuid()
        fake_instance = {'id': 7, 'name': 'fake!',
                         'uuid': instance_uuid,
                         'vcpus': 2, 'memory_mb': 2048}
        extra_specs = vm_util.ExtraSpecs()
        result = vm_util.get_vm_create_spec(fake.FakeFactory(),
                                            fake_instance, instance_uuid,
                                            'fake-datastore', [],
                                            extra_specs)
        expected = """{
            'files': {'vmPathName': '[fake-datastore]',
            'obj_name': 'ns0:VirtualMachineFileInfo'},
            'instanceUuid': '%(instance_uuid)s',
            'name': '%(instance_uuid)s', 'deviceChange': [],
            'extraConfig': [{'value': '%(instance_uuid)s',
                             'key': 'nvp.vm-uuid',
                             'obj_name': 'ns0:OptionValue'}],
            'memoryMB': 2048,
            'managedBy':  {'extensionKey': 'org.openstack.compute',
                           'type': 'instance',
                           'obj_name': 'ns0:ManagedByInfo'},
            'version': None,
            'obj_name': 'ns0:VirtualMachineConfigSpec',
            'guestId': 'otherGuest',
            'tools': {'beforeGuestStandby': True,
                      'beforeGuestReboot': True,
                      'beforeGuestShutdown': True,
                      'afterResume': True,
                      'afterPowerOn': True,
            'obj_name': 'ns0:ToolsConfigInfo'},
            'numCPUs': 2}""" % {'instance_uuid': instance_uuid}
        expected = re.sub(r'\s+', '', expected)
        result = re.sub(r'\s+', '', repr(result))
        self.assertEqual(expected, result)

    def test_get_vm_create_spec_with_allocations(self):
        instance_uuid = uuidutils.generate_uuid()
        fake_instance = {'id': 7, 'name': 'fake!',
                         'uuid': instance_uuid,
                         'vcpus': 2, 'memory_mb': 2048}
        cpu_limits = vm_util.CpuLimits(cpu_limit=7,
                                       cpu_reservation=6)
        extra_specs = vm_util.ExtraSpecs(cpu_limits=cpu_limits)
        result = vm_util.get_vm_create_spec(fake.FakeFactory(),
                                            fake_instance, instance_uuid,
                                            'fake-datastore', [],
                                            extra_specs)
        expected = """{
            'files': {'vmPathName': '[fake-datastore]',
            'obj_name': 'ns0:VirtualMachineFileInfo'},
            'instanceUuid': '%(instance_uuid)s',
            'name': '%(instance_uuid)s', 'deviceChange': [],
            'extraConfig': [{'value': '%(instance_uuid)s',
                             'key': 'nvp.vm-uuid',
                             'obj_name': 'ns0:OptionValue'}],
            'memoryMB': 2048,
            'managedBy':  {'extensionKey': 'org.openstack.compute',
                           'type': 'instance',
                           'obj_name': 'ns0:ManagedByInfo'},
            'version': None,
            'obj_name': 'ns0:VirtualMachineConfigSpec',
            'guestId': 'otherGuest',
            'tools': {'beforeGuestStandby': True,
                      'beforeGuestReboot': True,
                      'beforeGuestShutdown': True,
                      'afterResume': True,
                      'afterPowerOn': True,
            'obj_name': 'ns0:ToolsConfigInfo'},
            'cpuAllocation': {'reservation': 6,
                              'limit': 7,
                              'obj_name': 'ns0:ResourceAllocationInfo'},
            'numCPUs': 2}""" % {'instance_uuid': instance_uuid}
        expected = re.sub(r'\s+', '', expected)
        result = re.sub(r'\s+', '', repr(result))
        self.assertEqual(expected, result)

    def test_get_vm_create_spec_with_limit(self):
        instance_uuid = uuidutils.generate_uuid()
        fake_instance = {'id': 7, 'name': 'fake!',
                         'uuid': instance_uuid,
                         'vcpus': 2, 'memory_mb': 2048}
        cpu_limits = vm_util.CpuLimits(cpu_limit=7)
        extra_specs = vm_util.ExtraSpecs(cpu_limits=cpu_limits)
        result = vm_util.get_vm_create_spec(fake.FakeFactory(),
                                            fake_instance, instance_uuid,
                                            'fake-datastore', [],
                                            extra_specs)
        expected = """{
            'files': {'vmPathName': '[fake-datastore]',
            'obj_name': 'ns0:VirtualMachineFileInfo'},
            'instanceUuid': '%(instance_uuid)s',
            'name': '%(instance_uuid)s', 'deviceChange': [],
            'extraConfig': [{'value': '%(instance_uuid)s',
                             'key': 'nvp.vm-uuid',
                             'obj_name': 'ns0:OptionValue'}],
            'memoryMB': 2048,
            'managedBy':  {'extensionKey': 'org.openstack.compute',
                           'type': 'instance',
                           'obj_name': 'ns0:ManagedByInfo'},
            'version': None,
            'obj_name': 'ns0:VirtualMachineConfigSpec',
            'guestId': 'otherGuest',
            'tools': {'beforeGuestStandby': True,
                      'beforeGuestReboot': True,
                      'beforeGuestShutdown': True,
                      'afterResume': True,
                      'afterPowerOn': True,
            'obj_name': 'ns0:ToolsConfigInfo'},
            'cpuAllocation': {'limit': 7,
                              'obj_name': 'ns0:ResourceAllocationInfo'},
            'numCPUs': 2}""" % {'instance_uuid': instance_uuid}
        expected = re.sub(r'\s+', '', expected)
        result = re.sub(r'\s+', '', repr(result))
        self.assertEqual(expected, result)

    def test_get_vm_create_spec_with_share(self):
        instance_uuid = uuidutils.generate_uuid()
        fake_instance = {'id': 7, 'name': 'fake!',
                         'uuid': instance_uuid,
                         'vcpus': 2, 'memory_mb': 2048}
        cpu_limits = vm_util.CpuLimits(cpu_shares_level='high')
        extra_specs = vm_util.ExtraSpecs(cpu_limits=cpu_limits)
        result = vm_util.get_vm_create_spec(fake.FakeFactory(),
                                            fake_instance, instance_uuid,
                                            'fake-datastore', [],
                                            extra_specs)
        expected = """{
            'files': {'vmPathName': '[fake-datastore]',
            'obj_name': 'ns0:VirtualMachineFileInfo'},
            'instanceUuid': '%(instance_uuid)s',
            'name': '%(instance_uuid)s', 'deviceChange': [],
            'extraConfig': [{'value': '%(instance_uuid)s',
                             'key': 'nvp.vm-uuid',
                             'obj_name': 'ns0:OptionValue'}],
            'memoryMB': 2048,
            'managedBy':  {'extensionKey': 'org.openstack.compute',
                           'type': 'instance',
                           'obj_name': 'ns0:ManagedByInfo'},
            'version': None,
            'obj_name': 'ns0:VirtualMachineConfigSpec',
            'guestId': 'otherGuest',
            'tools': {'beforeGuestStandby': True,
                      'beforeGuestReboot': True,
                      'beforeGuestShutdown': True,
                      'afterResume': True,
                      'afterPowerOn': True,
            'obj_name': 'ns0:ToolsConfigInfo'},
            'cpuAllocation': {'shares': {'level': 'high',
                                         'shares': 0,
                                         'obj_name':'ns0:SharesInfo'},
                              'obj_name':'ns0:ResourceAllocationInfo'},
            'numCPUs': 2}""" % {'instance_uuid': instance_uuid}
        expected = re.sub(r'\s+', '', expected)
        result = re.sub(r'\s+', '', repr(result))
        self.assertEqual(expected, result)

    def test_get_vm_create_spec_with_share_custom(self):
        instance_uuid = uuidutils.generate_uuid()
        fake_instance = {'id': 7, 'name': 'fake!',
                         'uuid': instance_uuid,
                         'vcpus': 2, 'memory_mb': 2048}
        cpu_limits = vm_util.CpuLimits(cpu_shares_level='custom',
                                       cpu_shares_share=1948)
        extra_specs = vm_util.ExtraSpecs(cpu_limits=cpu_limits)
        result = vm_util.get_vm_create_spec(fake.FakeFactory(),
                                            fake_instance, instance_uuid,
                                            'fake-datastore', [],
                                            extra_specs)
        expected = """{
            'files': {'vmPathName': '[fake-datastore]',
            'obj_name': 'ns0:VirtualMachineFileInfo'},
            'instanceUuid': '%(instance_uuid)s',
            'name': '%(instance_uuid)s', 'deviceChange': [],
            'extraConfig': [{'value': '%(instance_uuid)s',
                             'key': 'nvp.vm-uuid',
                             'obj_name': 'ns0:OptionValue'}],
            'memoryMB': 2048,
            'managedBy':  {'extensionKey': 'org.openstack.compute',
                           'type': 'instance',
                           'obj_name': 'ns0:ManagedByInfo'},
            'version': None,
            'obj_name': 'ns0:VirtualMachineConfigSpec',
            'guestId': 'otherGuest',
            'tools': {'beforeGuestStandby': True,
                      'beforeGuestReboot': True,
                      'beforeGuestShutdown': True,
                      'afterResume': True,
                      'afterPowerOn': True,
            'obj_name': 'ns0:ToolsConfigInfo'},
            'cpuAllocation': {'shares': {'level': 'custom',
                                         'shares': 1948,
                                         'obj_name':'ns0:SharesInfo'},
                              'obj_name':'ns0:ResourceAllocationInfo'},
            'numCPUs': 2}""" % {'instance_uuid': instance_uuid}
        expected = re.sub(r'\s+', '', expected)
        result = re.sub(r'\s+', '', repr(result))
        self.assertEqual(expected, result)

    def test_create_vm(self):

        method_list = ['CreateVM_Task', 'get_dynamic_property']

        def fake_call_method(module, method, *args, **kwargs):
            expected_method = method_list.pop(0)
            self.assertEqual(expected_method, method)
            if (expected_method == 'CreateVM_Task'):
                return 'fake_create_vm_task'
            elif (expected_method == 'get_dynamic_property'):
                task_info = mock.Mock(state="success", result="fake_vm_ref")
                return task_info
            else:
                self.fail('Should not get here....')

        def fake_wait_for_task(self, *args):
            task_info = mock.Mock(state="success", result="fake_vm_ref")
            return task_info

        session = fake.FakeSession()
        fake_instance = mock.MagicMock()
        fake_call_mock = mock.Mock(side_effect=fake_call_method)
        fake_wait_mock = mock.Mock(side_effect=fake_wait_for_task)
        with contextlib.nested(
                mock.patch.object(session, '_wait_for_task',
                                  fake_wait_mock),
                mock.patch.object(session, '_call_method',
                                  fake_call_mock)
        ) as (wait_for_task, call_method):
            vm_ref = vm_util.create_vm(
                session,
                fake_instance,
                'fake_vm_folder',
                'fake_config_spec',
                'fake_res_pool_ref')
            self.assertEqual('fake_vm_ref', vm_ref)

            call_method.assert_called_once_with(mock.ANY, 'CreateVM_Task',
                'fake_vm_folder', config='fake_config_spec',
                pool='fake_res_pool_ref')
            wait_for_task.assert_called_once_with('fake_create_vm_task')

    @mock.patch.object(vm_util.LOG, 'warning')
    def test_create_vm_invalid_guestid(self, mock_log_warn):
        """Ensure we warn when create_vm() fails after we passed an
        unrecognised guestId
        """

        found = [False]

        def fake_log_warn(msg, values):
            if not isinstance(values, dict):
                return
            if values.get('ostype') == 'invalid_os_type':
                found[0] = True
        mock_log_warn.side_effect = fake_log_warn

        instance_values = {'id': 7, 'name': 'fake-name',
                           'uuid': uuidutils.generate_uuid(),
                           'vcpus': 2, 'memory_mb': 2048}
        instance = fake_instance.fake_instance_obj(
            context.RequestContext('fake', 'fake', is_admin=False),
            **instance_values)

        session = driver.VMwareAPISession()

        config_spec = vm_util.get_vm_create_spec(
            session.vim.client.factory,
            instance, instance.name, 'fake-datastore', [],
            vm_util.ExtraSpecs(),
            os_type='invalid_os_type')

        self.assertRaises(vexc.VMwareDriverException,
                          vm_util.create_vm, session, instance, 'folder',
                          config_spec, 'res-pool')
        self.assertTrue(found[0])

    def test_convert_vif_model(self):
        expected = "VirtualE1000"
        result = vm_util.convert_vif_model(network_model.VIF_MODEL_E1000)
        self.assertEqual(expected, result)
        expected = "VirtualE1000e"
        result = vm_util.convert_vif_model(network_model.VIF_MODEL_E1000E)
        self.assertEqual(expected, result)
        types = ["VirtualE1000", "VirtualE1000e", "VirtualPCNet32",
                 "VirtualVmxnet"]
        for type in types:
            self.assertEqual(type,
                             vm_util.convert_vif_model(type))
        self.assertRaises(exception.Invalid,
                          vm_util.convert_vif_model,
                          "InvalidVifModel")

    def test_power_on_instance_with_vm_ref(self):
        session = fake.FakeSession()
        fake_instance = mock.MagicMock()
        with contextlib.nested(
            mock.patch.object(session, "_call_method",
                              return_value='fake-task'),
            mock.patch.object(session, "_wait_for_task"),
        ) as (fake_call_method, fake_wait_for_task):
            vm_util.power_on_instance(session, fake_instance,
                                      vm_ref='fake-vm-ref')
            fake_call_method.assert_called_once_with(session.vim,
                                                     "PowerOnVM_Task",
                                                     'fake-vm-ref')
            fake_wait_for_task.assert_called_once_with('fake-task')

    def test_power_on_instance_without_vm_ref(self):
        session = fake.FakeSession()
        fake_instance = mock.MagicMock()
        with contextlib.nested(
            mock.patch.object(vm_util, "get_vm_ref",
                              return_value='fake-vm-ref'),
            mock.patch.object(session, "_call_method",
                              return_value='fake-task'),
            mock.patch.object(session, "_wait_for_task"),
        ) as (fake_get_vm_ref, fake_call_method, fake_wait_for_task):
            vm_util.power_on_instance(session, fake_instance)
            fake_get_vm_ref.assert_called_once_with(session, fake_instance)
            fake_call_method.assert_called_once_with(session.vim,
                                                     "PowerOnVM_Task",
                                                     'fake-vm-ref')
            fake_wait_for_task.assert_called_once_with('fake-task')

    def test_power_on_instance_with_exception(self):
        session = fake.FakeSession()
        fake_instance = mock.MagicMock()
        with contextlib.nested(
            mock.patch.object(session, "_call_method",
                              return_value='fake-task'),
            mock.patch.object(session, "_wait_for_task",
                              side_effect=exception.NovaException('fake')),
        ) as (fake_call_method, fake_wait_for_task):
            self.assertRaises(exception.NovaException,
                              vm_util.power_on_instance,
                              session, fake_instance,
                              vm_ref='fake-vm-ref')
            fake_call_method.assert_called_once_with(session.vim,
                                                     "PowerOnVM_Task",
                                                     'fake-vm-ref')
            fake_wait_for_task.assert_called_once_with('fake-task')

    def test_power_on_instance_with_power_state_exception(self):
        session = fake.FakeSession()
        fake_instance = mock.MagicMock()
        with contextlib.nested(
            mock.patch.object(session, "_call_method",
                              return_value='fake-task'),
            mock.patch.object(
                    session, "_wait_for_task",
                    side_effect=vexc.InvalidPowerStateException),
        ) as (fake_call_method, fake_wait_for_task):
            vm_util.power_on_instance(session, fake_instance,
                                      vm_ref='fake-vm-ref')
            fake_call_method.assert_called_once_with(session.vim,
                                                     "PowerOnVM_Task",
                                                     'fake-vm-ref')
            fake_wait_for_task.assert_called_once_with('fake-task')

    def test_create_virtual_disk(self):
        session = fake.FakeSession()
        dm = session.vim.service_content.virtualDiskManager
        with contextlib.nested(
            mock.patch.object(vm_util, "get_vmdk_create_spec",
                              return_value='fake-spec'),
            mock.patch.object(session, "_call_method",
                              return_value='fake-task'),
            mock.patch.object(session, "_wait_for_task"),
        ) as (fake_get_spec, fake_call_method, fake_wait_for_task):
            vm_util.create_virtual_disk(session, 'fake-dc-ref',
                                        'fake-adapter-type', 'fake-disk-type',
                                        'fake-path', 7)
            fake_get_spec.assert_called_once_with(
                    session.vim.client.factory, 7,
                    'fake-adapter-type',
                    'fake-disk-type')
            fake_call_method.assert_called_once_with(
                    session.vim,
                    "CreateVirtualDisk_Task",
                    dm,
                    name='fake-path',
                    datacenter='fake-dc-ref',
                    spec='fake-spec')
            fake_wait_for_task.assert_called_once_with('fake-task')

    def test_copy_virtual_disk(self):
        session = fake.FakeSession()
        dm = session.vim.service_content.virtualDiskManager
        with contextlib.nested(
            mock.patch.object(session, "_call_method",
                              return_value='fake-task'),
            mock.patch.object(session, "_wait_for_task"),
        ) as (fake_call_method, fake_wait_for_task):
            vm_util.copy_virtual_disk(session, 'fake-dc-ref',
                                      'fake-source', 'fake-dest')
            fake_call_method.assert_called_once_with(
                    session.vim,
                    "CopyVirtualDisk_Task",
                    dm,
                    sourceName='fake-source',
                    sourceDatacenter='fake-dc-ref',
                    destName='fake-dest')
            fake_wait_for_task.assert_called_once_with('fake-task')

    def _create_fake_vm_objects(self):
        fake_objects = fake.FakeRetrieveResult()
        fake_objects.add_object(fake.VirtualMachine())
        return fake_objects

    def test_get_values(self):
        objects = self._create_fake_vm_objects()
        query = vm_util.get_values_from_object_properties(
            fake.FakeObjectRetrievalSession(objects), objects)
        self.assertEqual('poweredOn', query['runtime.powerState'])
        self.assertEqual('guestToolsRunning',
                         query['summary.guest.toolsRunningStatus'])
        self.assertEqual('toolsOk', query['summary.guest.toolsStatus'])

    def test_reconfigure_vm(self):
        session = fake.FakeSession()
        with contextlib.nested(
            mock.patch.object(session, '_call_method',
                              return_value='fake_reconfigure_task'),
            mock.patch.object(session, '_wait_for_task')
        ) as (_call_method, _wait_for_task):
            vm_util.reconfigure_vm(session, 'fake-ref', 'fake-spec')
            _call_method.assert_called_once_with(mock.ANY,
                'ReconfigVM_Task', 'fake-ref', spec='fake-spec')
            _wait_for_task.assert_called_once_with(
                'fake_reconfigure_task')

    def test_get_network_attach_config_spec_opaque(self):
        vif_info = {'network_name': 'br-int',
            'mac_address': '00:00:00:ca:fe:01',
            'network_ref': {'type': 'OpaqueNetwork',
                            'network-id': 'fake-network-id',
                            'network-type': 'opaque'},
            'iface_id': 7,
            'vif_model': 'VirtualE1000'}
        result = vm_util.get_network_attach_config_spec(
                fake.FakeFactory(), vif_info, 1)
        card = 'ns0:VirtualEthernetCardOpaqueNetworkBackingInfo'
        expected = """{
            'extraConfig': [{'value': 7,
                             'key': 'nvp.iface-id.1',
                             'obj_name':'ns0:OptionValue'}],
            'deviceChange': [
                {'device': {
                     'macAddress':'00:00:00:ca:fe:01',
                     'addressType': 'manual',
                     'connectable': {
                         'allowGuestControl':True,
                         'startConnected': True,
                         'connected': True,
                         'obj_name':'ns0:VirtualDeviceConnectInfo'},
                     'backing': {
                         'opaqueNetworkType': 'opaque',
                         'opaqueNetworkId': 'fake-network-id',
                         'obj_name': '%(card)s'},
                     'key': -47,
                     'obj_name': 'ns0:VirtualE1000',
                     'wakeOnLanEnabled': True},
                 'operation': 'add',
                 'obj_name': 'ns0:VirtualDeviceConfigSpec'}],
            'obj_name':'ns0:VirtualMachineConfigSpec'}""" % {'card': card}
        expected = re.sub(r'\s+', '', expected)
        result = re.sub(r'\s+', '', repr(result))
        self.assertEqual(expected, result)

    def test_get_network_attach_config_spec_dvs(self):
        vif_info = {'network_name': 'br100',
            'mac_address': '00:00:00:ca:fe:01',
            'network_ref': {'type': 'DistributedVirtualPortgroup',
                            'dvsw': 'fake-network-id',
                            'dvpg': 'fake-group'},
            'iface_id': 7,
            'vif_model': 'VirtualE1000'}
        result = vm_util.get_network_attach_config_spec(
                fake.FakeFactory(), vif_info, 1)
        port = 'ns0:DistributedVirtualSwitchPortConnection'
        backing = 'ns0:VirtualEthernetCardDistributedVirtualPortBackingInfo'
        expected = """{
            'extraConfig': [{'value': 7,
                             'key': 'nvp.iface-id.1',
                             'obj_name': 'ns0:OptionValue'}],
            'deviceChange': [
                {'device': {'macAddress': '00:00:00:ca:fe:01',
                            'addressType': 'manual',
                            'connectable': {
                                'allowGuestControl': True,
                                'startConnected': True,
                                'connected': True,
                                'obj_name': 'ns0:VirtualDeviceConnectInfo'},
                 'backing': {
                     'port': {
                         'portgroupKey': 'fake-group',
                         'switchUuid': 'fake-network-id',
                         'obj_name': '%(obj_name_port)s'},
                     'obj_name': '%(obj_name_backing)s'},
                     'key': -47,
                     'obj_name': 'ns0:VirtualE1000',
                     'wakeOnLanEnabled': True},
                 'operation': 'add',
                 'obj_name': 'ns0:VirtualDeviceConfigSpec'}],
            'obj_name':'ns0:VirtualMachineConfigSpec'}""" % {
                    'obj_name_backing': backing,
                    'obj_name_port': port}
        expected = re.sub(r'\s+', '', expected)
        result = re.sub(r'\s+', '', repr(result))
        self.assertEqual(expected, result)

    def test_get_network_detach_config_spec(self):
        result = vm_util.get_network_detach_config_spec(
                fake.FakeFactory(), 'fake-device', 2)
        expected = """{
            'extraConfig': [{'value': 'free',
                             'key': 'nvp.iface-id.2',
                             'obj_name': 'ns0:OptionValue'}],
            'deviceChange': [{'device': 'fake-device',
                              'operation': 'remove',
                              'obj_name': 'ns0:VirtualDeviceConfigSpec'}],
            'obj_name':'ns0:VirtualMachineConfigSpec'}"""
        expected = re.sub(r'\s+', '', expected)
        result = re.sub(r'\s+', '', repr(result))
        self.assertEqual(expected, result)

    @mock.patch.object(vm_util, "get_vm_ref")
    def test_power_off_instance(self, fake_get_ref):
        session = fake.FakeSession()
        fake_instance = mock.MagicMock()
        with contextlib.nested(
            mock.patch.object(session, '_call_method',
                              return_value='fake-task'),
            mock.patch.object(session, '_wait_for_task')
        ) as (fake_call_method, fake_wait_for_task):
            vm_util.power_off_instance(session, fake_instance, 'fake-vm-ref')
            fake_call_method.assert_called_once_with(session.vim,
                                                     "PowerOffVM_Task",
                                                     'fake-vm-ref')
            fake_wait_for_task.assert_called_once_with('fake-task')
            self.assertFalse(fake_get_ref.called)

    @mock.patch.object(vm_util, "get_vm_ref", return_value="fake-vm-ref")
    def test_power_off_instance_no_vm_ref(self, fake_get_ref):
        session = fake.FakeSession()
        fake_instance = mock.MagicMock()
        with contextlib.nested(
            mock.patch.object(session, '_call_method',
                              return_value='fake-task'),
            mock.patch.object(session, '_wait_for_task')
        ) as (fake_call_method, fake_wait_for_task):
            vm_util.power_off_instance(session, fake_instance)
            fake_get_ref.assert_called_once_with(session, fake_instance)
            fake_call_method.assert_called_once_with(session.vim,
                                                     "PowerOffVM_Task",
                                                     'fake-vm-ref')
            fake_wait_for_task.assert_called_once_with('fake-task')

    @mock.patch.object(vm_util, "get_vm_ref")
    def test_power_off_instance_with_exception(self, fake_get_ref):
        session = fake.FakeSession()
        fake_instance = mock.MagicMock()
        with contextlib.nested(
            mock.patch.object(session, '_call_method',
                              return_value='fake-task'),
            mock.patch.object(session, '_wait_for_task',
                              side_effect=exception.NovaException('fake'))
        ) as (fake_call_method, fake_wait_for_task):
            self.assertRaises(exception.NovaException,
                              vm_util.power_off_instance,
                              session, fake_instance, 'fake-vm-ref')
            fake_call_method.assert_called_once_with(session.vim,
                                                     "PowerOffVM_Task",
                                                     'fake-vm-ref')
            fake_wait_for_task.assert_called_once_with('fake-task')
            self.assertFalse(fake_get_ref.called)

    @mock.patch.object(vm_util, "get_vm_ref")
    def test_power_off_instance_power_state_exception(self, fake_get_ref):
        session = fake.FakeSession()
        fake_instance = mock.MagicMock()
        with contextlib.nested(
            mock.patch.object(session, '_call_method',
                              return_value='fake-task'),
            mock.patch.object(
                    session, '_wait_for_task',
                    side_effect=vexc.InvalidPowerStateException)
        ) as (fake_call_method, fake_wait_for_task):
            vm_util.power_off_instance(session, fake_instance, 'fake-vm-ref')
            fake_call_method.assert_called_once_with(session.vim,
                                                     "PowerOffVM_Task",
                                                     'fake-vm-ref')
            fake_wait_for_task.assert_called_once_with('fake-task')
            self.assertFalse(fake_get_ref.called)

    def test_get_vm_create_spec_updated_hw_version(self):
        extra_specs = vm_util.ExtraSpecs(hw_version='vmx-08')
        instance_uuid = uuidutils.generate_uuid()
        fake_instance = {'id': 7, 'name': 'fake!',
                         'uuid': instance_uuid,
                         'vcpus': 2, 'memory_mb': 2048}
        result = vm_util.get_vm_create_spec(fake.FakeFactory(),
                                            fake_instance, instance_uuid,
                                            'fake-datastore', [],
                                            extra_specs=extra_specs)
        self.assertEqual('vmx-08', result.version)

    def test_vm_create_spec_with_profile_spec(self):
        instance_uuid = uuidutils.generate_uuid()
        fake_instance = {'id': 7, 'name': 'fake!',
                         'uuid': instance_uuid,
                         'vcpus': 2, 'memory_mb': 2048}
        datastore = ds_util.Datastore('fake-ds-ref', 'fake-ds-name')
        extra_specs = vm_util.ExtraSpecs()
        create_spec = vm_util.get_vm_create_spec(fake.FakeFactory(),
                                            fake_instance, instance_uuid,
                                            datastore.name, [],
                                            extra_specs,
                                            profile_spec='fake_profile_spec')
        self.assertEqual(['fake_profile_spec'], create_spec.vmProfile)

    @mock.patch.object(pbm, 'get_profile_id_by_name')
    def test_get_storage_profile_spec(self, mock_retrieve_profile_id):
        fake_profile_id = fake.DataObject()
        fake_profile_id.uniqueId = 'fake_unique_id'
        mock_retrieve_profile_id.return_value = fake_profile_id
        profile_spec = vm_util.get_storage_profile_spec(fake.FakeSession(),
                                                        'fake_policy')
        self.assertEqual('ns0:VirtualMachineDefinedProfileSpec',
                         profile_spec.obj_name)
        self.assertEqual(fake_profile_id.uniqueId, profile_spec.profileId)

    @mock.patch.object(pbm, 'get_profile_id_by_name')
    def test_storage_spec_empty_profile(self, mock_retrieve_profile_id):
        mock_retrieve_profile_id.return_value = None
        profile_spec = vm_util.get_storage_profile_spec(fake.FakeSession(),
                                                        'fake_policy')
        self.assertIsNone(profile_spec)

    def test_get_ephemeral_name(self):
        filename = vm_util.get_ephemeral_name(0)
        self.assertEqual('ephemeral_0.vmdk', filename)

@mock.patch.object(driver.VMwareAPISession, 'vim', stubs.fake_vim_prop)
class VMwareVMUtilGetHostRefTestCase(test.NoDBTestCase):
    # N.B. Mocking on the class only mocks test_*(), but we need
    # VMwareAPISession.vim to be mocked in both setUp and tests. Not mocking in
    # setUp causes object initialisation to fail. Not mocking in tests results
    # in vim calls not using FakeVim.
    @mock.patch.object(driver.VMwareAPISession, 'vim', stubs.fake_vim_prop)
    def setUp(self):
        super(VMwareVMUtilGetHostRefTestCase, self).setUp()
        fake.reset()
        vm_util.vm_refs_cache_reset()

        self.session = driver.VMwareAPISession()

        # Create a fake VirtualMachine running on a known host
        self.host_ref = fake._db_content['HostSystem'].keys()[0]
        self.vm_ref = fake.create_vm(host_ref=self.host_ref)

    @mock.patch.object(vm_util, 'get_vm_ref')
    def test_get_host_ref_for_vm(self, mock_get_vm_ref):
        mock_get_vm_ref.return_value = self.vm_ref

        ret = vm_util.get_host_ref_for_vm(self.session, 'fake-instance')

        mock_get_vm_ref.assert_called_once_with(self.session, 'fake-instance')
        self.assertEqual(self.host_ref, ret)

    @mock.patch.object(vm_util, 'get_vm_ref')
    def test_get_host_name_for_vm(self, mock_get_vm_ref):
        mock_get_vm_ref.return_value = self.vm_ref

        host = fake._get_object(self.host_ref)

        ret = vm_util.get_host_name_for_vm(self.session, 'fake-instance')

        mock_get_vm_ref.assert_called_once_with(self.session, 'fake-instance')
        self.assertEqual(host.name, ret)


import json
from elasticsearch import Elasticsearch
from elasticsearch import helpers

es_conf = [{"host": '0.0.0.0', "port": 9400}]

es = Elasticsearch("http://0.0.0.0:9400")

def create_index(index_name):
    global es
    indices = es.indices.get_alias().keys()
    if index_name in indices:
        print('已创建过index')
        return

    settings = {
        "settings": {
            "index": {
                "number_of_shards": "1",
                "number_of_replicas": "0"
            }
        },
        "mappings": {
            "properties": {
                # "id": {
                #     "type": "text"
                # },
                "triple_key": {
                    "type": "text",
                    "analyzer": "ik_smart",
                    "search_analyzer": "ik_smart",
                    "index": True
                },
                "triple": {
                    "type": "text"
                }
            }

        }
    }
    # 发送到ES
    es.indices.create(index=index_name, body=json.dumps(settings))
    print('创建完成')

def insert_into_es_bulk(data_json, index_name):
    global es
    actions = []
    for i in data_json:
        action = {
            "_index": index_name,
            "_type": "_doc",
            "_source": {
            }
        }
        # id = i[2]
        # action['_id'] = id
        action['_source'] = {
            "triple_key": i[0],
            "triple": i[1]
        }
        actions.append(action)

    helpers.bulk(es, actions)

def get_from_es():
    es = Elasticsearch()
    res = es.search()
    print("Got %d Hits:" % res['hits']['total']['value'])
    for hit in res['hits']['hits']:
        # print("%(timestamp)s %(author)s: %(text)s" % hit["_source"])
        print(hit["_source"])

def main(index_name):
    # 取数据

    data_list = []
    with open("knowledge/Knowledge_20211215.txt", "r") as f:
        lines = f.readlines()
        for line in lines:
            tmp = line.split("\t")
            data_list.append([tmp[0] + " " + tmp[1], tmp[0] + " ||| " + tmp[1] + " ||| " + tmp[2]])

    # 建索引
    create_index(index_name)

    # 存入es
    # insert_into_es(data, index_name)
    insert_into_es_bulk(data_list, index_name)

def delete_index(index_name):
    global es
    res = es.indices.delete(index_name)
    print(res)

if __name__ == '__main__':
    index_name = 'kgclue'
    delete_index(index_name)
    main(index_name)