import copy
from typing import List

import torch

from torchsim.core.graph.unit import Unit
from torchsim.core.memory.tensor_creator import TensorCreator
from torchsim.core.graph.worker_node_base import WorkerNodeBase
from torchsim.core.graph.slot_container import Inputs, MemoryBlocks
from torchsim.core.nodes.disentangled_world_node import DisentangledWorldNodeParams
from torchsim.core.physics_model.pymunk_physics import PymunkParams

class DisentangledWorldRendererUnit(Unit):
    bitmap: torch.Tensor
    _params: DisentangledWorldNodeParams

    def __init__(self, creator: TensorCreator, params: DisentangledWorldNodeParams):
        super().__init__(creator.device)

        self._params = copy.copy(params)

        self._render_world = params.render_world_class((params.sx, params.sy), **params.render_world_params)

        width = self._render_world.world_size[0]
        height = self._render_world.world_size[1]
        self.bitmap = creator.zeros((width, height, 3),
                                     dtype=self._float_dtype,
                                     device=self._device)

    def step(self, instances: torch.Tensor):
        params = PymunkParams()
        self._params.sx = params.sx
        self._params.sy = params.sy
        self._params.shape_max = params.shape_max
        self._params.color_max = params.color_max
        self._params.world_dims = params.world_dims
        self._params.max_velocity = params.max_velocity
        self.bitmap.copy_(self._render_world.to_tensor(instances, params))

class DisentangledWorldRendererInputs(Inputs):
    def __init__(self, owner):
        super().__init__(owner)
        self.latent = self.create("Latent vars")

class DisentangledWorldRendererOutputs(MemoryBlocks):
    def __init__(self, owner):
        super().__init__(owner)
        self.bitmap = self.create("Bitmap")

    def prepare_slots(self, unit: DisentangledWorldRendererUnit):
        self.bitmap.tensor = unit.bitmap

class DisentangledWorldRendererNode(WorkerNodeBase[DisentangledWorldRendererInputs, DisentangledWorldRendererOutputs]):

    def __init__(self, params: DisentangledWorldNodeParams, name="DisentangledWorldRenderer"):
        super().__init__(name=name,
                         inputs=DisentangledWorldRendererInputs(self),
                         outputs=DisentangledWorldRendererOutputs(self))

        self._params = params.clone()

    def _create_unit(self, creator: TensorCreator) -> Unit:
        self._creator = creator

        return DisentangledWorldRendererUnit(creator, self._params)

    def _step(self):
        self._unit.step(self.inputs.latent.tensor)


from flask import Response, request
from database.models import Movie, User
from flask_restplus import Resource,fields
from flask_jwt_extended import jwt_required, get_jwt_identity
from mongoengine.errors import FieldDoesNotExist, NotUniqueError, \
                                DoesNotExist,ValidationError
import resources.errors as errors
from app import api

class MoviesApi(Resource):
    @api.response(200, 'Success')
    @api.response(400, 'Validation Error')
    def get(self):
        movies = Movie.objects().to_json()
        return Response(movies, mimetype="application/json", status = 200)

    @jwt_required()
    def post(self):
        try:
            user_id = get_jwt_identity()
            body = request.get_json()
            user = User.objects.get(id=user_id)
            movie = Movie(**body, added_by= user)
            movie.save()
            user.update(push__movies=movie)
            user.save()
            id = movie.id
            return {"id": str(id)}, 200
        except(FieldDoesNotExist, ValidationError):
            raise errors.SchemaValidationError
        except(NotUniqueError):
            raise errors.MovieAlreadyExistsError
        except Exception as e:
            raise errors.InternalServerError

class MoviesApiId(Resource):
    @api.response(200, 'Success')
    @api.response(400, 'Validation Error')
    def get(self, id):
        movie = Movie.objects(id=id).to_json()
        return Response(movie, mimetype="application/json", status = 200)

    @jwt_required()
    def put(self,id):
        try:
            user_id = get_jwt_identity()
            Movie.objects.get(id=id, added_by=user_id)
            body = request.get_json()
            Movie.objects.get(id = id).update(**body)
            return '', 200
        except errors.InvalidQueryError:
           raise errors.SchemaValidationError
        except DoesNotExist:
           raise errors.UpdatingMovieError
        except Exception:
           raise errors.InternalServerError

    @jwt_required()
    def delete(self, id):
        try:
            user_id = Movie.objects.get(id=id).delete()
            movie = Movie.objects.get(id=id, added_by=user_id)
            movie.delete()
            return '', 200
        except DoesNotExist:
            raise errors.MovieNotExistsError
        except Exception:
            raise errors.InternalServerError


from django.shortcuts import render, get_object_or_404, redirect

from django.http import HttpResponse
from django.http import Http404

from .models import Articles
from random import randint

# Create your views here.

# def articles_home(request):
#     obj_query_set = Articles.objects.all()
#     string1 = '<h1> I am from articles articles_home view</h1>'
#     return HttpResponse(string1)

def articles_home(request):
    # get set of all objects of Articles class
    obj_query_set = Articles.objects.all()

    # get all objects id queryset
    all_ids = Articles.objects.values_list('id', flat = True)  # it is an iterable set
    all_ids_list = list(all_ids) # it's a list of ids

    # let's create random id
    random_id = randint(1, len(all_ids))
    random_obj = Articles.objects.get(id=random_id)

    # creating context
    context = {
        'all_obj' : Articles.objects.all(),
        'rand_obj' : random_obj
    }
    return render(request, 'articles/articles_home.html', context)

def article_detail(request, id):
    all_ids = Articles.objects.values_list('id', flat = True)  # it is an iterable set
    object = Articles.objects.get(id = id) if id in all_ids else None
    return render(request, 'articles/detail.html', {'obj' : object})

# Raising a 404 error
def article_random_detail(request, id):
    try:
        object = Articles.objects.get(pk=id)  # this is same as Articles.objects.get(id = id)
    except Articles.DoesNotExist:
        raise Http404('Article does not exist')

    # A shortcut: get_object_or_404()
    # we can use it instead of try except statement
    # from django.shortcuts import render, get_object_or_404
    # object = get_object_or_404(Articles, pk=id)

    return render(request, 'articles/detail.html', {'obj' : object})

# view for search id
def article_search_detail(request):
    # print(dir(request))
    print(request.GET) # look at terminal to see the output of print function

    # extracting data from the request using get method
    query_dict = request.GET
    # query_dict is a dictionary
    # but it is not possible to get data writing like request.GET ['q] or query_dict['q]
    # we sould grab data using 'get' function

    query = query_dict.get('q')   # <input type="text"  name="q">
    # or, query = rquest.GET.get('q)

    # get data fffrom database
    object = get_object_or_404(Articles, pk=query)
    return render(request, 'articles/search.html', {'obj' : object})
    # we could use detail page instead of search page like below
    # return render(request, 'articles/detail.html', {'obj' : object})
    # it would be good for us as we don't need to create extra search.html page

def article_create(request):

    # sometimes GET and POST methods are called simultaneously
    # on that time, GET request will show 'None, None' value in terminal
    # so to avoid this we should use the following code

    context = {}
    if request.method == 'POST':
        title = request.POST.get('title')
        content = request.POST.get('content')
        print(title, content)
        object = Articles.objects.create(title = title, content = content)
        context['obj'] = object
        context['created'] = True

    """
    title = request.POST.get('title')
    content = request.POST.get('content')
    print(title, content)
    object = Articles.objects.create(title = title, content = content)
    context = {
        'obj' : object,
        'created' : True     # this value indicates that for is created
    }
    """
    return render(request, 'articles/create.html', context = context)

# redirect problem
def article_create_redirect(request):
    if request.method == 'POST':
        title = request.POST.get('title')
        content = request.POST.get('content')
        print(title, content)
        obj = Articles.objects.create(title=title, content=content)
        return redirect('articles:articles_home')
        # redirect('app_name:url_name')
        # here redirected to article detail page

        # if i want to redirect to article_detail page, i have to insert an extra argument for 'id'
        # redirect('app_name:url_name', 'argument_needed_for_url')
        # for instance, redirect('articles:article_detail', obj.id)

    return render(request, 'articles/create.html')

"""
I apologize for the misunderstanding.
The `autocomplete="off"` attribute is used to prevent browsers from autofilling form fields,
but it does not prevent form resubmission when navigating back and forward.

To prevent duplicate form submissions when navigating back and forward,
you can implement a technique called Post/Redirect/Get (PRG).
The PRG pattern involves redirecting the user to another URL after successfully submitting the form.
This way, when the user navigates back,
they will be taken to the redirected URL instead of resubmitting the form.

Here's an example of how you can modify your `article_create` function to implement the PRG pattern:

```python
from django.shortcuts import redirect

def article_create(request):
    if request.method == 'POST':
        title = request.POST.get('title')
        content = request.POST.get('content')
        print(title, content)
        article = Articles.objects.create(title=title, content=content)
        return redirect('article_detail', id=article.id)  # Redirect to article detail page

    return render(request, 'articles/create.html')
```

In this example, after creating the `Articles` object,
the user is redirected to the `article_detail` page for the newly created article.
This redirection helps avoid form resubmission when the user navigates back to the form page.

Make sure to replace `'article_detail'` with the actual URL name or path for
your article detail page in the `redirect()` function.

By implementing the PRG pattern,
you should prevent duplicate form submissions when navigating back and forward in the browser.

"""


import argparse
import glob
import json
import msgpack
import numpy as np
import torch

from transformers import OpenAIGPTTokenizer, OpenAIGPTLMHeadModel
from transformers import GPT2Tokenizer, GPT2LMHeadModel
from transformers import XLNetTokenizer, XLNetLMHeadModel
from transformers import TransfoXLTokenizer, TransfoXLLMHeadModel

parser = argparse.ArgumentParser(description='Collect per positions losses.')
parser.add_argument('--save', type=str, default='../results/results_books.npy',
                    help='Name of save file')
parser.add_argument('--data', type=str, default='../data/wiki40b/',
                    help='Path to data')
parser.add_argument('--batch', type=int, default=8,
                    help='Batch size for SRL parser.')
parser.add_argument('--dataset', type=str, default='books',
                    choices=['books', 'wiki', 'litbank', 'research'], help='Dataset type')
args = parser.parse_args()

def get_gpt():
    tokenizer = OpenAIGPTTokenizer.from_pretrained('openai-gpt')
    model = OpenAIGPTLMHeadModel.from_pretrained('openai-gpt')
    model.eval()
    return tokenizer, model

def get_gpt2():
    tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
    model = GPT2LMHeadModel.from_pretrained('gpt2')
    model.eval()
    return tokenizer, model

def get_tranxl():
    tokenizer = TransfoXLTokenizer.from_pretrained('transfo-xl-wt103')
    model = TransfoXLLMHeadModel.from_pretrained('transfo-xl-wt103')
    model.eval()
    return tokenizer, model

def get_xlnet(): # Doesn't work
    tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased')
    model = XLNetLMHeadModel.from_pretrained('xlnet-large-cased')
    model.eval()
    return tokenizer, model

def get_wiki_data(path):
    files = glob.glob(path+"/*")
    for fname in files:
        with open(fname, "rb") as buf:
            unpacker = msgpack.Unpacker(buf, raw=False)
            for unpacked in unpacker:
                yield unpacked

def get_books_data(path):
    data = []
    collect = True
    with open(path, "r") as f:
        one_book = []
        for i, line in enumerate(f):
            if collect:
                one_book.append(line)
                if len(one_book) == 50:
                    data.append(one_book)
                    one_book = []
                    collect = False
            if i % 10000 == 0:
                collect = True
    all_data = []
    for book in data:
        lines = " ".join([l.strip() for l in book])
        all_data.append(lines)
    for d in all_data[1:]:
        yield d

def get_litbank_data(path):
    litbank_file = glob.glob(path + "*")
    data = []
    for fname in litbank_file:
        book = []
        section = []
        with open(fname, "r") as f:
            spaces = 0
            for line in f:
                if line.isspace():
                    spaces += 1
                else:
                    if spaces < 2:
                        section.append(line.strip())
                    else:
                        book.append(section)
                        section = []
                        section.append(line.strip())
                    spaces = 0
            if section:
                book.append(section)
        data.append(book)

    all_data = []
    for book in data:
        new_book = [" ".join(sec) for sec in book if len(sec) > 50]
        all_data.append(new_book)

    all_data = [sec for book in all_data for sec in book]
    for d in all_data:
        yield d

def get_research_data(path):
    filenames = glob.glob(path+"/*.json")
    all_intros = []
    for path in filenames:
        with open(path, "r") as f:
            doc = json.load(f)
            intro = []
            for ent in doc['body_text']:
                if "Introduction" == ent['title']:
                    intro.append((ent['startOffset'], ent['sentence']))
            intro = sorted(intro)
            intro = [b for a, b in intro]
            intro = " ".join(intro)
            all_intros.append(intro)
    for d in all_intros:
        yield d

def  main(args):
    # TODO: check uninitialized weights
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

    if args.dataset == "books":
        get_data = get_books_data
    elif args.dataset == "litbank":
        get_data = get_litbank_data
    elif args.dataset == "research":
        get_data = get_research_data
    else:
        get_data = get_wiki_data

    configs = {"gpt": get_gpt,
               "gpt2": get_gpt2,
               "tranxl": get_tranxl} #,
               # "xlnet": get_xlnet}
    for name, get_config in configs.items():
        print("Running", name)
        paragraphs = get_data(args.data)

        tokenizer, model = get_config()
        model = model.to(device)
        loss_fct = torch.nn.CrossEntropyLoss(reduction="none")

        done = False
        count = 0
        while not done:
            tokenized_paras = []
            for para in paragraphs:
                toked = tokenizer.encode(para, return_tensors="pt", truncation=True, max_length=512)
                if toked.shape[1] >= 512:
                    tokenized_paras.append(toked[:, :512])
                    if len(tokenized_paras) == args.batch:
                        break
            if len(tokenized_paras) < args.batch:
                done = True
            if not tokenized_paras:
                continue
            tokenized_paras = torch.cat(tokenized_paras, axis=0)
            tokenized_paras = tokenized_paras.to(device)

            with torch.no_grad():
                out = model(tokenized_paras, labels=tokenized_paras, return_dict=True)
                if name == "tranxl":
                    loss = out.losses.cpu()
                else:
                    shift_logits = out.logits[..., :-1, :].contiguous()
                    shift_labels = tokenized_paras[..., 1:].contiguous()
                    # Flatten the tokens
                    loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))
                save_results(loss.cpu().numpy(), args.save, name)
                count += 1
                if count > 3000:
                    break
    return

def save_results(loss, savefile, key_name):
    filename = savefile.replace(".npy", "-" + key_name + ".npy")
    print("saving to filename...", filename)
    with open(filename, "ab") as f:
        np.save(f, loss)

if __name__ == "__main__":
    main(args)


import random
import curses
from curses import initscr,curs_set,newwin,endwin,KEY_RIGHT,KEY_LEFT,KEY_UP,KEY_DOWN,beep,start_color,noecho;
from random import randrange;
scr=initscr()
noecho()
start_color()
#N=30 # heigt of field
#M=60 # width of field
N=scr.getmaxyx()[0]  # heigt of field
M=scr.getmaxyx()[1]  # width of field
count=0 # codes collected initally
d=4 # no. of codes initally
b=1 # no. of bomb initally
s=10 # score added when bomb is diffused
score=0 # inital score
level=0	# initalising level
flag=0
Life=3 # no. of chances
e=0 # no. of enemies
L=5# no. of levels +1
sc=1 # score added when a code is collected
bomb=0 # no. of bomb diffused
space=5 # border width from edges in which the bomb and codes should not be placed
defaultkey=KEY_RIGHT # direction in which robo move initally
x=5
y=3
#-------------------------------------------------------------------------------------------------------------------------------------
#different robo bomb desis designs , same for codes
import design
#-------------------------------------------------------------------------------------
class Keys:
	quit=[27,ord('q'),ord('Q')]
	up=[ord('w'),ord('W'),KEY_UP]
	down=[ord('s'),ord('S'),KEY_DOWN]
	left=[ord('a'),ord('A'),KEY_LEFT]
	right=[ord('d'),ord('D'),KEY_RIGHT]
	pause=[ord('p'),ord('P')]
	allkey=quit+up+down+right+left+pause
	choice=[ord('0'),ord('1'),ord('2'),ord('3'),ord('4'),ord('5'),ord('6'),ord('7'),ord('8'),ord('9')]
#-----------------------------------------------------------------------------------------
class Screen:
	def __init__(self):
		curs_set(0);
		self.height=N
		self.width=M
		self.win=newwin(N,M,0,0)
		self.win.border('|','|','~','~','+','+','+','+');
		self.win.keypad(1)
	def end(self):
		endwin()
	def clear(self):
		screen.win.clear()
		self.win.border('|','|','~','~','+','+','+','+');
	def window(self):
		return self.win
#-------------------------------------------------------------------------------------------------------------------------------------
screen=Screen()
#-------------------------------------------------------------------------------------------------------------------------------------
class Enemy:
	def __init__(self,choice,fieldpos,robopos):
		self.shape=design.roboshape(choice)
		self.pos=[]
		self.places=reduce(lambda x,y:x+y,robopos,[])
		self.fieldpos=reduce(lambda x,y:x+y,fieldpos,[])
		self.places.extend(self.fieldpos)
		self.enemycount=e
		self.enemypos=[]
		self.height=N
		self.width=M
		self.choice=choice
		self.check=[]
	def Enemyposdecider(self):
		self.enemypos=[]
		while (len(self.enemypos)!=self.enemycount):
			plac=[random.randrange(space,self.width-space),random.randrange(space,self.height-space)]
			if plac not in self.places and plac not in self.enemypos:
				if plac:
					self.enemypos.append(plac)
	def Enemtinit(self):
		ecount=0
		while (ecount!=self.enemycount):
			self.pos.append(design.roboposition(self.choice,x,y))
			ecount+=1
	def checkexistence(self,ecount):
			check1=reduce(lambda x,y:x+y,self.pos[ecount],[])
			check2=map(lambda xx:filter(lambda yy: xx==yy,check1),self.fieldpos)
			self.check=reduce(lambda x,y:x+y,check2,[])
	def move(self):
		ecount=0
		i=1
		while (ecount!=self.enemycount):
			plac=self.enemypos[ecount]
			self.enemypos[ecount]=[(plac[0]+2*i)%(M-space),(plac[1])%(N-space)]
			plac=self.enemypos[ecount]
			if plac not in self.fieldpos and plac[0]>=space and plac[1]>=space:
				self.checkexistence(ecount)
				if len(self.check)==0:
					map(lambda xx:map(lambda y:screen.win.addch(self.pos[ecount][xx][y][1],self.pos[ecount][xx][y][0],' '),range(0,len(self.pos[ecount][xx]))),range(0,len(self.pos[ecount])))
				x=plac[0]
				y=plac[1]
				self.pos[ecount]=design.roboposition(self.choice,x,y)
				self.checkexistence(ecount)
				if len(self.check)==0:
					map(lambda xx:map(lambda yy:screen.win.addch(self.pos[ecount][xx][yy][1],self.pos[ecount][xx][yy][0],self.shape[xx][yy],curses.A_BOLD),range(0,len(self.pos[ecount][xx]))),range(0,len(self.pos[ecount])))
			ecount+=1
	def getpos(self):
		return self.pos
	def whatatpos(self):
		return self.wap
#-------------------------------------------------------------------------------------------------------------------------------------
class Robot:
	def __init__(self,choice,i,j):
		self.shape=design.roboshape(choice)
		self.pos=design.roboposition(choice,i,j)
		self.x=i
		self.y=j
		self.choice=choice
		self.wap=[]
	def move(self,i,j):
		map(lambda x:map(lambda y:screen.win.addch(self.pos[x][y][1],self.pos[x][y][0],' '),range(0,len(self.pos[x]))),range(0,len(self.pos)))
		self.x+=i
		self.y+=j
		self.pos=design.roboposition(self.choice,self.x,self.y)
		self.wap=reduce(lambda xx,yy:xx+yy,map(lambda xx:map(lambda yy:screen.win.inch(self.pos[xx][yy][1],self.pos[xx][yy][0]) & 255,range(0,len(self.pos[xx]))),range(0,len(self.pos))),[])
		map(lambda xx:map(lambda yy:screen.win.addch(self.pos[xx][yy][1],self.pos[xx][yy][0],self.shape[xx][yy],curses.A_BOLD),range(0,len(self.pos[xx]))),range(0,len(self.pos)))
	def getpos(self):
		return self.pos
	def whatatpos(self):
		return self.wap
#------------------------------------------------------------------------------------------------------------------------------------
class Intro:
	Key=Keys()
	def __init__(self):
		self.flag=0
		self.choice=0
		self.choice=1
		self.EnemyRobochoice=5
		self.PoliceRobochoice=4
		self.bombchoice=6
		self.codechoice=5
	def firstscreen(self):
		N=scr.getmaxyx()[0]  # heigt of field
		M=scr.getmaxyx()[1]  # width of field
		screen.win.addstr((N/2)-4,M/2,'  1.Start Game  ',curses.A_REVERSE)
		screen.win.addstr((N/2)-2,M/2,'  2.Settings  ',curses.A_REVERSE)
		screen.win.addstr((N/2),M/2,'  3.Instructions  ',curses.A_REVERSE)
		screen.win.addstr((N/2)+2,M/2,'  4.About  ',curses.A_REVERSE)
		screen.win.addstr((N/2)+6,M/2,'  6.Credits  ',curses.A_REVERSE)
		screen.win.addstr((N/2)+4,M/2,'  5.Exit the game  ',curses.A_REVERSE)
		self.choice=screen.win.getch()
		screen.clear()
		if self.choice==ord('1'):
			i=0
#			screen.win.addstr(N/2,M/2,'LOADING PLEASE WAIT.........',curses.A_BOLD)
			load="LOADING PLEASE WAIT........."
			for data in load:
				screen.win.addstr(N/2,M/2+i,data,curses.A_BOLD)
				screen.win.timeout(200)
				screen.win.getch()
				i+=1
			screen.clear()
		elif self.choice==ord('2'):
			screen.clear()
			self.secondscreen()
		elif self.choice==ord('3'):
			File=open("Instruction.txt",'r')
			i=0
			for data in File:
				screen.win.addstr(N/6+i,M/7,data,curses.A_BOLD)
				i+=1
			screen.win.border('|','|','~','~','+','+','+','+');

			choice2=screen.win.getch()
			screen.clear()
			self.firstscreen()
		elif self.choice==ord('4'):
			File=open("About.txt",'r')
			i=0
			for data in File:
				screen.win.addstr(N/3+i,6,data,curses.A_BOLD)
				i+=1
			screen.win.border('|','|','~','~','+','+','+','+');
			choice2=screen.win.getch()
			screen.clear()
			self.firstscreen()
		elif self.choice==ord('5'):
			self.flag=1
			screen.clear()
		elif self.choice==ord('6'):
			File=open("Credits.txt",'r')
			i=0
			for data in File:
				screen.win.addstr(N/4+i,M/6,data,curses.A_BOLD)
				i+=1
			screen.win.border('|','|','~','~','+','+','+','+');
			choice2=screen.win.getch()
			screen.clear()
			self.firstscreen()
		else:
			screen.clear()
			self.firstscreen()
	def secondscreen(self):
		N=scr.getmaxyx()[0]  # heigt of field
		M=scr.getmaxyx()[1]  # width of field
		screen.win.addstr((N/2)-4,M/2,'  1.Choose Your robot  ',curses.A_REVERSE)
		screen.win.addstr((N/2)-2,M/2,'  2.Choose Enemy robot  ',curses.A_REVERSE)
		screen.win.addstr((N/2),M/2,'  3.Choose Bomb  ',curses.A_REVERSE)
		screen.win.addstr((N/2)+2,M/2,'  4.Choose Diffudecode  ',curses.A_REVERSE)
		screen.win.addstr((N/2)+4,M/2,'  5.Back  ',curses.A_REVERSE)
		choice1=screen.win.getch()
		screen.clear()
		if choice1==ord('1'):
			self.selectrobo()
			self.PoliceRobochoice=self.choice
			screen.clear()
			self.secondscreen()
		elif choice1==ord('2'):
			self.selectrobo()
			self.EnemyRobotchoice=self.choice
			screen.clear()
			self.secondscreen()
		elif choice1==ord('3'):
			self.selectbomb()
			self.bombchoice=self.choice
			screen.clear()
			self.secondscreen()
		elif choice1==ord('4'):
			self.selectbomb()
			self.codechoice=self.choice
			screen.clear()
			self.secondscreen()
		elif choice1==ord('5'):
			screen.clear()
			self.firstscreen()
		else:
			screen.clear()
			self.secondscreen()
	def selectrobo(self):
			screen.win.addstr(18,4,'  Choose from option  ',curses.A_BOLD)
			xx=0
			yy=0
			for i in range(0,10):
				xx+=8
				if(xx==48):
					xx=8
					yy+=8
				robot=Robot(i,xx,6+yy)
				robot.move(0,0)
				screen.win.addstr(2+yy,xx,str(i),curses.A_BOLD)
			self.choice=screen.win.getch()
			if self.choice in Key.choice:
				self.choice=self.choice-ord('0')
			else:
				screen.clear()
				self.selectrobo()
	def selectbomb(self):
			screen.win.addstr(18,4,'  Choose from option  ',curses.A_BOLD)
			xx=0
			yy=0
			for i in range(0,10):
				xx+=8
				if(xx==48):
					xx=8
					yy+=8
				field=Field([],[],0)
				robot=field.display(i,xx,6+yy)
				screen.win.addstr(2+yy,xx,str(i),curses.A_BOLD)
			self.choice=screen.win.getch()
			if self.choice in Key.choice:
				self.choice=self.choice-ord('0')
			else:
				screen.clear()
				self.selectbomb()
#----------------------------------------------------------------------------------------------------------------------------------
class Field:
	def __init__(self,robopos,fieldpos,count):
		self.fieldpos=[]
		self.fieldpos.extend(fieldpos)
		self.places=reduce(lambda x,y:x+y,robopos,[])
		self.places.extend(fieldpos)
		self.count=count
		self.height=N
		self.width=M
		self.pos=[]
		self.shape=[]
		self.position=[]
		self.check=[]
	def checkexistence(self,ecount):
			check1=reduce(lambda x,y:x+y,self.pos,[])
			check2=map(lambda xx:filter(lambda yy: xx==yy,check1),self.places)
			self.check=reduce(lambda x,y:x+y,check2,[])
	def display(self,choice,x,y):
			self.pos=design.bombposition(choice,x,y)
			self.shape=design.bombshape(choice)
			map(lambda xx:map(lambda yy:screen.win.addch(self.pos[xx][yy][1],self.pos[xx][yy][0],self.shape[xx][yy],curses.A_BOLD),range(0,len(self.pos[xx]))),range(0,len(self.pos)))

	def plant(self,choice):
		temp=[]
		while (len(self.position)!=self.count):
			plac=[random.randrange(space,self.width-space),random.randrange(space,self.height-space)]
			x=plac[0]
			y=plac[1]
			self.pos=design.bombposition(choice,x,y)
			self.shape=design.bombshape(choice)
			self.checkexistence(1)
			tt=reduce(lambda x,y:x+y,self.pos,[])
			if len(self.check)==0:
				if plac:
					self.position.append(self.pos)
					temp.append(tt)
					map(lambda xx:map(lambda yy:screen.win.addch(self.pos[xx][yy][1],self.pos[xx][yy][0],self.shape[xx][yy],curses.A_BOLD),range(0,len(self.pos[xx]))),range(0,len(self.pos)))
		self.fieldpos.extend(temp)
	def getpos(self):
		return self.position
	def getfieldpos(self):
		return self.fieldpos
#-------------------------------------------------------------------------------------------------------------------------------
Key=Keys()
key=defaultkey
intro =Intro()
intro.firstscreen()
robot =Robot(intro.PoliceRobochoice,5,5)
flag=intro.flag
life=Life
N=scr.getmaxyx()[0]  # heigt of field
M=scr.getmaxyx()[1]  # width of field
def checkexistence(b,a):
	check1=reduce(lambda x,y:x+y,a,[])
	check2=map(lambda xx:filter(lambda yy: xx==yy,check1),b)
	return reduce(lambda x,y:x+y,check2,[])
#------------------------------------------------------------------------------------------------------------------------------
if __name__ == "__main__":
	while (level != L and life!=0 and flag!=1):
		flag=0
		aa=[]
		robot.x=5
		robot.y=5
		position=robot.getpos()
		Bomb=Field(position,[],b)
		Bomb.plant(intro.bombchoice)
		fpos=Bomb.getfieldpos()
		Code=Field(position,fpos,d)
		Code.plant(intro.codechoice)
		fieldpos=Code.getfieldpos()
		enemy=Enemy(intro.EnemyRobochoice,fieldpos,position)
		enemy.Enemyposdecider()
		enemy.Enemtinit()
		count=0
		bomb=0
		while key not in Key.quit:
			try:
				screen.win.border('|','|','~','~','+','+','+','+');
				screen.win.addstr(0,2,'CODE COLLECTED:'+str(count)+' REMAINING:'+str(d-count),curses.A_BOLD)
				screen.win.addstr(0,M/2,'SCORE:'+str(score),curses.A_BOLD)
				screen.win.addstr(N-1,M/2,'LEVEL:'+str(level),curses.A_BOLD)
				screen.win.addstr(0,M/2+20,'LIFE:'+str(life),curses.A_BOLD)
				screen.win.addstr(N-1,1,'BOMB:'+str(bomb)+':DIFFUSED REMAINING:'+str(b-bomb),curses.A_BOLD)
				enemy.move()
				screen.win.timeout(180-L*10)
				getkey=screen.win.getch()
				key=key if getkey not in Key.allkey else getkey
				if(key in Key.right or key in Key.left or key in Key.up or key in Key.down):
					i=(key in Key.right and 1 or key in Key.left and -1)
					j= (key in Key.up and -1 or key in Key.down and 1)
					robot.move(i,j)
					aa=robot.whatatpos()
				elif(key in Key.pause):
					screen.win.border('|','|','~','~','+','+','+','+');
					screen.win.addstr(N-1,5,"PRESS 'p' OR 'P' TO RESUME",curses.A_BOLD)
					while 1:
						getkey=screen.win.getch()
						if(getkey in Key.pause):
							key=0
							screen.win.border('|','|','~','~','+','+','+','+');
							break
				enemypos=enemy.getpos()
				enemypos=reduce(lambda x,y:x+y,reduce(lambda x,y:x+y,enemypos,[]),[])
				robopos=robot.getpos()
				robopos=reduce(lambda x,y:x+y,robopos,[])
				collision1=map(lambda xx:filter(lambda yy: xx==yy,robopos),enemypos)
				collision2=filter(lambda xx:xx[0]==0 or xx[0]==M-1 or xx[1]==0 or xx[1]==N-1,robopos)
				collision1=reduce(lambda x,y:x+y,collision1,[])
				collision=collision1+collision2
				if len(collision)!=0:
					flag=2
					break
				for i in range(0,b-bomb):
					if(len(checkexistence(robopos,Bomb.position[i]))!=0):
						bombpos=Bomb.position[i]
						map(lambda xx:map(lambda yy:screen.win.addch(bombpos[xx][yy][1],bombpos[xx][yy][0],' '),range(0,len(bombpos[xx]))),range(0,len(bombpos)))
						Bomb.position.remove(bombpos)
						if count==d:
							score+=s
							bomb+=1
							screen.win.addstr(N-1,1,'BOMB:'+str(bomb)+':DIFFUSED REMAINING:'+str(b-bomb),curses.A_BOLD)
							break
						else:
							screen.win.addstr(N-1,1,'BOMB EXPLODED',curses.A_BOLD)
							beep()
							beep()
							flag=2
							break
				for i in range(0,d-count):
					if(len(checkexistence(robopos,Code.position[i]))!=0):
#						print "dd"
						codepos=Code.position[i]
						map(lambda xx:map(lambda yy:screen.win.addch(codepos[xx][yy][1],codepos[xx][yy][0],' '),range(0,len(codepos[xx]))),range(0,len(codepos)))
						Code.position.remove(codepos)
						count+=1
						score+=sc
						break
				if(bomb==b or flag!=0):
					break
			except curses.error:
				flag=2
				beep()
				break
			except KeyboardInterrupt:
				flag=1
				break
		screen.clear()
		if key in Key.quit:
			beep()
			break
		elif flag==0:
			d+=2
			b+=1
			s+=10
			sc+=1
			e+=1
			level+=1
			if life !=Life:
			  life+=1
		elif flag==2:
		 	life-=1
		key=defaultkey
		N=scr.getmaxyx()[0]  # heigt of field
		M=scr.getmaxyx()[1]  # width of field
	N=scr.getmaxyx()[0]  # heigt of field
	M=scr.getmaxyx()[1]  # width of field
	screen.win.addstr(N/2,M/8,"SCORE : "+ str(score),curses.A_BOLD)
	if(level==L and bomb==b-1):
		screen.win.addstr(N/2,(2*M)/5,"CONGRATULATIONS YOU WON",curses.A_BOLD)
		screen.win.addstr(N/2,(3*M)/4,"LEVEL : "+ str(level-1),curses.A_BOLD)
	else:
		screen.win.addstr(N/2,(2*M)/5,"GAME OVER",curses.A_BOLD)
		screen.win.addstr(N/2,(3*M)/4,"LEVEL : "+ str(level),curses.A_BOLD)
	screen.win.timeout(400)
	screen.win.getch()
	i=1
	for i in range(1,N/2-1):
		for j in range(1,M-1):
			screen.win.timeout(2)
			screen.win.getch()
			screen.win.addstr(i,j,"*",curses.A_BOLD)
			screen.win.addstr(N-1-i,M-1-j,"*",curses.A_BOLD)
	screen.win.timeout(2000)
	screen.win.getch()
#----------------------------------------------------------------------------------------------------------------------------------
	screen.end()
#--------------------------------------------------------------------------------------------------------------------------------
#if(level==L and bomb==b-1):
#	print "YOU WIN"
#	print "LEVEL :" +str(level-1)
#else:
#	print "GAME OVER"
#	print "LEVEL :" +str(level)
#print "SCORE :" +str(score)
#--------------------------------------------------------------------------------------------------------------------------------


# -*- coding: utf-8 -*-
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.support.wait import WebDriverWait

from my_expected_conditions import wait_for_new_element_with_class_loaded
from pages.abstract_page import AbstractPage

class TenthRequestPage(AbstractPage):
    def __init__(self, driver):
        super(TenthRequestPage, self).__init__(driver)

    def scroll_page(self):
        for i in range(6):
            # count how many paragraphs we have
            init_count = len(self.driver.find_elements_by_css_selector(".jscroll-added"))

            # scroll to bottom
            self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")

            # wait until we have new paragraph
            WebDriverWait(self.driver, 1000).until(
                wait_for_new_element_with_class_loaded((By.CLASS_NAME, "jscroll-added"), init_count))

            # wait until loader in new paragraph is gone
            WebDriverWait(self.driver, 1000).until(
                expected_conditions.invisibility_of_element_located((By.CLASS_NAME, "jscroll-loading")))

        # scroll again to reach page bottom
        self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")

        # and scroll back up
        self.driver.execute_script("window.scrollTo(0, 0);")


'''
0408 Multiple DTW -> Get their index order
'''
__author__ = 'Victoria'
# 2018-03-05

import numpy as np
import pandas as pd
import os
import re
from datetime import timedelta
from math import exp, sqrt
from itertools import combinations, permutations
import json
import matplotlib.pyplot as plt
from src.map_matching.read_data import get_wuhu_cellSheet, get_road_tests
from src.pre_processing.road_network import Graph, get_graph, Vertex
from src.pre_processing.read_data import get_cellSheet
from src.pre_processing.funs import cal_distance, get_datetime, output_geo_js, output_data_js, interpolate
from src.map_matching.evaluation import get_test_obs
from src.map_matching.CTrack import CTrack
from src.work.oneuser_alldata_0307 import trajectories_tojson , save_json, save_js
from src.single_analysis.user_trajectory import deal_same_obs

class Multipath_HMM(CTrack):
    def __init__(self,
                 obses,
                 g: Graph,
                 interpolate_interval=1,
                 smoothing=0,
                 is_direction=False,
                 direction_window=5,
                 penalty=0.9,
                 fill_num=None,
                 interpolate=True):
        '''
        Initialize CTrack
        :param obses: A list of DataFrame [time, lon, lat], time spacing may not be one second.
        :param g: Road network [Graph]
        :param interpolate_interval: Interval seconds of processed observables. Default = 1.
        :param smoothing: Window length of smoothing. If smoothing == 0, do not smooth.
        :param penalty: Direction penalty used in transition score.
        :param interpolate: Whether to interpolate trajectory.
        :var self.obs: The interpolated observables [DataFrame]
        :var self.N: The number of road segments
        :var self.T: The length of observables time. (The row number of self.obs)
        :var self.roads: DataFrame [index, road<Vertex>], index is pos<String: '$lon|$lat'>.
                         Roads has been already shrunken.
        '''
        self.obses = obses
        self.mean_obs = Observations(obses).get_mean_obs(fill_num)
        super().__init__(self.mean_obs, g, interpolate_interval, smoothing,
                         is_direction, direction_window, penalty, interpolate)

class DTW:
    '''
    Dynamic Time Wrapping Algorithm.
    '''
    def __init__(self, ts1, ts2, is_norm=False):
        '''
        :param ts1: Time series 1 with m time periods. A list [[lon, lat]].
        :param ts2: Time series 2 with n time periods. A list.
        :param is_norm: whether normalization before dtw.
        '''
        if is_norm:
            self.ts1 = self.normalization(ts1)
            self.ts2 = self.normalization(ts2)
        else:
            self.ts1 = ts1
            self.ts2 = ts2
        self.ots1, self.ots2 = ts1, ts2
        self.m = len(ts1)
        self.n = len(ts2)
        self.directions = [[0, 1], [1, 1], [1, 0]]

    def normalization(self, ts):
        ts = np.matrix(ts).T
        m = ts.mean(axis=1)
        std = ts.std(axis=1)
        ts[0] = (ts[0] - m[0]) / std[0]
        ts[1] = (ts[1] - m[1]) / std[1]
        return ts.T.tolist()

    def dtw(self):
        mat = np.ones((self.m, self.n)) + np.inf
        mat_ind = {}
        start_nodes = [(0, 0)]
        mat[start_nodes[0]] = 0
        layer_cnt = 0
        end_node = (self.m-1, self.n-1)

        print('m = %d, n = %d' % (self.m, self.n))
        while len(start_nodes) != 0:
            next_nodes = []
            for node in start_nodes:
                for dir in self.directions:
                    succ = (node[0] + dir[0], node[1] + dir[1])
                    if 0 <= succ[0] < self.m and 0 <= succ[1] < self.n:
                        new_dist = mat[node] + cal_distance(self.ts1[succ[0]], self.ts2[succ[1]])
                        if succ in mat_ind.keys():
                            if new_dist < mat[succ]:
                                mat_ind[succ] = [node]
                                mat[succ] = new_dist
                            elif new_dist == mat[succ]:
                                mat_ind[succ].append(node)
                        else:
                            mat_ind[succ] = [node]
                            mat[succ] = new_dist
                        if succ not in next_nodes:
                            next_nodes.append(succ)
            start_nodes = next_nodes
            layer_cnt += 1
            print('Layer %d is finished' % layer_cnt)

        print('Minimum distance = %f' % mat[end_node])

        return mat, mat_ind

    def is_in_bound(self, i, j):
        return 0 <= i < self.m and 0 <= j < self.n

    def dtw_boundary(self):
        '''
        Get a rough upper boundary
        :return:
        '''
        min_dist = 0
        node = [0, 0]
        # for i in range(max(self.m, self.n)-1):
        #     nxt = [node[0] + 1, node[1] + 1]
        #     if nxt[0] >= self.m:
        #         nxt[0] = self.m - 1
        #     if nxt[1] >= self.n:
        #         nxt[1] = self.n - 1
        #     max_dist += cal_distance(self.ts1[nxt[0]], self.ts2[nxt[1]])
        #     node = nxt
        while node != [self.m - 1, self.n - 1]:
            value = np.inf
            for dir in self.directions:
                tmp = [node[0] + dir[0], node[1] + dir[1]]
                if not self.is_in_bound(tmp[0], tmp[1]):
                    continue
                tmp_value = cal_distance(self.ts1[tmp[0]], self.ts2[tmp[1]])
                if tmp_value < value:
                    nxt = tmp
                    value = tmp_value
            min_dist += value
            node = nxt

        return min_dist

    def get_dist_matrix(self, ts1, ts2):
        dist_mat = np.zeros((self.m, self.n))
        for i in range(self.m):
            for j in range(self.n):
                dist_mat[i, j] = cal_distance(ts1[i], ts2[j])
        return dist_mat

    @staticmethod
    def get_derivatives(ts):
        ts.insert(0, ts[0])
        ts.append(ts[-1])
        dts = [[(ts[i+1][0] - ts[i-1][0])/2, (ts[i+1][1] - ts[i-1][1])/2] for i in range(1, len(ts) - 1)]
        return dts

    def get_derivatives_dist_matrix(self, ts1, ts2):
        dts1 = self.get_derivatives(ts1)
        dts2 = self.get_derivatives(ts2)
        return self.get_dist_matrix(dts1, dts2)

    def quick_dtw(self, is_derivatives=False):
        mat = np.ones((self.m, self.n)) + np.inf
        if is_derivatives:
            dist_mat = self.get_derivatives_dist_matrix(self.ts1, self.ts2)
        else:
            dist_mat = self.get_dist_matrix(self.ts1, self.ts2)
        mat_ind = {}
        start_nodes = [(0, 0)]
        end_node = (self.m - 1, self.n - 1)
        mat[start_nodes[0]] = 0
        layer_cnt = 0

        max_dist = self.dtw_boundary()

        print('m = %d, n = %d, max_dist = %f' % (self.m, self.n, max_dist))
        while len(start_nodes) != 0:
            next_nodes = []
            for node in start_nodes:
                if max_dist < mat[node] < np.inf:
                    continue
                for dir in self.directions:
                    succ = (node[0] + dir[0], node[1] + dir[1])
                    if 0 <= succ[0] < self.m and 0 <= succ[1] < self.n:
                        new_dist = mat[node] + dist_mat[succ]
                        if new_dist > max_dist:
                            continue
                        if succ in mat_ind.keys():
                            if new_dist < mat[succ]:
                                mat_ind[succ] = [node]
                                mat[succ] = new_dist
                            elif new_dist == mat[succ]:
                                mat_ind[succ].append(node)
                        else:
                            mat_ind[succ] = [node]
                            mat[succ] = new_dist
                        if succ not in next_nodes:
                            next_nodes.append(succ)
            start_nodes = next_nodes
            layer_cnt += 1
            if len(mat[mat != np.inf]) != 0:
                print('Layer %d is finished. Max = %f' % (layer_cnt, mat[mat != np.inf].max()))

        print('Minimum distance = %f' % mat[end_node])

        return mat, mat_ind

    def shortest_path_dtw(self, is_derivatives=False):
        '''
        Solving DTW with shortest path algorithm
        :param is_derivatives:
        :return:
        '''
        mat = np.ones((self.m, self.n)) + np.inf
        if is_derivatives:
            dist_mat = self.get_derivatives_dist_matrix(self.ts1, self.ts2)
        else:
            dist_mat = self.get_dist_matrix(self.ts1, self.ts2)
        mat_ind = {}
        mat[0, 0] = 0

        for layer in range(1, self.m + self.n):
            for i in range(layer+1):
                node = (i, layer-i)
                if not self.is_in_bound(node[0], node[1]):
                    continue
                dist = dist_mat[node]
                for dir in self.directions:
                    lst_node = (node[0] - dir[0], node[1] - dir[1])
                    if not self.is_in_bound(lst_node[0], lst_node[1]):
                        continue
                    tmp_value = mat[lst_node] + dist
                    if tmp_value < mat[node]:
                        mat[node] = tmp_value
                        mat_ind[node] = [lst_node]
                    elif tmp_value == mat[node]:
                        mat_ind[node].append(lst_node)

        # print('Minimum distance = %f' % mat[self.m-1, self.n-1])

        return mat, mat_ind

    def get_trace(self, mat_ind):
        '''
        Now only get one best trace
        :param mat_ind:
        :return:
        '''
        trace = [(self.m-1, self.n-1)]
        node = trace[0]
        while node != (0, 0):
            last_node = mat_ind[node][0]
            trace.append(last_node)
            node = last_node

        # Print trace
        trace_mat = np.zeros((self.m, self.n))
        for t in trace:
            trace_mat[t[0], t[1]] = 1
        # print(trace_mat)

        return trace, trace_mat

    def get_new_ts(self, mat_ind=None):
        if mat_ind is None:
            _, mat_ind = self.shortest_path_dtw()
        trace, _ = self.get_trace(mat_ind)
        ts1, ts2 = self.ts1, self.ts2
        nts1, nts2 = [ts1[0]], [ts2[0]]
        for t in trace[::-1]:
            nts1.append(ts1[t[0]])
            nts2.append(ts2[t[1]])
        return nts1, nts2

    def get_average_dist(self, mat_ind=None):
        if mat_ind is None:
            _, mat_ind = self.shortest_path_dtw()
        nts1, nts2 = self.get_new_ts(mat_ind)
        dist = 0
        for t in range(len(nts1)):
            dist += cal_distance(nts1[t], nts2[t])
        return dist / len(nts1)

    def get_median_dist(self, mat_ind=None):
        if mat_ind is None:
            _, mat_ind = self.shortest_path_dtw()
        nts1, nts2 = self.get_new_ts(mat_ind)
        dist = []
        for t in range(len(nts1)):
            dist.append(cal_distance(nts1[t], nts2[t]))
        return np.median(dist)

    def get_middle_ts(self, mat_ind=None):
        nts1, nts2 = self.get_new_ts(mat_ind)
        return ((np.array(nts1) + np.array(nts2)) / 2).tolist()

    def plot(self, trace, title='DTW'):
        ts1 = np.matrix(self.ots1).T.tolist()
        ts2 = np.matrix(self.ots2).T.tolist()
        plt.title(title)
        plt.plot(ts1[0], ts1[1], color='blue', linewidth="0.8")
        plt.plot(ts2[0], ts2[1], color='red', linewidth="0.8")
        for line in trace:
            p1 = self.ots1[line[0]]
            p2 = self.ots2[line[1]]
            plt.plot([p1[0], p2[0]], [p1[1], p2[1]], color='black', linewidth="0.4")
        plt.grid(True, linestyle="--", color="black", linewidth="0.4")
        plt.savefig('../../res/work/0401_ddtw/pic/%s.png' % title)
        plt.show()

class MultiDTW:
    def __init__(self, tses):
        '''
        Multiple trajectories dynamic time wrapping
        :param tses: A list of ts. [ts1, ..., tsn]. tsi = [[lon, lat]]
        :var self.k: Number of time series
        :var self.dim: Length of each time series
        :var self..directions: All direction allowed
        '''
        self.tses = tses
        self.k = len(tses)
        self.dim = tuple([len(ts) for ts in tses])
        self.directions = self.get_directions(len(tses))

    @staticmethod
    def dtw_tses_single_time(tses):
        '''
        Do DTW between each two time series, and return a list of new time series.
        :param tses:
        :return:
        '''
        inds = list(combinations(np.arange(len(tses)), 2))
        ntses = []
        for ind in inds:
            ts1 = tses[ind[0]]
            ts2 = tses[ind[1]]
            dtw = DTW(ts1, ts2)
            ntses.append(dtw.get_middle_ts())

    @staticmethod
    def get_directions(k):
        '''
        Get valid directions with dimension k.
        :param k:
        :return: A list.
        e.x: k = 2
        return [[1, 0], [0, 1], [1, 1]]
        k = 3
        return [[1, 0, 0], [0, 1, 0], [1, 1, 1]
        '''
        res = list(set(combinations([0, 1]*k, k)))
        res.remove(tuple([0]*k))
        return res

    def is_in_bound(self, pos):
        for k in range(len(self.dim)):
            if not 0 <= pos[k] < self.dim[k]:
                return False
        return True

    def cal_multi_distance(self, ind):
        '''
        ind is a tuple
        :param ind:
        e.x:
            ind = (1, 2, 4)
            Calculate distance between ts0_1, ts1_2, ts2_4.
        :return:
        '''
        ind = list(ind)
        combs = list(combinations(range(len(ind)), 2))
        dist = 0
        for comb in combs:
            ts1_ind = comb[0]
            ts2_ind = comb[1]
            ts1_t = ind[ts1_ind]
            ts2_t = ind[ts2_ind]
            dist += cal_distance(self.tses[ts1_ind][ts1_t], self.tses[ts2_ind][ts2_t])
        return dist / len(combs)

    def dtw_tses(self):
        '''
        Multiple DTW
        :return:
        '''
        mat = np.ones(self.dim) + np.inf
        mat_ind = {}
        mat[tuple([0]*self.k)] = 0

        for layer in range(1, sum(self.dim)):
            inds = self.get_ksum(layer)
            for node in inds:
                if not self.is_in_bound(node):
                    continue
                dist = self.cal_multi_distance(node)
                for dir in self.directions:
                    lst_node = tuple((np.array(node) - np.array(dir)).tolist())
                    if not self.is_in_bound(lst_node):
                        continue
                    tmp_value = mat[lst_node] + dist
                    if tmp_value < mat[node]:
                        mat[node] = tmp_value
                        mat_ind[node] = [lst_node]
                    elif tmp_value == mat[node]:
                        mat_ind[node].append(lst_node)
            if len(mat[mat != np.inf]) != 0:
                print('Layer %d is finished. Max = %f' % (layer, mat[mat != np.inf].max()))

        print('Minimum distance = %f' % mat[tuple([-1]*self.k)])

        return mat, mat_ind

    def get_trace(self, mat_ind=None):
        '''
        Now only get one best trace
        :param mat_ind:
        :return:
        '''
        if mat_ind is None:
            _, mat_ind = self.dtw_tses()
        trace = [tuple([k-1 for k in self.dim])]
        node = trace[0]
        while node != tuple([0]*self.k):
            last_node = mat_ind[node][0]
            trace.append(last_node)
            node = last_node

        return trace

    def plot(self, title='DTW of multiple TS', trace=None):
        if trace is None:
            trace = self.get_trace()
        plt.title(title)
        colors = ['red', 'blue', 'cyan', 'orange']
        lin_colors = ['black', 'brown', 'purple', 'gray']
        for i, ts in enumerate(self.tses):
            ts = np.matrix(ts).T.tolist()
            plt.plot(ts[0], ts[1], color=colors[i % len(colors)], linewidth="2")

        for i, line in enumerate(trace):
            combs = list(combinations(range(len(line)), 2))
            for comb in combs:
                ts1_ind = comb[0]
                ts2_ind = comb[1]
                ts1_t = line[ts1_ind]
                ts2_t = line[ts2_ind]
                p1 = self.tses[ts1_ind][ts1_t]
                p2 = self.tses[ts2_ind][ts2_t]
                plt.plot([p1[0], p2[0]], [p1[1], p2[1]], color=lin_colors[i % len(lin_colors)], linewidth="0.5")
        plt.grid(True, linestyle="--", color="black", linewidth="0.4")
        plt.savefig('../../res/work/0401_ddtw/pic/%s.png' % title)
        plt.show()

    def get_ksum(self, n):
        res = get_ksum(n, self.k)
        return [tuple(x) for x in res]

class MultiDTWIndx:
    def __init__(self, obses, interval=1):
        '''
        Get all the obses' index order
        :param obses: Input a list of DataFrame -> [pd.DataFrame(['dates', 'lon', 'lat'])]
                      Must ordered by dates.
        :param interval: Interpolate interval. Default 1.
        '''
        self.obses = self.set_origin_destination(obses)
        self.tses, self.new_obses = self.interpolate(interval)
        self.interval = interval

    def set_origin_destination(self, obses):
        '''
        Set all the observations have the same origin point and destination point
        :param obses: [DataFrame['lon', 'lat', 'dates']]
        :return: A new processed obses
        '''
        origins = [[obs.lon[0], obs.lat[0]] for obs in obses]
        destinations = [[list(obs.lon)[-1], list(obs.lat)[-1]] for obs in obses]
        origin = [np.mean([x[0] for x in origins]), np.mean([x[1] for x in origins])]
        destination = [np.mean([x[0] for x in destinations]), np.mean([x[1] for x in destinations])]
        for i in range(len(obses)):
            obses[i].lon[0] = origin[0]
            obses[i].lat[0] = origin[1]
            obses[i].lon[len(obses[i])-1] = destination[0]
            obses[i].lat[len(obses[i]) - 1] = destination[1]
        return obses

    def interpolate(self, interval):
        '''
        Interpolate all the observations with the given interval.
        :param interval:
        :return:
        '''
        tses = []
        new_obses = []
        for obs in self.obses:
            ts, tmp = self.interpolate_single(obs, interval)
            new_obses.append(tmp)
            tses.append(ts)
        return tses, new_obses

    @staticmethod
    def interpolate_single(obs, interval):
        '''
        Interpolate one observation with the given interval.
        :param obs: DataFrame
        :param interval:
        :return: ts: [[lon, lat]]; new_obs: DataFrame
        '''
        new_obs = interpolate(obs, interval, origin_reserve=True)
        ts = [[new_obs.lon[i], new_obs.lat[i]] for i in range(len(new_obs))]
        return ts, new_obs

    def run(self):
        ts0 = self.tses[0]
        new_obs = self.new_obses[0]
        # new_obs.lon = [round(x, 7) for x in new_obs.lon]
        # new_obs.lat = [round(x, 7) for x in new_obs.lat]
        new_obs['traj'] = -1
        for i, ts in enumerate(self.tses):
            obs = self.obses[i]
            dtw = DTW(ts0, ts, is_norm=False)
            nts0, nts = dtw.get_new_ts()
            # nts = [[round(x[0], 7), round(x[1], 7)] for x in nts]
            # nts0 = [[round(x[0], 7), round(x[1], 7)] for x in nts0]
            # obs.lon = [round(x, 7) for x in obs.lon]
            # obs.lat = [round(x, 7) for x in obs.lat]
            for t in range(len(obs)):
                # if [obs.lon[t], obs.lat[t]] in nts:
                ind = nts.index([obs.lon[t], obs.lat[t]])
                ind2 = (new_obs.lon == nts0[ind][0]) & (new_obs.lat == nts0[ind][1])
                new_obs.lon[ind2] = obs.lon[t]
                new_obs.lat[ind2] = obs.lat[t]
                new_obs.traj[ind2] = i
            print(new_obs[round(new_obs.lon, 6) == 117.229595])
            print(len(new_obs))
        new_obs = new_obs[new_obs.traj != -1]
        return new_obs

    def run_one_by_one(self):
        '''
        Get the fused observations.
        Method:
            1. Take obses[0] and obses[1], interpolate them and get the ts format.
            2. Fuse them into a new observation(only have observation points, no interpolate points).
            3. Take the new observation and obses[k+1], repeat Step 1 and Step 2.
        :return: A fused observation: DataFrame['lon', 'lat', 'dates']
        '''
        new_obs = self.obses[0]

        for i in range(1, len(self.obses)):
            obs = self.obses[i]
            ts0, obs0 = self.interpolate_single(new_obs, self.interval)
            ts, _ = self.interpolate_single(obs, self.interval)
            dtw = DTW(ts0, ts, is_norm=False)
            nts0, nts = dtw.get_new_ts()

            obs0['traj'] = 0
            for t in range(len(obs)):
                ind = nts.index([obs.lon[t], obs.lat[t]])
                ind2 = (obs0.lon == nts0[ind][0]) & (obs0.lat == nts0[ind][1])
                obs0.lon[ind2] = obs.lon[t]
                obs0.lat[ind2] = obs.lat[t]
                obs0.traj[ind2] = 1
            obs0 = obs0[obs0.traj == 1]
            obs0 = obs0.drop(['traj'], axis=1)
            new_obs = pd.concat([new_obs, obs0])
            # in deal_same_obs already has sort_values() and reset_index() method
            new_obs = deal_same_obs(new_obs)

        return new_obs[['lon', 'lat', 'dates']]

    def plot(self, new_obs=None):
        if new_obs is None:
            new_obs = self.run_one_by_one()
        print(new_obs)
        plt.title('DTW Index Sort')
        for obs in self.obses:
            plt.plot(obs.lon, obs.lat, linewidth="3")

        plt.plot(new_obs.lon, new_obs.lat, color='red', linewidth="1")

        plt.show()

def get_ksum(n, k, tmp_number=[], res=[]):
    '''
    K natural numbers sums up is n. Already given res (a list of natural numbers)
    :param n: The sum of k numbers.
    :param k: Number of natural numbers.
    :param tmp_number: Now the given numbers.
    :param res: The Final result. Always input []
    :return:
    '''
    if len(tmp_number) == k - 1:
        tmp_number.append(n - sum(tmp_number))
        return [tmp_number]
    for i in range(n-sum(tmp_number)+1):
        ntmp_number = tmp_number.copy()
        ntmp_number.append(i)
        a = get_ksum(n, k, ntmp_number, res)
        if len(tmp_number) == k - 2:
            res.extend(a)
        else:
            res = a

    return res

class Observations:
    def __init__(self, obses):
        '''
        List of observations.
        :param obses: A list of DataFrame [dates, lon, lat].
        :var self.obses: A list of Observation.
        '''
        self.obses = []
        for i in range(len(obses)):
            self.obses.append(Observation(obses[i]))
        self.mean_obs = self.get_mean_obs()

    def plot_obses(self):
        for obs in self.obses:
            plt.plot(obs.lon, obs.lat)

    def len_obses(self):
        l = []
        for obs in self.obses:
            l.append(len(obs))
        return l

    def fill_with_dist(self, num):
        '''
        Let all observations filled with distanc
        :param num:
        :return:
        '''
        obses = self.obses
        res = []
        for obs in obses:
            res.append(obs.fill_with_dist(num))
        return res

    def plot_obs(self, fill_num=None):
        data = self.mean_obs
        fig = plt.figure()
        # plt.show()
        ax = fig.add_subplot(1, 1, 1)
        plt.grid(True)  # 添加网格
        plt.ion()  # interactive mode on
        for i in range(len(self.obses)):
            obs = self.obses[i].obs
            ax.plot(obs.lon, obs.lat)
            plt.pause(0.2)
        for i in range(0, len(data), 5):
            ax.plot(data.lon[:i], data.lat[:i], color='red')
            plt.pause(0.01)
        plt.pause(5)

    def get_mean_obs(self, fill_num=None):
        if fill_num is None:
            total_dist = []
            for obs in self.obses:
                total_dist.append(obs.get_total_dist())
            fill_num = np.mean(total_dist) // 10
            print('Fill num = %d' % fill_num)

        obses = self.fill_with_dist(fill_num)
        lon = []
        lat = []

        for t in range(len(obses[0])):
            lon.append(np.mean([obs.lon[t] for obs in obses]))
            lat.append(np.mean([obs.lat[t] for obs in obses]))

        return pd.DataFrame({'lon': lon, 'lat': lat, 'dates': obses[0].dates})

    def save_json(self, save_path, user_id='', is_json=True, fill_num=None, need_mean=True):
        obses = []

        for i in range(len(self.obses)):
            tmp = self.obses[i].obs
            obses.append(tmp)

        if need_mean:
            obses.append(self.get_mean_obs(fill_num))

        trajectories_tojson(obses, user_id, save_path, is_json=is_json)

class Observation:
    def __init__(self, obs):
        '''
        :param obs: A DataFrame observation [dates, lon, lat]
        '''
        self.obs = obs

    def get_obs_len(self):
        return len(self.obs)

    def get_total_dist(self):
        dist = 0
        for i in range(1, len(self.obs)):
            dist += cal_distance([self.obs.lon[i-1], self.obs.lat[i-1]],
                                 [self.obs.lon[i], self.obs.lat[i]])
        return dist

    def fill_with_dist(self, num):
        '''
        Fill the observation with distance.
        :param num: The num of final observation.
        :return: A new observation.
        '''
        total_dist = self.get_total_dist()
        each_dist = total_dist / (num - 1)

        dist = 0

        obs = self.obs
        lon = [obs.lon[0]]
        lat = [obs.lat[0]]
        dates = [obs.dates[0]]
        for i in range(1, len(obs)):
            p1 = [obs.lon[i-1], obs.lat[i-1]]
            p2 = [obs.lon[i], obs.lat[i]]
            d_have = cal_distance(p1, p2)
            d_need = each_dist - dist

            while d_have >= d_need:
                p_need = d_need / d_have
                p1 = [(1 - p_need) * p1[0] + p_need * p2[0], (1 - p_need) * p1[1] + p_need * p2[1]]
                lon.append(p1[0])
                lat.append(p1[1])
                dates.append(obs.dates[i-1] + (obs.dates[i] - obs.dates[i-1]) * p_need)

                dist = 0
                d_have = cal_distance(p1, p2)
                d_need = each_dist - dist

            dist += d_have

        return pd.DataFrame({'lon': lon, 'lat': lat, 'dates': dates})

    def plot_obs(self, fill_num=None, one_by_one=False):
        if fill_num is None:
            data = self.obs
        else:
            data = self.fill_with_dist(fill_num)
        fig = plt.figure()
        # plt.show()
        ax = fig.add_subplot(1, 1, 1)
        plt.grid(True)  # 添加网格
        plt.ion()  # interactive mode on
        if one_by_one:
            for i in range(len(data)):
                ax.plot(data.lon[:i], data.lat[:i])
                plt.pause(0.2)
        else:
            ax.plot(data.lon, data.lat)
        plt.pause(2)

    def cal_time_interval(self):
        total_time =  (max(self.obs.dates) - min(self.obs.dates)).total_seconds()
        return total_time / (len(self.obs) - 1)

def multi_map_matching(obses, graph, save_name,
                       mdtw_interval=5,
                       mm_smoothing=10,
                       mm_interpolate_interval=10,
                       save_dir='../../res/work/0417_multi_mm/'):
    mdtw = MultiDTWIndx(obses, interval=mdtw_interval)
    new_obs = mdtw.run_one_by_one()
    print(new_obs)
    output_data_js(new_obs,  save_dir+save_name+'_raw.js')
    track = CTrack(new_obs, graph, smoothing=mm_smoothing, interpolate_interval=mm_interpolate_interval)
    output_data_js(track.obs, save_dir+save_name+'_processed.js')
    states = track.viterbi()
    output_data_js(states, save_dir+save_name+'.js')
    return states

def single_map_matching(obs, graph, save_name,
                        mm_smoothing=10,
                        mm_interpolate_interval=10,
                        save_dir='../../res/work/0417_multi_mm/'):
    output_data_js(obs,  save_dir+save_name+'_raw.js')
    track = CTrack(obs, graph, smoothing=mm_smoothing, interpolate_interval=mm_interpolate_interval)
    output_data_js(track.obs, save_dir+save_name+'_processed.js')
    states = track.viterbi()
    output_data_js(states, save_dir+save_name+'.js')
    return states

def dist_trajectories(ts1, ts2):
    assert len(ts2) == len(ts1)
    dist = 0
    for i in range(len(ts1)):
        dist += cal_distance(ts1[i], ts2[i])
    return dist / len(ts1)

def test_observation(trail, direction, n):
    '''
    Test the class observation.
    :param trail: The number of trail.
    :param direction: [0/1]
                      If direction == 1: positive direction
                      else: negative direction
    :param n: The number of test.
    :return:
    '''
    obs = Observation(get_test_obs(trail, direction, n))
    print(obs.obs)
    print('-'*10)
    print(obs.fill_with_dist(5))
    print(obs.get_total_dist())
    obs.plot_obs(3)

def test_observations(trail, direction, fill_num):
    '''
    Test observation list.
    :param trail:
    :param direction:
    :param fill_num:
    :return:
    '''
    obses = []
    for n in range(1, 11):
        obses.append(get_test_obs(trail, direction, n))
    obses = Observations(obses)
    # obses.plot_obs(fill_num)
    new_obses = obses.get_mean_obs(fill_num)
    Observation(new_obses).plot_obs()
    print(new_obses)

def test_dtw(user_number, path_need):
    with open('../../res/work/0311_random_trajectories/random_user%d.json' % user_number) as data_file:
        data = json.load(data_file)

    obses = []
    for i in path_need[:2]:
        tmp = data['trajectories'][i]
        dates = [get_datetime(t) for t in tmp['dates']]
        obs = pd.DataFrame({'lon': tmp['lon'], 'lat': tmp['lat'], 'dates': dates})
        obs = interpolate(obs, interval=1)
        obs = [[obs.lon[i], obs.lat[i]] for i in range(len(obs))]
        obses.append(obs)

    dtw = DTW(obses[0], obses[1], is_norm=False)
    mat, mat_ind = dtw.shortest_path_dtw(is_derivatives=True)
    # mat, mat_ind = dtw.quick_dtw(is_derivatives=False)
    # mat, mat_ind = dtw.dtw()
    trace, trace_mat = dtw.get_trace(mat_ind)
    dtw.plot(trace, 'random_user%d_%d_%d DDTW Plot' % (user_number, path_need[0], path_need[1]))
    # ts1, ts2 = dtw.get_new_ts()
    # print(dist_trajectories(ts1, ts2))
    # return trace

def test_muti_dtw(user_number, path_need):
    with open('../../res/work/0311_random_trajectories/random_user%d.json' % user_number) as data_file:
        data = json.load(data_file)

    obses = []
    for i in path_need:
        tmp = data['trajectories'][i]
        dates = [get_datetime(t) for t in tmp['dates']]
        obs = pd.DataFrame({'lon': tmp['lon'], 'lat': tmp['lat'], 'dates': dates})
        obs = interpolate(obs, interval=60)
        obs = [[obs.lon[i], obs.lat[i]] for i in range(len(obs))]
        obses.append(obs)

    dtw = MultiDTW(obses)
    # mat, mat_ind = dtw.dtw_tses()
    dtw.plot()
    # mat, mat_ind = dtw.quick_dtw(is_derivatives=False)
    # mat, mat_ind = dtw.dtw()
    # trace, trace_mat = dtw.get_trace(mat_ind)
    # dtw.plot(trace, 'random_user%d_%d_%d DDTW Plot' % (user_number, path_need[0], path_need[1]))
    # ts1, ts2 = dtw.get_new_ts()
    # print(dist_trajectories(ts1, ts2))
    # return trace

def test_hefei(user_number, path_need):
    with open('../../res/work/0311_random_trajectories/random_user%d.json' % user_number) as data_file:
        data = json.load(data_file)

    obses = []
    for i in path_need:
        tmp = data['trajectories'][i]
        dates = [get_datetime(t) for t in tmp['dates']]
        obs = pd.DataFrame({'lon': tmp['lon'], 'lat': tmp['lat'], 'dates': dates})
        print(Observation(obs).cal_time_interval())
        obses.append(obs)

    obses = Observations(obses)
    # obses.plot_obs()
    obses.save_json('../../res/work/0315_similar_trajectories/random_user%d.json' % user_number,
                    data['user_id'], is_json=True)
    obses.save_json('../../res/work/0315_similar_trajectories/random_user%d.js' % user_number,
                    data['user_id'], is_json=False, fill_num=30)

    return obses

def obs_to_ts(obs, interval=1):
    '''
    Transform observation to ts
    :param obs: DataFrame['lon', 'lat', 'dates']
    :param interval: The interval to interpolate the observation
    :var ts: [[lon, lat]]
    :return: ts
    '''
    obs = interpolate(obs, interval=interval)
    ts = []
    for t in range(len(obs)):
        ts.append([obs.lon[t], obs.lat[t]])

    return ts

def test_multi_mm(user_number, path_need, smoothing=2, interpolate_interval=15, interpolate=True):
    '''
    Test Multi_mm class.
    :param user_number: Random User number use
    :param path_need: Path number used
    :param smoothing:
    :param interpolate_interval:
    :param interpolate:
    :return:
    '''

    # Get observations
    with open('../../res/work/0311_random_trajectories/random_user%d.json' % user_number) as data_file:
        data = json.load(data_file)

    obses = []
    for i in path_need:
        tmp = data['trajectories'][i]
        dates = [get_datetime(t) for t in tmp['dates']]
        obs = pd.DataFrame({'lon': tmp['lon'], 'lat': tmp['lat'], 'dates': dates})
        obses.append(obs)

    # Get graph
    graph = get_graph('../../data/hefei_road/link_baidu.txt')

    matched_obses = []
    mean_obs = Observations(obses).get_mean_obs(30)
    obses.append(mean_obs)
    for obs in obses:
        track = CTrack(obs, g=graph, smoothing=smoothing,
                       interpolate_interval=interpolate_interval,
                       interpolate=interpolate)
        matched_obs = track.viterbi()
        matched_obses.append(matched_obs)

    trajectories_tojson(matched_obses, data['user_id'],
                        '../../res/work/0315_similar_trajectories/multiHMM_random_user%d.js' % user_number,
                        is_json=False)
    trajectories_tojson(matched_obses, data['user_id'],
                        '../../res/work/0315_similar_trajectories/multiHMM_random_user%d.json' % user_number,
                        is_json=True)

def test_multi_index(user_number, path_need):
    with open('../../res/work/0311_random_trajectories/random_user%d.json' % user_number) as data_file:
        data = json.load(data_file)

    obses = []
    for i in path_need:
        tmp = data['trajectories'][i]
        dates = [get_datetime(t) for t in tmp['dates']]
        obs = pd.DataFrame({'lon': tmp['lon'], 'lat': tmp['lat'], 'dates': dates})
        obses.append(obs)

    mdtw = MultiDTWIndx(obses, interval=5)
    new_obs = mdtw.run_one_by_one()
    mdtw.plot(new_obs)
    obses.append(new_obs)
    trajectories_tojson(obses, data['user_id'],
                        '../../res/work/0411_dtw_index_sort/multiHMM_random_user%d.js' % user_number,
                        is_json=False)

# dijkstra算法实现，有向图和路由的源点作为函数的输入，最短路径最为输出
def dijkstra(graph, src=0):
    # 判断图是否为空，如果为空直接退出
    if graph is None:
        return None
    nodes = [i for i in range(len(graph))]  # 获取图中所有节点
    visited=[]  # 表示已经路由到最短路径的节点集合
    if src in nodes:
        visited.append(src)
        nodes.remove(src)
    else:
        return None
    distance={src:0}  # 记录源节点到各个节点的距离
    for i in nodes:
        distance[i]=graph[src][i]  # 初始化
    # print(distance)
    path={src:{src:[]}}  # 记录源节点到每个节点的路径
    k=pre=src
    while nodes:
        mid_distance=float('inf')
        for v in visited:
            for d in nodes:
                new_distance = graph[src][v]+graph[v][d]
                if new_distance < mid_distance:
                    mid_distance=new_distance
                    graph[src][d]=new_distance  # 进行距离更新
                    k=d
                    pre=v
        distance[k]=mid_distance  # 最短路径
        path[src][k]=[i for i in path[src][pre]]
        path[src][k].append(k)
        # 更新两个节点集合
        visited.append(k)
        nodes.remove(k)
        print(visited,nodes)  # 输出节点的添加过程
    return distance,path

def cal_avg_time_interval():
    users = []
    num = []
    intervals = []
    for user_number in range(8, 63):
        with open('../../res/work/0311_random_trajectories/random_user%d.json' % user_number) as data_file:
            data = json.load(data_file)
        for i in range(len(data['trajectories'])):
            tmp = data['trajectories'][i]
            dates = [get_datetime(t) for t in tmp['dates']]
            obs = pd.DataFrame({'lon': tmp['lon'], 'lat': tmp['lat'], 'dates': dates})

            users.append(user_number)
            num.append(i)
            intervals.append(Observation(obs).cal_time_interval())
        out = pd.DataFrame({'user': users, 'num': num, 'time_interval': intervals})
        out.to_csv('../../res/work/0506_big_eye/time_interval/time_interval.csv')

    import seaborn as sns
    sns.set_palette('deep', desat=.6)
    sns.set_context(rc={'figure.figsize': (8, 5)})
    plt.hist(intervals, bins=20)
    plt.grid(True, linestyle="--", color="black", linewidth=".5", alpha=.4)
    plt.title('Cellular Data Time Interval Histogram')
    plt.xlabel('Time interval(s)')
    plt.ylabel('Frequency')
    plt.show()

if __name__ == '__main__':
    # test_observation(1,1,1)
    # test_observations(1, 1, 10)
    # test_hefei(8, [15, 25, 26])
    # test_multi_mm(8, [15, 25, 26])
    # test_dtw(8, [15, 25])
    # print(len(get_ksum(10, 3)))
    # test_muti_dtw(8, [15, 25, 26])
    # test_multi_index(8, [15, 25, 26])
    # test_multi_index(20, [15, 38, 2, 7, 19])
    cal_avg_time_interval()

    # graph_list = [[0, 2, 1, 4, 5, 1],
    #               [1, 0, 4, 2, 3, 4],
    #               [2, 1, 0, 1, 2, 4],
    #               [3, 5, 2, 0, 3, 3],
    #               [2, 4, 3, 4, 0, 1],
    #               [3, 4, 7, 3, 1, 0]]
    #
    # distance, path = dijkstra(graph_list, 0)  # 查找从源点0开始带其他节点的最短路径
    # print(distance, path)


import datetime, os, pandas as pd, numpy as np, requests, time, urllib3
from bs4 import BeautifulSoup

class Metadata(object):
    '''
    '''

    identifiers = None
    metadata_filepath = None

    def request_bulk_metadata(self, date_of_last_request):
        '''
        Requests bulk metadata from OAI2.
        Returns (rows, requestDate) where
        '''

        rows = []
        resumptionToken = 'placeholder'
        url = 'http://export.arxiv.org/oai2?verb=ListRecords&set=physics:astro-ph&metadataPrefix=arXiv'
        results = None

        # If we have specified the date of the last request, add it to the URL
        if date_of_last_request:
            url += '&from=' + date_of_last_request.strftime('%Y-%m-%d')

        # Continue requesting until we are not given any more resumption tokens
        while resumptionToken is not None:
            # Send request and receive results, waiting specified time if necessary
            while results == None:
                try:
                    print('Requesting: ' + url)
                    results = requests.get(url).text
                except urllib3.exceptions.HTTPError as e:
                    wait = int(e.headers.get('Retry-After'))
                    print('HTTPError: Waiting ' + str(wait) + 's to retry requesting metadata...')
                    time.sleep(wait)

            # Parse with Beautiful Soup
            soup = BeautifulSoup(results, 'xml')
            records = soup.find_all('record')
            for record in records:
                # Get header data
                identifier = record.find('identifier')
                datestamp = record.find('datestamp')
                spec = record.find('setSpec')

                # Get metadata
                filename = record.find('id')
                created = record.find('created')
                updated = record.find('updated')
                authors = []
                for author in record.find_all('author'):
                    forenames = author.forenames
                    keyname = author.keyname
                    if forenames and keyname:
                        authors.append(author.forenames.text.strip() + ' ' + author.keyname.text.strip())
                author_str = ', '.join(authors)
                title = record.find('title')
                categories = record.find('categories')
                journal = record.find('journal-ref')
                doi = record.find('doi')
                abstract = record.find('abstract')
                comments = record.find('comment')

                # Save current record as a row in the table
                row = {
                    'identifier': getattr(identifier, 'text', None),
                    'filename': getattr(filename, 'text', None),
                    'spec': getattr(spec, 'text', None),
                    'title': getattr(title, 'text', None),
                    'datestamp': getattr(datestamp, 'text', None),
                    'created': getattr(created, 'text', None),
                    'updated': getattr(updated, 'text', None), # may have more than one instance that we're missing
                    'authors': author_str,
                    'categories': getattr(categories, 'text', None),
                    'journal': getattr(journal, 'text', None),
                    'doi': getattr(doi, 'text', None),
                    'abstract': getattr(abstract, 'text', None),
                    'comments': getattr(comments, 'text', None)
                }
                rows.append(row)

            # Get resumption token if provided
            resumptionToken = soup.find('resumptionToken')

            # Continue if we have resumption token
            if resumptionToken is not None:
                print('Status: ' + str(int(resumptionToken['cursor']) + 1) + '—' + str(len(rows)) + '/' + str(resumptionToken['completeListSize']) + '...')
                resumptionToken = resumptionToken.text
                url = 'http://export.arxiv.org/oai2?verb=ListRecords&resumptionToken=' + resumptionToken
                time.sleep(20) # avoid 503 status

        return rows

    def update(self):
        '''
        Checks if an update is needed. If it is needed, gathers
        '''
        metadata_filepath = 'arxiv_metadata_astroph.csv'

        if os.path.exists(metadata_filepath):
            # If the metadata file exists, load it into a data frame
            existing_metadata_df = pd.read_csv(metadata_filepath,
                                               dtype={'filename': str,
                                                      'filename_parsed': str,
                                                      'identifier': str,
                                                      'updated': str,
                                                      'doi': str},
                                               parse_dates=['date_retrieved'])
            # Get the date of the last request
            existing_metadata_df['date_retrieved'] = existing_metadata_df['date_retrieved'].apply(lambda x:x.replace(tzinfo=datetime.timezone.utc))
            date_of_last_request = existing_metadata_df['date_retrieved'].max()
            print(metadata_filepath + ' last updated on ' + date_of_last_request.strftime('%Y-%m-%d'))
            print('Updating...')
            # Send a request to access metadata since that date
            records = self.request_bulk_metadata(date_of_last_request + datetime.timedelta(days=1))
            # Create data frame for records to specify additional info
            if len(records) > 0:
                print('Number of new records found: ' + str(len(records)))
                records_df = pd.DataFrame(records)
                records_df['date_retrieved'] = np.full(len(records_df), datetime.datetime.now())
                records_df['filename_parsed'] = existing_metadata_df['filename'].str.replace('/', '')
                # Update metadata file
                metadata_df = pd.concat([existing_metadata_df, records_df], axis=0, sort=True, ignore_index=True)
                metadata_df.to_csv(metadata_filepath, index=False)
                print('Metadata has been updated.')
            else:
                print('No additional records found. Metadata is up to date.')
        else:
            # If the metadata file doesn't exist, request all metadata
            print(metadata_filepath + ' is being created...')
            records = request_bulk_metadata(None)
            # Load records into data frame
            metadata_df = pd.DataFrame(records)
            # Add a column to specify additional info
            metadata_df['date_retrieved'] = np.full(len(metadata_df), datetime.datetime.now())
            metadata_df['filename_parsed'] = metadata_df['filename'].str.replace('/', '')
            # Save it to CSV
            metadata_df.to_csv(metadata_filepath, index=False)
            print('Metadata has been saved.')

    def get_identifiers(self):
        # Grab identifiers from metadata
        metadata_df = pd.read_csv(self.metadata_filepath,
                                   dtype={'filename': str,
                                          'filename_parsed': str,
                                          'identifier': str,
                                          'updated': str,
                                          'doi': str},
                                   parse_dates=['date_retrieved'])
        self.identifiers = metadata_df['filename_parsed']

    def __init__(self, update=False):
        self.metadata_filepath = 'arxiv_metadata_astroph.csv'
        # Automatically check for any updates
        if update:
            self.update()
        self.get_identifiers()


import unittest
from database import create_connection
from CreditCards import CreditCard
from datetime import datetime
from payment_plan import PaymentPlan

class TestDatabase(unittest.TestCase):

    def setUp(self):
        self.connection = create_connection()
        self.cursor = self.connection.cursor()

        # Eliminamos todas las tarjetas de la tabla
        self.cursor.execute("DELETE FROM credit_card")
        self.connection.commit()

    def tearDown(self):
        self.connection.close()

    def test_insert_card(self, card=None):
        # Create a CreditCard object if not provided
        if card is None:
            card = CreditCard(
                card_number="556677",
                owner_id="1020889955",
                owner_name="Estudiante pelao",
                bank_name="Bancolombia",
                due_date="31/12/2027",
                franchise="VISA",
                payment_day=10,
                monthly_fee=24000,
                interest_rate=3.1
            )

        cursor = self.connection.cursor()

        # Check if the card already exists
        cursor.execute("SELECT COUNT(*) FROM credit_card WHERE card_number = %s", (card.card_number,))
        card_count = cursor.fetchone()[0]

        if card_count > 0:
            raise Exception("Ya existe una tarjeta con el mismo número")

        # Convertir la fecha de vencimiento a un objeto de tipo `date`
        due_date = datetime.strptime(card.due_date, "%d/%m/%Y").date()

        # Guardar la tarjeta en la base de datos
        cursor.execute(
            "INSERT INTO credit_card (card_number, owner_id, owner_name, bank_name, due_date, franchise, payment_day, monthly_fee, interest_rate) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)",
            (
                card.card_number,
                card.owner_id,
                card.owner_name,
                card.bank_name,
                due_date,
                card.franchise,
                card.payment_day,
                card.monthly_fee,
                card.interest_rate,
            ),
        )
        self.connection.commit()

        # Verificar que la tarjeta se haya creado
        cursor.execute("SELECT COUNT(*) FROM credit_card WHERE card_number = %s", (card.card_number,))
        card_count = cursor.fetchone()[0]

        if card_count == 0:
            raise Exception("La tarjeta no fue creada")

        result = "Tarjeta guardada exitosamente"

        # Comparar el resultado con el mensaje esperado
        self.assertEqual(result, "Tarjeta guardada exitosamente")
        return result

    def is_credit_card_expired(self, card):
        # Obtener la fecha actual
        current_date = datetime.now().date()

        # Convertir la fecha de vencimiento de la tarjeta a datetime.date
        due_date = datetime.strptime(card.due_date, "%d/%m/%Y").date()

        # Comparar la fecha actual con la fecha de vencimiento de la tarjeta
        return current_date > due_date

    def test_expired_credit_card(self):
        # Crear una tarjeta de crédito vencida
        card = CreditCard(
            card_number="442233",
            owner_id="1010123456",
            owner_name="Comprador compulsivo",
            bank_name="Popular",
            due_date="31/12/2022",  # Fecha pasada, tarjeta vencida
            franchise="Mastercard",
            payment_day=5,
            monthly_fee=34000,
            interest_rate=3.4,
        )

        # Verificar si la tarjeta está vencida
        is_expired = self.is_credit_card_expired(card)

        # Comprobar si se obtiene el mensaje correcto y es True
        self.assertTrue(is_expired, "NO SE PUEDE AGREGAR PORQUE ESTÁ VENCIDA")

    def test_existing_credit_card_2(self):
        # Crear una tarjeta de crédito con los mismos datos de entrada
        card = CreditCard(
            card_number="556677",
            owner_id="1020889955",
            owner_name="Estudiante pelao",
            bank_name="Bancolombia",
            due_date="31/12/2027",
            franchise="VISA",
            payment_day=10,
            monthly_fee=24000,
            interest_rate=3.1,
        )

        # Llamar al método de prueba `test_insert_card` directamente
        self.test_insert_card()

        # Intentar insertar la misma tarjeta una segunda vez
        # Esto debería fallar y lanzar la excepción que estás esperando
        with self.assertRaises(Exception) as e:
            self.test_insert_card()

        # Comprobar si se obtiene el mensaje correcto
        self.assertEqual(str(e.exception), "Ya existe una tarjeta con el mismo número")

    def test_T4_insert_credit_card(self):
        # Crear una tarjeta de crédito con los datos del T4
        card = CreditCard(
            card_number="223344",
            owner_id="1010123456",
            owner_name="Comprador compulsivo",
            bank_name="Falabella",
            due_date="31/12/2025",
            franchise="VISA",
            payment_day=16,
            monthly_fee=0,
            interest_rate=3.4,
        )

        # Insertar la tarjeta por primera vez (esto debería ser exitoso)
        mensaje = self.test_insert_card()

        # Verificar si el mensaje es igual a "Tarjeta guardada exitosamente"
        self.assertEqual(mensaje, "Tarjeta guardada exitosamente")

    def test_T5_insert_credit_card(self):
        # Crear una tarjeta con los valores proporcionados
        card = CreditCard(
            card_number="445566",
            owner_id="1010123456",
            owner_name="Comprador compulsivo",
            bank_name="BBVA",
            due_date="31/12/2027",
            franchise="Mastercard",
            payment_day=5,
            monthly_fee=34000,
            interest_rate=0,
        )

        # Llamar al método de prueba `test_insert_card` con la tarjeta creada
        result = self.test_insert_card(card)

        # Verificar si la tarjeta se guardó exitosamente
        self.assertEqual(result, "Tarjeta guardada exitosamente")

if __name__ == "__main__":
    unittest.main()

codigo limpio proyecto Daniel Lasso/R1tests.py


import pandas as pd
from transformers import GPT2LMHeadModel, GPT2Tokenizer, TextDataset, DataCollatorForLanguageModeling
from transformers import TrainingArguments, Trainer
import re

artist_list = [
    'JAY-Z',
    'Eminem',
    'Kendrick Lamar',
    'Lil Wayne',
    'Nicki Minaj',
    'Snoop Dogg',
    'Nas',
    'Drake'
]

artist_dict = {}

# for index, row in df.iterrows():
for artist in artist_list:
    artist_filter = df[df['artist'].isin([artist])]['lyrics'].tolist()
    artist_filter_corpus = '\n'.join([str(item) for item in artist_filter])
    # print(artist_filter_corpus)
    # artist_filter_text = artist_filter
    artist_filter_text = re.sub('\[(.*).\]', '', artist_filter_corpus)
    artist_filter_text = artist_filter_text.replace('\n\n', '\n')
    artist_filter_text = artist_filter_text.replace('\n\n', '\n')
    artist_filter_text = artist_filter_text.replace('\n\n', '\n')
    # print(artist_filter_text)
    file_path = "./" + artist.lower() + "_" + "full_corpus.txt"
    with open(file_path, "w") as text_file:
        text_file.write(artist_filter_text)
    # artist_dict[artist] = file_path

    # Initiate the GPT-2 pre-trained model, plus the tokenizer
    model_name = "gpt2-medium"
    model = GPT2LMHeadModel.from_pretrained(model_name)
    tokenizer = GPT2Tokenizer.from_pretrained(model_name)

    # Tokenize the lyrics and prepare dataset

    # We'll set up the dataset through the tokenizer, referring to
    # the file we just wrote as the basis.
    train_dataset = TextDataset(
        tokenizer=tokenizer,
        file_path=file_path,  # Save the all_lyrics string to a file and provide its path here
        block_size=128
    )
    data_collator = DataCollatorForLanguageModeling(
        tokenizer=tokenizer,
        mlm=False
    )

    # Set up training arguments; these can be modified depending on
    # available architecture.
    training_args = TrainingArguments(
        output_dir="./results",
        overwrite_output_dir=True,
        num_train_epochs=1,
        per_device_train_batch_size=32,
        save_steps=10_000,
        save_total_limit=2,
    )

    # Initiate the Trainer function and start training!
    trainer = Trainer(
        model=model,
        args=training_args,
        data_collator=data_collator,
        train_dataset=train_dataset,
    )

    trainer.train()

    artist_dict[artist] = model

    input_text = "In a cosmic sort of way"
    input_ids = tokenizer.encode(input_text, return_tensors='pt')

    output = artist_dict['JAY-Z'].generate(input_ids, max_length=100, num_return_sequences=5, temperature=0.9, do_sample=True)

    for i, text in enumerate(output):
        print(f"Generated Text {i+1}: {tokenizer.decode(text)}")
        print()

    break