import matplotlib.pyplot as plt
import random
import numpy as np

def gradient_regression(X,y,alpha,b,w):
    dw = 0.0
    db = 0.0
    # we make the model by using all the samples
    for i in range(len(X)):
        aux = -2.0*(y[i] - (w*X[i] + b))
        db = db + aux # this solver can easily overflow
        dw = dw + X[i]*aux # this solver can easily overflow
    aux = 1.0/float(len(X))
    b = b - aux*db*alpha
    w = w - aux*dw*alpha
    return b, w

def gradient_regression2(X,y,alpha,b,w):
    aux = -2*(y-(w*X+b)).sum()
    b = b - alpha*aux/float(len(X))
    w = w - alpha*aux/float(len(X))
    return b,w

def plot(fig,X,y,b,w,epochs):
    axs[fig].plot(X,y,"yo", label= "Samples")
    X = np.array(X)
    axs[fig].plot(X,w*X+b, "k-",label="Regression loss: " + "{:9.2f}".format(loss(X,y,b,w)))
    axs[fig].set_xlabel("{:5.0f}".format(epochs) + "epochs")
    axs[fig].legend()
    axs[fig].grid()
    return

def model(X,y,alpha,b,w,epochs):
    fig = 0
    for e in range(epochs):
        b,w = gradient_regression(X,y,alpha,b,w)
        if e % 3000 == 0:
            plot(fig,X,y,b,w,e)
            fig += 1
    return b,w

def prediction(x,b,w):
    return x*w+b

def loss(X,y,b,w):
    sum = 0
    for i in range(len(X)):
        sum += (y[i]-prediction(X[i],b,w)) ** 2
    return sum/len(X)

def create_samples(n):
    y = []
    X = list(range(40))
    for i in range(len(X)):
        y.append(20+X[i]+random.random()*20)
    return X,y

def create_samples2(n):
    X = np.array(list(range(40)))
    y = 20+X+np.random.rand(40)*20
    return X,y

X,y = create_samples2(40)

# in this example, we dont change epoch's value, since we just show
# plots for the values (0,3000,6000,9000)

EPOCHS = 9001

# Linear regression evolution
fig, axs = plt.subplots(1,4,figsize=(15,4))
# b and w parameters can be better initialized
b,w = model(X,y,0.001, random.random(), random.random(), EPOCHS)
y = []
for i in range(len(X)):
    y.append((X[i]-10)**2 + random.random()*20*abs(len(X)/2-i))
# loss evolution
fig, axs = plt.subplots(1,4,figsize=(15,4))
# b and w parameters can be better initialized
model(X,y,0.001,random.random(), random.random(), EPOCHS)

plt.show()

print(prediction(10,b,w))
print(prediction(20,b,w))


from __future__ import annotations

import os
from typing import Any, AnyStr, AsyncGenerator, Dict, Generator, List, Tuple, TYPE_CHECKING, Union

if TYPE_CHECKING:
    from werkzeug.datastructures import Headers  # noqa: F401

    from .wrappers.response import Response  # noqa: F401

FilePath = Union[bytes, str, os.PathLike]

# The possible types that are directly convertible or are a Response object.
ResponseValue = Union[
    "Response",
    AnyStr,
    Dict[str, Any],  # any jsonify-able dict
    AsyncGenerator[bytes, None],
    Generator[bytes, None, None],
]
StatusCode = int

# the possible types for an individual HTTP header
HeaderName = str
HeaderValue = Union[str, List[str], Tuple[str, ...]]

# the possible types for HTTP headers
HeadersValue = Union["Headers", Dict[HeaderName, HeaderValue], List[Tuple[HeaderName, HeaderValue]]]

# The possible types returned by a route function.
ResponseReturnValue = Union[
    ResponseValue,
    Tuple[ResponseValue, HeadersValue],
    Tuple[ResponseValue, StatusCode],
    Tuple[ResponseValue, StatusCode, HeadersValue],
]


import os
from pathlib import Path

from cryptography.fernet import Fernet
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric import padding

def encrypt_dataset(
    rsa_keys_dir: Path, dataset_dir: Path, output_dir: Path
) -> None:
    """
    Encrypts a dataset as follows:
        1. Generates the symmetric key for this dataset.
        2. Encrypts the dataset using the symmetric key.
        3. Encrypts the symmetric key using the public rsa key.
    """

    if not rsa_keys_dir.exists():
        print("The RSA keys directory has to exist")
        raise SystemExit(1)

    if not dataset_dir.exists():
        print(
            "The directory containing the dataset files to encrypt has to exist."
        )
        raise SystemExit(1)

    if not output_dir.exists():
        os.makedirs(output_dir)

    public_key_location = rsa_keys_dir / "microdata_public_key.pem"

    if not public_key_location.is_file():
        print(f"Public key {public_key_location} not found")
        raise SystemExit(1)

    # Read public key from file
    with open(public_key_location, "rb") as key_file:
        public_key = serialization.load_pem_public_key(
            key_file.read(), backend=default_backend()
        )

    csv_files = [
        file for file in dataset_dir.iterdir() if file.suffix == ".csv"
    ]

    if len(csv_files) == 0:
        print(f"No csv files found in {dataset_dir}")
        raise SystemExit(1)

    if len(csv_files) > 1:
        print(f"There should only be one csv file in {dataset_dir}")
        raise SystemExit(1)

    csv_file = csv_files[0]
    dataset_name = csv_file.stem

    if dataset_name != dataset_dir.stem:
        print(
            f"The csv file name {dataset_name} should match "
            f"the dataset directory name {dataset_dir.stem}."
        )
        raise SystemExit(1)

    dataset_output_dir = output_dir / dataset_name
    os.makedirs(dataset_output_dir)

    encrypted_file = dataset_output_dir / f"{dataset_name}.csv.encr"
    encrypted_symkey_file = dataset_output_dir / f"{dataset_name}.symkey.encr"

    # Generate and store symmetric key for this file
    symkey = Fernet.generate_key()

    # Encrypt csv file
    with open(csv_file, "rb") as file:
        data = file.read()  # Read the bytes of the input file

    fernet = Fernet(symkey)
    encrypted = fernet.encrypt(data)

    with open(encrypted_file, "wb") as file:
        file.write(encrypted)

    print(f"Csv file {csv_file} encrypted into {encrypted_file}")

    encrypted_sym_key = public_key.encrypt(
        symkey,
        padding.OAEP(
            mgf=padding.MGF1(algorithm=hashes.SHA256()),
            algorithm=hashes.SHA256(),
            label=None,
        ),
    )

    # Store encrypted symkey to file
    with open(encrypted_symkey_file, "wb") as file:
        file.write(encrypted_sym_key)

    print(f"Key file for {csv_file} encrypted into {encrypted_symkey_file}")
    print("Encryption done!")


"""
created by VinnyHu
Distant supervision for relation extraction via Piecewise  Convolutional Neural Networks
using mask to implement pici-wise pooling ,learned by openNRE
used for Relation Extraction
learn how to write great codes!
"""
import torch
import torch.nn as nn
import torch.nn.functional as F
from config import DefaultConfig
from .BasicModule import BasicModule

import math

class PCNN(BasicModule):

    def __init__(self,opt,token2id,vectors = None):
        super(PCNN,self).__init__()

        self.token2id = token2id
        self.max_length = opt.max_length
        self.num_token = len(token2id)  #the num of word

        self.num_pos = self.max_length * 2
        self.word_dim = opt.word_embedding_dim
        self.pos_dim = opt.pos_size
        self.kernel_num = opt.kernel_num
        self.input_size = self.word_dim + self.pos_dim * 2
        self.blank_padding = opt.blank_padding
        self.fc = nn.Linear(in_features=self.kernel_num * 3,out_features=opt.num_classes)

        # initial word2vec
        self.word_embedding = nn.Embedding(num_embeddings=self.num_token,embedding_dim=self.word_dim)
        if vectors is not None:
            print("Initializing word embedding with word2vec...")
            vectors = torch.from_numpy(vectors)
            if self.num_token == len(vectors) + 2:
                unk = torch.randn(1, self.word_dim) / math.sqrt(self.word_dim)
                blk = torch.zeros(1, self.word_dim)  # 是否可以改进，比如随机成一个分布
                self.word_embedding.weight.data.copy_(torch.cat([vectors, unk, blk], 0))
            else:
                self.word_embedding.weight.data.copy_(vectors)
            print('Finished!')

        #position embedding
        self.pos1_Embedding = nn.Embedding(num_embeddings=self.num_pos,embedding_dim=self.pos_dim,padding_idx=0)
        self.pos2_Embedding = nn.Embedding(num_embeddings=self.num_pos,embedding_dim=self.pos_dim,padding_idx=0)

        self.dropout = nn.Dropout(opt.dropout)

        self.padding_size = opt.padding_size
        self.act = F.relu
        self.softmax = nn.Softmax(-1)

        # convs = [
        #     nn.Sequential(
        #         nn.Conv1d(in_channels=self.input_size,
        #                   out_channels=self.kernel_num,
        #                   kernel_size=kernel_size,
        #                   padding=self.padding_size  #why?
        #                   )
        #     )
        #     for kernel_size in opt.kernel_sizes
        # ]
        # self.conv = nn.ModuleList(convs)
        self.conv =  nn.Conv1d(in_channels=self.input_size,
                          out_channels=self.kernel_num,
                          kernel_size=opt.kernel_sizes,
                          padding=self.padding_size  #why?
                          )
        self.pool = nn.MaxPool1d(self.max_length)

        self.tanh = F.tanh

        self.mask_embedding = nn.Embedding(4,3)
        self.mask_embedding.weight.data.copy_(torch.FloatTensor([[0,0,0],[1,0,0],[0,1,0],[0,0,1]]))
        self.mask_embedding.weight.requires_grad = False
        self.minus = -1000

    def forward(self,token,pos1,pos2,mask):

        # if len(token.size()) != 2 or token.size() != pos1.size() or token.size() != pos2.size():
        #     raise Exception("Size of token, pos1 ans pos2 should be (B, L)")

        x = torch.cat([self.word_embedding(token),
                       self.pos1_Embedding(pos1),
                       self.pos2_Embedding(pos2)
                       ],2) # B L EM

        x = x.transpose(1,2) # B EM L conv1d B in_channels L

        x = self.conv(x)    #B H L
        mask = 1 - self.mask_embedding(mask).transpose(1,2)

        pool1 = self.pool(self.act(x + self.minus * mask[:,0:1,:]))
        pool2 = self.pool(self.act(x + self.minus * mask[:,1:2,:]))
        pool3 = self.pool(self.act(x + self.minus * mask[:,2:3,:]))

        x = torch.cat([pool1,pool2,pool3],1)
        x = x.squeeze(2) # B 3H

        x = self.tanh(x)
        x = self.fc(x)
        x = self.dropout(x)
        x = self.softmax(x)

        return x


import json as j
import yaml as y

def itemenc(item, cnt=1):
	encs = "enchantments";
	jsond = dict()
	if item.find("(")==-1:
		jsond["type"] = item.upper();
		jsond["amount"] = cnt;
	else:
		jsond["type"] = item[:item.find("(")].upper();
		jsond["amount"] = cnt;
		jsond[encs] = list();
		ench = item[item.find("(")+1:item.find(")")];
		ench = ench.split(sep=",");
		for i in range(len(ench)):
			ench[i] = ench[i].strip();
			ench[i] = ench[i].split(sep="+");
			jsond[encs].append(dict());
			jsond[encs][-1]["type"] = ench[i][0];
			jsond[encs][-1]["level"] = int(ench[i][1]);
	return j.dumps(jsond);

def kitenc(item, name, items):
	kit = dict();
	kit["symbol"] = dict();
	kit["symbol"]["item"] = item;
	kit["symbol"]["name"] = name;
	kit["items"] = items;
	return kit;

def main(ifile, ofile):
	item = str();
	case = int(ifile.readline().strip("\n"));
	kits = dict();
	for cntk in range(case):
		pro = ifile.readline();
		while pro == '\n':
			pro = ifile.readline();
		pro = pro.strip("\n");
		if pro.find("(")==-1:
			pro = pro.split();
			citem = int(pro[0]);
			idn = pro[1];
			item = itemenc(pro[2]);
			name = pro[3];
		else:
			pros = pro.split();
			citem = int(pros[0]);
			idn = pros[1];
			name = pros[-1];
			item = str();
			for i in pros[2:-1]:
				item += i;
			item = itemenc(item);
		items = list();
		for cnti in range(citem):
			kitem = ifile.readline().strip("\n");
			ind = kitem.rfind(' ');
#			sitem = kitem.split();
			if (ind!=-1) and (kitem[ind+1:].isdigit()):
				items.append(itemenc(kitem[:ind], int(kitem[ind+1:])));
			else:
				items.append(itemenc(kitem));

		kit = kitenc(item, name, items);
		kits[idn] = kit;

	ofile.write(y.dump({"kits":kits}));

if __name__ == "__main__":
	ifile = open("kits.in", "r");
	ofile = open("kits.yml", "w");
#	ifile = open(input("input file:"), "r");
#	ofile = open(input("output file:"), "w");
	main(ifile, ofile);
	ifile.close();
	ofile.close();
	print("解析完成，已存储为 kits.yml\n是否写入 ./plugins/UhcCore/kits.yml [Y/n] : ", end='')
	if input()[0]=='Y':
		ifile = open("kits.in", "r");
		ofile = open("plugins/UhcCore/kits.yml", "w");
		main(ifile, ofile);
		ifile.close();
		ofile.close();


from enum import Enum, auto

try:
    import CynanBotCommon.utils as utils
except:
    from ...CynanBotCommon.utils import utils

class PkmnCatchType(Enum):

    GREAT = auto()
    NORMAL = auto()
    ULTRA = auto()

    @classmethod
    def fromStr(cls, text: str):
        if not utils.isValidStr(text):
            raise ValueError(f'text argument is malformed: \"{text}\"')

        text = text.lower()

        if text == 'great':
            return PkmnCatchType.GREAT
        elif text == 'normal':
            return PkmnCatchType.NORMAL
        elif text == 'ultra':
            return PkmnCatchType.ULTRA
        else:
            raise ValueError(f'unknown PkmnCatchType: \"{text}\"')

    def getSortOrder(self) -> int:
        if self is PkmnCatchType.GREAT:
            return 1
        elif self is PkmnCatchType.NORMAL:
            return 0
        elif self is PkmnCatchType.ULTRA:
            return 2
        else:
            raise RuntimeError(f'unknown PkmnCatchType: \"{self}\"')

    def toStr(self) -> str:
        if self is PkmnCatchType.GREAT:
            return 'great'
        elif self is PkmnCatchType.NORMAL:
            return 'normal'
        elif self is PkmnCatchType.ULTRA:
            return 'ultra'
        else:
            raise RuntimeError(f'unknown PkmnCatchType: \"{self}\"')


from flask import render_template
from app import app
import requests
from bs4 import BeautifulSoup

url = "http://static.cricinfo.com/rss/livescores.xml"
teams = (
    "Chennai", "Delhi",
    "Punjab", "Kolkata",
    "Mumbai", "Rajasthan",
    "Bangalore", "Hyderabad"
    )

@app.route('/')
@app.route('/index')
def index():
	r = requests.get(url)

	while r.status_code is not 200:
	        r = requests.get(url)

	soup = BeautifulSoup(r.text)
	titles = soup.find_all("title")
	size = len(titles)
	matches = []
	for match in range(size):
	    if len(set(titles[match].text.split(" ")).intersection(set(teams))) > 0:
	    	matches.append(titles[match].text)
	return '''

    <title>Live IPL Score</title>

''' + '
'.join(matches) + '''

'''


from dataclasses import fields
from unicodedata import category
from django import forms
from blog.models import *
from lappyng_app.models import *
from django.core import validators

class CommentForm(forms.ModelForm):
    class Meta:
        model = Comment
        fields = ('name', 'email', 'body')

        widgets = {
            'name': forms.TextInput(attrs={'class': 'form-control', 'placeholder':'Enter Name'}),
            'email': forms.TextInput(attrs={'class': 'form-control', 'placeholder':'Enter Email'}),
            'body': forms.Textarea(attrs={'class': 'form-control'}),

        }

class ProductReviewForm(forms.ModelForm):
    ONE = '10'
    TWO = '20'
    THREE = '30'
    FOUR = '40'
    FIVE = '50'
    SIX = '60'
    SEVEN = '70'
    EIGHT = '80'
    NINE = '90'
    TEN = '100'
    CHOOSE = ''
    RATING_LIST = [
        (ONE, 1),
        (TWO, 2),
        (THREE, 3),
        (FOUR, 4),
        (FIVE, 5),
        (SIX, 6),
        (SEVEN, 7),
        (EIGHT, 8),
        (NINE, 9),
        (TEN, 10),
        (CHOOSE, 'Choose Rating'),
    ]

    full_name = forms.CharField(
                widget = forms.TextInput(
                    attrs={'class':'form-control', 'placeholder':'Fullname'}
                ),
                )
    email = forms.EmailField(
                widget = forms.EmailInput(
                    attrs={'class':'form-control', 'placeholder':'Email'}
                ),
                )
    rating = forms.CharField(
                widget = forms.Select(
                    attrs={'class':'form-control',},
                    choices=RATING_LIST,
                ),
                )

    review = forms.CharField(
                widget = forms.Textarea(
                    attrs={'class':'form-control',}
                ),
                )
    botcatcher = forms.CharField(
        required=False,
        widget=forms.HiddenInput,
        validators=[validators.MaxLengthValidator(0)])

    class Meta:
        model = ProductReview
        exclude = ('created_at', 'updated', 'product')

class ProductRequestForm(forms.ModelForm):
    name = forms.CharField(
                widget = forms.TextInput(
                    attrs={'class':'form-control', 'placeholder':'Fullname'}
                ),
                )
    email = forms.EmailField(
                widget = forms.EmailInput(
                    attrs={'class':'form-control', 'placeholder':'Email'}
                ),
                )
    phone = forms.CharField(
                widget = forms.TextInput(
                    attrs={'class':'form-control', 'placeholder':'Phone'}
                ),
                )
    description = forms.CharField(
                widget = forms.Textarea(
                    attrs={'class':'form-control',}
                ),
                )
    botcatcher = forms.CharField(
        required=False,
        widget=forms.HiddenInput,
        validators=[validators.MaxLengthValidator(0)])

    class Meta:
        model = ProductRequest
        exclude = ('created', 'modified', 'product')

class SearchForm(forms.ModelForm):
    title = forms.CharField(widget=forms.TextInput(
        attrs={'class':'form-control', 'placeholder':'Search your site'}
    ))
    category = forms.ModelChoiceField(
        queryset=Category.objects.all(),
        empty_label='All Categories',
        widget=forms.Select(attrs={'class':'categori-search-option'})
    )
    botcatcher = forms.CharField(
        required=False,
        widget=forms.HiddenInput,
        validators=[validators.MaxLengthValidator(0)])

    class Meta():
        model = Products
        fields = ['title', 'category']


from django.shortcuts import redirect
from django.http import HttpResponse, HttpResponseForbidden, JsonResponse
from django.core.exceptions import PermissionDenied
from django.core.paginator import Paginator
from django.conf import settings
from django.contrib import messages
# from django.contrib.gis.geos import Point
from django.contrib.gis.measure import D
from django.template import loader
from django.utils import timezone
from django.views.decorators.clickjacking import xframe_options_exempt
from django.views.decorators.csrf import ensure_csrf_cookie
from time import time
from urllib import parse as urlparse
import simplejson as json
from django.views.decorators.csrf import csrf_exempt
from django.db.models import Q
from .models import FileCategory, Project, ProjectFile, ProjectPosition, UserAlert, VolunteerRelation, Group, Event, ProjectRelationship
from .helpers.projects import projects_tag_counts
from .sitemaps import SitemapPages
from common.helpers.db import unique_column_values
from common.helpers.s3 import presign_s3_upload, user_has_permission_for_s3_file, delete_s3_file
from common.helpers.tags import get_tags_by_category,get_tag_dictionary
from common.helpers.form_helpers import is_co_owner_or_staff, is_co_owner, is_co_owner_or_owner, is_creator_or_staff
from .forms import ProjectCreationForm, EventCreationForm, GroupCreationForm
from common.helpers.qiqo_chat import get_user_qiqo_iframe
from democracylab.models import Contributor, get_request_contributor
from common.models.tags import Tag
from common.helpers.constants import FrontEndSection
from democracylab.emails import send_to_project_owners, send_to_project_volunteer, HtmlEmailTemplate, send_volunteer_application_email, \
    send_volunteer_conclude_email, notify_project_owners_volunteer_renewed_email, notify_project_owners_volunteer_concluded_email, \
    notify_project_owners_project_approved, contact_democracylab_email
from common.helpers.front_end import section_url, get_page_section
from common.helpers.caching import update_cached_project_url
from distutils.util import strtobool
from django.views.decorators.cache import cache_page
import requests

# TODO: Set getCounts to default to false if it's not passed? Or some hardening against malformed API requests
@cache_page(1200) #cache duration in seconds, cache_page docs: https://docs.djangoproject.com/en/2.1/topics/cache/#the-per-view-cache
def tags(request):
    url_parts = request.GET.urlencode()
    query_terms = urlparse.parse_qs(
        url_parts, keep_blank_values=0, strict_parsing=0)
    if 'category' in query_terms:
        category = query_terms.get('category')[0]
        queryset = get_tags_by_category(category)
        countoption = bool(strtobool(query_terms.get('getCounts')[0]))
        if countoption == True:
            activetagdict = projects_tag_counts()
            querydict = {tag.tag_name:tag for tag in queryset}
            resultdict = {}

            for slug in querydict.keys():
                resultdict[slug] = Tag.hydrate_tag_model(querydict[slug])
                resultdict[slug]['num_times'] = activetagdict[slug] if slug in activetagdict else 0
            tags = list(resultdict.values())
        else:
            tags = list(queryset.values())
    else:
        countoption = bool(strtobool(query_terms.get('getCounts')[0]))
        if countoption == True:
            queryset = Tag.objects.all()
            activetagdict = projects_tag_counts()
            querydict = {tag.tag_name:tag for tag in queryset}
            resultdict = {}

            for slug in querydict.keys():
                resultdict[slug] = Tag.hydrate_tag_model(querydict[slug])
                resultdict[slug]['num_times'] = activetagdict[slug] if slug in activetagdict else 0
            tags = list(resultdict.values())
        else:
            queryset = Tag.objects.all()
            tags = list(queryset.values())
    return JsonResponse(tags, safe=False)

def to_rows(items, width):
    rows = [[]]
    row_number = 0
    column_number = 0
    for item in items:
        rows[row_number].append(item)
        column_number += 1
        if column_number >= width:
            column_number = 0
            rows.append([])
            row_number += 1
    return rows

def to_tag_map(tags):
    tag_map = ((tag.tag_name, tag.display_name) for tag in tags)
    return list(tag_map)

# TODO: Pass csrf token in ajax call so we can check for it
@csrf_exempt
def group_create(request):
    if not request.user.is_authenticated():
        return redirect(section_url(FrontEndSection.LogIn))

    user = get_request_contributor(request)
    if not user.email_verified:
        # TODO: Log this
        return HttpResponse(status=403)

    group = GroupCreationForm.create_group(request)
    return JsonResponse(group.hydrate_to_json())

def group_edit(request, group_id):
    if not request.user.is_authenticated():
        return redirect('/signup')

    group = None
    try:
        group = GroupCreationForm.edit_group(request, group_id)
    except PermissionDenied:
        return HttpResponseForbidden()

    if request.is_ajax():
        return JsonResponse(group.hydrate_to_json())
    else:
        return redirect('/index/?section=AboutGroup&id=' + group_id)

# TODO: Pass csrf token in ajax call so we can check for it
@csrf_exempt
def group_delete(request, group_id):
    # if not logged in, send user to login page
    if not request.user.is_authenticated():
        return HttpResponse(status=401)
    try:
        GroupCreationForm.delete_group(request, group_id)
    except PermissionDenied:
        return HttpResponseForbidden()
    return HttpResponse(status=204)

def get_group(request, group_id):
    group = Group.objects.get(id=group_id)

    if group is not None:
        if group.is_searchable or is_creator_or_staff(get_request_contributor(request), group):
            return JsonResponse(group.hydrate_to_json())
        else:
            return HttpResponseForbidden()
    else:
        return HttpResponse(status=404)

@csrf_exempt
def group_add_project(request, group_id):
    body = json.loads(request.body)
    group = Group.objects.get(id=group_id)

    if group is not None and body["project_ids"] is not None:
        if not is_creator_or_staff(get_request_contributor(request), group):
            return HttpResponseForbidden()

        projects = Project.objects.filter(id__in=body["project_ids"])

        for project in projects:
            ProjectRelationship.create(group, project)

        return HttpResponse(status=204)
    else:
        return HttpResponse(status=404)

def group_delete_project(request, group_id):
    body = json.loads(request.body)
    group = Group.objects.get(id=group_id)
    project = Project.objects.get(id=body["project_id"])

    if group is not None and project is not None:
        if is_creator_or_staff(get_request_contributor(request), group):
            relationship = ProjectRelationship.objects.get(relationship_project=project.id, relationship_group=group.id)

            if relationship is not None:
                relationship.delete()
                return HttpResponse(status=204)

    return HttpResponse(status=404)

# TODO: Pass csrf token in ajax call so we can check for it
@csrf_exempt
def event_create(request):
    if not request.user.is_authenticated():
        return redirect(section_url(FrontEndSection.LogIn))

    user = get_request_contributor(request)
    if not user.email_verified:
        # TODO: Log this
        return HttpResponse(status=403)

    event = None
    try:
        event = EventCreationForm.create_event(request)
    except PermissionDenied:
        return HttpResponseForbidden()
    return JsonResponse(event.hydrate_to_json())

def event_edit(request, event_id):
    if not request.user.is_authenticated():
        return redirect('/signup')

    event = None
    try:
        event = EventCreationForm.edit_event(request, event_id)
    except PermissionDenied:
        return HttpResponseForbidden()

    if request.is_ajax():
        return JsonResponse(event.hydrate_to_json())
    else:
        return redirect('/index/?section=AboutEvent&id=' + event_id)

# TODO: Pass csrf token in ajax call so we can check for it
@csrf_exempt
def event_delete(request, event_id):
    # if not logged in, send user to login page
    if not request.user.is_authenticated():
        return HttpResponse(status=401)
    try:
        EventCreationForm.delete_event(request, event_id)
    except PermissionDenied:
        return HttpResponseForbidden()
    return HttpResponse(status=204)

def get_event(request, event_id):
    event = Event.objects.get(id=event_id)

    if event is not None:
        if event.is_searchable or is_creator_or_staff(get_request_contributor(request), event):
            return JsonResponse(event.hydrate_to_json())
        else:
            return HttpResponseForbidden()
    else:
        return HttpResponse(status=404)

def event_add_project(request, event_id):
    body = json.loads(request.body)
    event = Event.objects.get(id=event_id)

    if event is not None and body["project_ids"] is not None:
        if not is_creator_or_staff(get_request_contributor(request), event):
            return HttpResponseForbidden()

        projects = Project.objects.filter(id__in=body["project_ids"])

        for project in projects:
            ProjectRelationship.create(event, project)

        return HttpResponse(status=204)
    else:
        return HttpResponse(status=404)

def event_delete_project(request, event_id):
    body = json.loads(request.body)
    event = Event.objects.get(id=event_id)
    project = Project.objects.get(id=body["project_id"])

    if event is not None and project is not None:
        if is_creator_or_staff(get_request_contributor(request), event):
            relationship = ProjectRelationship.objects.get(relationship_project=project.id, relationship_event=event.id)

            if relationship is not None:
                relationship.delete()
                return HttpResponse(status=204)

    return HttpResponse(status=404)

# TODO: Pass csrf token in ajax call so we can check for it
@csrf_exempt
def project_create(request):
    if not request.user.is_authenticated():
        return redirect(section_url(FrontEndSection.LogIn))

    user = get_request_contributor(request)
    if not user.email_verified:
        # TODO: Log this
        return HttpResponse(status=403)

    project = ProjectCreationForm.create_or_edit_project(request, None)
    return JsonResponse(project.hydrate_to_json())

def project_edit(request, project_id):
    if not request.user.is_authenticated():
        return redirect('/signup')

    try:
        project = ProjectCreationForm.create_or_edit_project(request, project_id)
        # TODO:
        # update_cached_project_url(project_id)
    except PermissionDenied:
        return HttpResponseForbidden()

    if request.is_ajax():
        return JsonResponse(project.hydrate_to_json())
    else:
        return redirect('/index/?section=AboutProject&id=' + project_id)

# TODO: Pass csrf token in ajax call so we can check for it
@csrf_exempt
def project_delete(request, project_id):
    # if not logged in, send user to login page
    if not request.user.is_authenticated():
        return HttpResponse(status=401)
    try:
        ProjectCreationForm.delete_project(request, project_id)
    except PermissionDenied:
        return HttpResponseForbidden()
    return HttpResponse(status=204)

def get_project(request, project_id):
    project = Project.objects.get(id=project_id)

    if project is not None:
        if project.is_searchable or is_co_owner_or_staff(get_request_contributor(request), project):
            return JsonResponse(project.hydrate_to_json())
        else:
            return HttpResponseForbidden()
    else:
        return HttpResponse(status=404)

def approve_project(request, project_id):
    project = Project.objects.get(id=project_id)
    user = get_request_contributor(request)

    if project is not None:
        if user.is_staff:
            project.is_searchable = True
            project.save()
            SitemapPages.update()
            notify_project_owners_project_approved(project)
            messages.success(request, 'Project Approved')
            return redirect('/index/?section=AboutProject&id=' + str(project.id))
        else:
            return HttpResponseForbidden()
    else:
        return HttpResponse(status=404)

@ensure_csrf_cookie
@xframe_options_exempt
def index(request):
    template = loader.get_template('new_index.html')
    context = {
        'DLAB_PROJECT_ID': settings.DLAB_PROJECT_ID or '',
        'PROJECT_DESCRIPTION_EXAMPLE_URL': settings.PROJECT_DESCRIPTION_EXAMPLE_URL,
        'POSITION_DESCRIPTION_EXAMPLE_URL': settings.POSITION_DESCRIPTION_EXAMPLE_URL,
        'STATIC_CDN_URL': settings.STATIC_CDN_URL,
        'HEADER_ALERT': settings.HEADER_ALERT,
        'SPONSORS_METADATA': settings.SPONSORS_METADATA,
        'userImgUrl' : '',
        'PAYPAL_ENDPOINT': settings.PAYPAL_ENDPOINT,
        'PAYPAL_PAYEE': settings.PAYPAL_PAYEE,
        'PRESS_LINKS': settings.PRESS_LINKS,
        'organizationSnippet': loader.render_to_string('scripts/org_snippet.txt'),
        'GR_SITEKEY': settings.GR_SITEKEY,
        'FAVICON_PATH': settings.FAVICON_PATH,
        'BLOG_URL': settings.BLOG_URL,
        'EVENT_URL': settings.EVENT_URL,
        'PRIVACY_POLICY_URL': settings.PRIVACY_POLICY_URL
    }
    if settings.HOTJAR_APPLICATION_ID:
        context['hotjarScript'] = loader.render_to_string('scripts/hotjar_snippet.txt',
                                                          {'HOTJAR_APPLICATION_ID': settings.HOTJAR_APPLICATION_ID})

    GOOGLE_CONVERSION_ID = None
    page = get_page_section(request.get_full_path())
    if page and settings.GOOGLE_CONVERSION_IDS and page in settings.GOOGLE_CONVERSION_IDS:
        GOOGLE_CONVERSION_ID = settings.GOOGLE_CONVERSION_IDS[page]
    if settings.GOOGLE_PROPERTY_ID:
        context['googleScript'] = loader.render_to_string('scripts/google_snippet.txt',
                                                          {
                                                              'GOOGLE_PROPERTY_ID': settings.GOOGLE_PROPERTY_ID,
                                                              'GOOGLE_ADS_ID': settings.GOOGLE_ADS_ID,
                                                              'GOOGLE_CONVERSION_ID': GOOGLE_CONVERSION_ID
                                                          })

    if settings.GOOGLE_TAGS_ID:
        google_tag_context = {'GOOGLE_TAGS_ID': settings.GOOGLE_TAGS_ID}
        context['googleTagsHeadScript'] = loader.render_to_string('scripts/google_tag_manager_snippet_head.txt', google_tag_context)
        context['googleTagsBodyScript'] = loader.render_to_string('scripts/google_tag_manager_snippet_body.txt', google_tag_context)

    if hasattr(settings, 'SOCIAL_APPS_VISIBILITY'):
        context['SOCIAL_APPS_VISIBILITY'] = json.dumps(settings.SOCIAL_APPS_VISIBILITY)

    if hasattr(settings, 'HERE_CONFIG'):
        context['HERE_CONFIG'] = settings.HERE_CONFIG

    if request.user.is_authenticated():
        contributor = Contributor.objects.get(id=request.user.id)
        context['userID'] = request.user.id
        context['emailVerified'] = contributor.email_verified
        context['email'] = contributor.email
        context['firstName'] = contributor.first_name
        context['lastName'] = contributor.last_name
        context['isStaff'] = contributor.is_staff
        context['volunteeringUpForRenewal'] = contributor.is_up_for_volunteering_renewal()
        context['QIQO_IFRAME_URL'] = get_user_qiqo_iframe(contributor)

        thumbnail = ProjectFile.objects.filter(file_user=request.user.id,
                                               file_category=FileCategory.THUMBNAIL.value).first()
        if thumbnail:
            context['userImgUrl'] = thumbnail.file_url

    return HttpResponse(template.render(context, request))

def get_site_stats(request):
    active_volunteers = VolunteerRelation.objects.filter(deleted=False)

    stats = {
        'projectCount': Project.objects.filter(is_searchable=True, deleted=False).count(),
        'userCount': Contributor.objects.filter(is_active=True).count(),
        'activeVolunteerCount': active_volunteers.distinct('volunteer__id').count(),
        'dlVolunteerCount': active_volunteers.filter(is_approved=True, project__id=settings.DLAB_PROJECT_ID).count()
    }

    return JsonResponse(stats)

# TODO: Pass csrf token in ajax call so we can check for it
@csrf_exempt
def add_alert(request):
    body = json.loads(request.body)
    UserAlert.create_or_update(
        email=body['email'], filters=body['filters'], country=body['country'], postal_code=body['postal_code'])
    return HttpResponse(status=200)

def my_projects(request):
    contributor = get_request_contributor(request)
    response = {}
    if contributor is not None:
        owned_projects = Project.objects.filter(project_creator_id=contributor.id)
        volunteering_projects = contributor.volunteer_relations.all()
        response = {
            'owned_projects': [project.hydrate_to_list_json() for project in owned_projects],
            'volunteering_projects': volunteering_projects.exists() and list(map(lambda volunteer_relation: volunteer_relation.hydrate_project_volunteer_info(), volunteering_projects))
        }
    return JsonResponse(response)

def my_groups(request):
    contributor = get_request_contributor(request)
    response = {}
    if contributor is not None:
        owned_groups = Group.objects.filter(group_creator_id=contributor.id)
        response = {
            'owned_groups': [group.hydrate_to_list_json() for group in owned_groups],
        }
    return JsonResponse(response)

def my_events(request):
    contributor = get_request_contributor(request)
    response = {}
    if contributor is not None:
        owned_events = Event.objects.filter(event_creator_id=contributor.id)
        response = {
            'owned_events': [event.hydrate_to_list_json() for event in owned_events],
        }
    return JsonResponse(response)

def projects_list(request):
    url_parts = request.GET.urlencode()
    query_params = urlparse.parse_qs(url_parts, keep_blank_values=0, strict_parsing=0)
    project_relationships = None

    if 'group_id' in query_params:
        project_relationships = ProjectRelationship.objects.filter(relationship_group=query_params['group_id'][0])
    elif 'event_id' in query_params:
        project_relationships = ProjectRelationship.objects.filter(relationship_event=query_params['event_id'][0])

    if project_relationships is not None:
        project_ids = list(map(lambda relationship: relationship.relationship_project.id, project_relationships))
        project_list = Project.objects.filter(id__in=project_ids, is_searchable=True)
    else:
        project_list = Project.objects.filter(is_searchable=True)

    if request.method == 'GET':
        project_list = apply_tag_filters(project_list, query_params, 'issues', projects_by_issue_areas)
        project_list = apply_tag_filters(project_list, query_params, 'tech', projects_by_technologies)
        project_list = apply_tag_filters(project_list, query_params, 'role', projects_by_roles)
        project_list = apply_tag_filters(project_list, query_params, 'org', projects_by_orgs)
        project_list = apply_tag_filters(project_list, query_params, 'orgType', projects_by_org_types)
        project_list = apply_tag_filters(project_list, query_params, 'stage', projects_by_stage)
        if 'keyword' in query_params:
            project_list = project_list & projects_by_keyword(query_params['keyword'][0])

        if 'locationRadius' in query_params:
            project_list = projects_by_location(project_list, query_params['locationRadius'][0])

        if 'location' in query_params:
            project_list = projects_by_legacy_city(project_list, query_params['location'][0])

        project_list = project_list.distinct()

        if 'sortField' in query_params:
            project_list = projects_by_sortField(project_list, query_params['sortField'][0])
        else:
            project_list = projects_by_sortField(project_list, '-project_date_modified')

        project_count = len(project_list)

        project_paginator = Paginator(project_list, settings.PROJECTS_PER_PAGE)

        if 'page' in query_params:
            project_list_page = project_paginator.page(query_params['page'][0])
            project_pages = project_paginator.num_pages
        else:
            project_list_page = project_list
            project_pages = 1

    response = projects_with_meta_data(project_list_page, project_pages, project_count)

    return JsonResponse(response)

def recent_projects_list(request):
    if request.method == 'GET':
        url_parts = request.GET.urlencode()
        query_params = urlparse.parse_qs(url_parts, keep_blank_values=0, strict_parsing=0)
        project_count = int(query_params['count'][0]) if 'count' in query_params else 3
        project_list = Project.objects.filter(is_searchable=True)
        # Filter out the DemocracyLab project
        if settings.DLAB_PROJECT_ID.isdigit():
            project_list = project_list.exclude(id=int(settings.DLAB_PROJECT_ID))
        project_list = projects_by_sortField(project_list, '-project_date_modified')[:project_count]
        hydrated_project_list = list(project.hydrate_to_tile_json() for project in project_list)
        return JsonResponse({'projects': hydrated_project_list})

def apply_tag_filters(project_list, query_params, param_name, tag_filter):
    if param_name in query_params:
        tag_dict = get_tag_dictionary()
        tags_to_filter_by = query_params[param_name][0].split(',')
        tags_to_filter_by = clean_nonexistent_tags(tags_to_filter_by, tag_dict)
        if len(tags_to_filter_by):
            project_list = project_list & tag_filter(tags_to_filter_by)

    return project_list

def clean_nonexistent_tags(tags, tag_dict):
    return list(filter(lambda tag: tag in tag_dict, tags))

def projects_by_keyword(keyword):
    return Project.objects.filter(Q(project_name__icontains=keyword)
                                  | Q(project_short_description__icontains=keyword)
                                  | Q(project_description__icontains=keyword)
                                  | Q(project_description_solution__icontains=keyword)
                                  | Q(project_description_actions__icontains=keyword))

def projects_by_sortField(project_list, sortField):
    return project_list.order_by(sortField)

def projects_by_location(project_list, param):
    param_parts = param.split(',')
    location = Point(float(param_parts[1]), float(param_parts[0]))
    radius = float(param_parts[2])
    project_list = project_list.filter(project_location_coords__distance_lte=(location, D(mi=radius)))
    return project_list

def projects_by_legacy_city(project_list, param):
    param_parts = param.split(', ')
    if len(param_parts) > 1:
        project_list = project_list.filter(project_city=param_parts[0], project_state=param_parts[1])
    return project_list

def projects_by_issue_areas(tags):
    return Project.objects.filter(project_issue_area__name__in=tags)

def projects_by_technologies(tags):
    return Project.objects.filter(project_technologies__name__in=tags)

def projects_by_orgs(tags):
    return Project.objects.filter(project_organization__name__in=tags)

def projects_by_org_types(tags):
    return Project.objects.filter(project_organization_type__name__in=tags)

def projects_by_stage(tags):
    return Project.objects.filter(project_stage__name__in=tags)

def projects_by_roles(tags):
    # Get roles by tags
    positions = ProjectPosition.objects.filter(position_role__name__in=tags).select_related('position_project')

    # Get the list of projects linked to those roles
    return Project.objects.filter(positions__in=positions)

def project_countries():
    return unique_column_values(Project, 'project_country', lambda country: country and len(country) == 2)

def projects_with_meta_data(projects, project_pages, project_count):
    return {
        'projects': [project.hydrate_to_tile_json() for project in projects],
        'availableCountries': project_countries(),
        'tags': list(Tag.objects.values()),
        'numPages': project_pages,
        'numProjects': project_count
    }

def available_tag_filters(projects, selected_tag_filters):
    project_tags = projects_tag_counts(projects)
    # Remove any filters that are already selected
    for tag in selected_tag_filters:
        if project_tags[tag]:
            project_tags.pop(tag)
    return project_tags

def presign_project_thumbnail_upload(request):
    uploader = request.user.username
    file_name = request.GET['file_name'][:150]
    file_type = request.GET['file_type']
    file_extension = file_type.split('/')[-1]
    unique_file_name = file_name + '_' + str(time())
    s3_key = 'thumbnails/%s/%s.%s' % (
        uploader, unique_file_name, file_extension)
    return presign_s3_upload(
        raw_key=s3_key, file_name=file_name, file_type=file_type, acl="public-read")

def volunteer_operation_is_authorized(request, volunteer_relation):
    project_volunteers = VolunteerRelation.objects.filter(project=volunteer_relation.project)
    authorized_usernames = ([volunteer_relation.project.project_creator.username]
        + list(map(lambda co: co.volunteer.username, list(filter(lambda v: v.is_co_owner, project_volunteers)))))
    return request.user.username in authorized_usernames

# TODO: Pass csrf token in ajax call so we can check for it
@csrf_exempt
def delete_uploaded_file(request, s3_key):
    uploader = request.user.username
    has_permisson = user_has_permission_for_s3_file(uploader, s3_key)

    if has_permisson:
        delete_s3_file(s3_key)
        return HttpResponse(status=202)
    else:
        # TODO: Log this
        return HttpResponse(status=401)

# TODO: Pass csrf token in ajax call so we can check for it
@csrf_exempt
def contact_project_owner(request, project_id):
    if not request.user.is_authenticated():
        return HttpResponse(status=401)

    user = get_request_contributor(request)
    if not user.email_verified:
        return HttpResponse(status=403)

    body = json.loads(request.body)
    message = body['message']

    project = Project.objects.get(id=project_id)
    email_subject = '{firstname} {lastname} would like to connect with {project}'.format(
                    firstname=user.first_name,
                    lastname=user.last_name,
                    project=project.project_name)
    email_template = HtmlEmailTemplate(use_signature=False)\
        .paragraph('\"{message}\" - {firstname} {lastname}'.format(
            message=message,
            firstname=user.first_name,
            lastname=user.last_name))\
        .paragraph('To contact this person, email them at {email}'.format(email=user.email))
    send_to_project_owners(project=project, sender=user, subject=email_subject, template=email_template)
    return HttpResponse(status=200)

# TODO: Pass csrf token in ajax call so we can check for it
@csrf_exempt
def contact_project_volunteers(request, project_id):
    if not request.user.is_authenticated():
        return HttpResponse(status=401)

    user = get_request_contributor(request)

    body = json.loads(request.body)
    subject = body['subject']
    message = body['message']

    project = Project.objects.get(id=project_id)
    if not user.email_verified or not is_co_owner_or_owner(user, project):
        return HttpResponse(status=403)

    volunteers = VolunteerRelation.get_by_project(project)

    email_subject = '{project}: {subject}'.format(
        project=project.project_name,
        subject=subject)
    email_template = HtmlEmailTemplate(use_signature=False) \
        .paragraph('\"{message}\" - {firstname} {lastname}'.format(
        message=message,
        firstname=user.first_name,
        lastname=user.last_name)) \
        .paragraph('To reply, email at {email}'.format(email=user.email))
    for volunteer in volunteers:
        # TODO: See if we can send emails in a batch
        # https://docs.djangoproject.com/en/2.2/topics/email/#topics-sending-multiple-emails
        send_to_project_volunteer(volunteer, email_subject, email_template)
    return HttpResponse(status=200)

# TODO: Pass csrf token in ajax call so we can check for it
@csrf_exempt
def contact_project_volunteer(request, application_id):
    if not request.user.is_authenticated():
        return HttpResponse(status=401)

    user = get_request_contributor(request)
    volunteer_relation = VolunteerRelation.objects.get(id=application_id)
    project = volunteer_relation.project

    body = json.loads(request.body)
    subject = body['subject']
    message = body['message']

    # TODO: Condense common code between this and contact_project_volunteers
    if not user.email_verified or not is_co_owner_or_owner(user, project):
        return HttpResponse(status=403)

    email_subject = '{project}: {subject}'.format(
        project=project.project_name,
        subject=subject)
    email_template = HtmlEmailTemplate(use_signature=False) \
        .paragraph('\"{message}\" - {firstname} {lastname}'.format(
        message=message,
        firstname=user.first_name,
        lastname=user.last_name)) \
        .paragraph('To reply, email at {email}'.format(email=user.email))
    send_to_project_volunteer(volunteer_relation, email_subject, email_template)
    return HttpResponse(status=200)

# TODO: Pass csrf token in ajax call so we can check for it
@csrf_exempt
def volunteer_with_project(request, project_id):
    if not request.user.is_authenticated():
        return HttpResponse(status=401)

    user = get_request_contributor(request)
    if not user.email_verified:
        return HttpResponse(status=403)

    project = Project.objects.get(id=project_id)
    body = json.loads(request.body)
    projected_end_date = body['projectedEndDate']
    message = body['message']
    role = body['roleTag']
    volunteer_relation = VolunteerRelation.create(
        project=project,
        volunteer=user,
        projected_end_date=projected_end_date,
        role=role,
        application_text=message)
    send_volunteer_application_email(volunteer_relation)
    return HttpResponse(status=200)

# TODO: Pass csrf token in ajax call so we can check for it
@csrf_exempt
def renew_volunteering_with_project(request, application_id):
    if not request.user.is_authenticated():
        return HttpResponse(status=401)

    user = get_request_contributor(request)
    volunteer_relation = VolunteerRelation.objects.get(id=application_id)

    if not user.id == volunteer_relation.volunteer.id:
        return HttpResponse(status=403)

    body = json.loads(request.body)
    volunteer_relation.projected_end_date = body['projectedEndDate']
    volunteer_relation.re_enrolled_last_date = timezone.now()
    volunteer_relation.re_enroll_reminder_count = 0
    volunteer_relation.re_enroll_last_reminder_date = None
    volunteer_relation.save()

    notify_project_owners_volunteer_renewed_email(volunteer_relation, body['message'])
    return HttpResponse(status=200)

# TODO: Pass csrf token in ajax call so we can check for it
@csrf_exempt
def conclude_volunteering_with_project(request, application_id):
    if not request.user.is_authenticated():
        return HttpResponse(status=401)

    user = get_request_contributor(request)
    volunteer_relation = VolunteerRelation.objects.get(id=application_id)

    if not user.id == volunteer_relation.volunteer.id:
        return HttpResponse(status=403)

    send_volunteer_conclude_email(user, volunteer_relation.project.project_name)

    body = json.loads(request.body)
    volunteer_relation.delete()

    notify_project_owners_volunteer_concluded_email(volunteer_relation, body['message'])
    return HttpResponse(status=200)

# TODO: Pass csrf token in ajax call so we can check for it
@csrf_exempt
def accept_project_volunteer(request, application_id):
    # Redirect to login if not logged in
    if not request.user.is_authenticated():
        return redirect(section_url(FrontEndSection.LogIn, {'prev': request.get_full_path()}))

    volunteer_relation = VolunteerRelation.objects.get(id=application_id)
    about_project_url = section_url(FrontEndSection.AboutProject, {'id': str(volunteer_relation.project.id)})
    if volunteer_relation.is_approved:
        messages.add_message(request, messages.ERROR, 'This volunteer has already been approved.')
        return redirect(about_project_url)

    if volunteer_operation_is_authorized(request, volunteer_relation):
        # Set approved flag
        volunteer_relation.is_approved = True
        volunteer_relation.approved_date = timezone.now()
        volunteer_relation.save()
        update_project_timestamp(request, volunteer_relation.project)
        if request.method == 'GET':
            messages.add_message(request, messages.SUCCESS, volunteer_relation.volunteer.full_name() + ' has been approved as a volunteer.')
            return redirect(about_project_url)
        else:
            return HttpResponse(status=200)
    else:
        messages.add_message(request, messages.ERROR, 'You do not have permission to approve this volunteer.')
        return redirect(about_project_url)

# TODO: Pass csrf token in ajax call so we can check for it
@csrf_exempt
def promote_project_volunteer(request, application_id):
    volunteer_relation = VolunteerRelation.objects.get(id=application_id)
    if volunteer_operation_is_authorized(request, volunteer_relation):
        # Set co_owner flag
        volunteer_relation.is_co_owner = True
        volunteer_relation.save()
        update_project_timestamp(request, volunteer_relation.project)
        return HttpResponse(status=200)
    else:
        raise PermissionDenied()

# TODO: Pass csrf token in ajax call so we can check for it
@csrf_exempt
def reject_project_volunteer(request, application_id):
    volunteer_relation = VolunteerRelation.objects.get(id=application_id)
    if volunteer_operation_is_authorized(request, volunteer_relation):
        body = json.loads(request.body)
        message = body['rejection_message']
        email_template = HtmlEmailTemplate()\
        .paragraph('The project owner of {project_name} has declined your application for the following reason:'.format(project_name=volunteer_relation.project.project_name))\
        .paragraph('\"{message}\"'.format(message=message))
        email_subject = 'Your application to join {project_name}'.format(
            project_name=volunteer_relation.project.project_name)
        send_to_project_volunteer(volunteer_relation=volunteer_relation,
                                  subject=email_subject,
                                  template=email_template)
        update_project_timestamp(request, volunteer_relation.project)
        volunteer_relation.delete()
        return HttpResponse(status=200)
    else:
        raise PermissionDenied()

# TODO: Pass csrf token in ajax call so we can check for it
@csrf_exempt
def dismiss_project_volunteer(request, application_id):
    volunteer_relation = VolunteerRelation.objects.get(id=application_id)
    if volunteer_operation_is_authorized(request, volunteer_relation):
        body = json.loads(request.body)
        message = body['dismissal_message']
        email_template = HtmlEmailTemplate()\
        .paragraph('The owner of {project_name} has removed you from the project for the following reason:'.format(
            project_name=volunteer_relation.project.project_name))\
        .paragraph('\"{message}\"'.format(message=message))
        email_subject = 'You have been dismissed from {project_name}'.format(
            project_name=volunteer_relation.project.project_name)
        send_to_project_volunteer(volunteer_relation=volunteer_relation,
                               subject=email_subject,
                               template=email_template)
        update_project_timestamp(request, volunteer_relation.project)
        volunteer_relation.delete()
        return HttpResponse(status=200)
    else:
        raise PermissionDenied()

# TODO: Pass csrf token in ajax call so we can check for it
@csrf_exempt
def demote_project_volunteer(request, application_id):
    volunteer_relation = VolunteerRelation.objects.get(id=application_id)
    if volunteer_operation_is_authorized(request, volunteer_relation):
        volunteer_relation.is_co_owner = False
        volunteer_relation.save()
        update_project_timestamp(request, volunteer_relation.project)
        body = json.loads(request.body)
        message = body['demotion_message']
        email_template = HtmlEmailTemplate()\
        .paragraph('The owner of {project_name} has removed you as a co-owner of the project for the following reason:'.format(
            project_name=volunteer_relation.project.project_name))\
        .paragraph('\"{message}\"'.format(message=message))
        email_subject = 'You have been removed as a co-owner from {project_name}'.format(
            project_name=volunteer_relation.project.project_name)
        send_to_project_volunteer(volunteer_relation=volunteer_relation,
                               subject=email_subject,
                               template=email_template)
        return HttpResponse(status=200)
    else:
        raise PermissionDenied()

# TODO: Pass csrf token in ajax call so we can check for it
@csrf_exempt
def leave_project(request, project_id):
    volunteer_relation = VolunteerRelation.objects.filter(project_id=project_id, volunteer_id=request.user.id).first()
    if request.user.id == volunteer_relation.volunteer.id:
        body = json.loads(request.body)
        message = body['departure_message']
        if len(message) > 0:
            email_template = HtmlEmailTemplate()\
            .paragraph('{volunteer_name} is leaving {project_name} for the following reason:'.format(
                volunteer_name=volunteer_relation.volunteer.full_name(),
                project_name=volunteer_relation.project.project_name))\
            .paragraph('\"{message}\"'.format(message=message))
        else:
            email_template = HtmlEmailTemplate() \
                .paragraph('{volunteer_name} is leaving {project_name} for unspecified reasons.'.format(
                volunteer_name=volunteer_relation.volunteer.full_name(),
                project_name=volunteer_relation.project.project_name))
        email_subject = '{volunteer_name} is leaving {project_name}'.format(
            volunteer_name=volunteer_relation.volunteer.full_name(),
            project_name=volunteer_relation.project.project_name)
        send_to_project_owners(project=volunteer_relation.project,
                               sender=volunteer_relation.volunteer,
                               subject=email_subject,
                               template=email_template)
        update_project_timestamp(request, volunteer_relation.project)
        volunteer_relation.delete()
        return HttpResponse(status=200)
    else:
        raise PermissionDenied()

def update_project_timestamp(request, project):
    if not request.user.is_staff:
        project.update_timestamp()

#This will ask Google if the recaptcha is valid and if so send email, otherwise return an error.
#TODO: Return text strings to be displayed on the front end so we know specifically what happened
@csrf_exempt
def contact_democracylab(request):
    #first prepare all the data from the request body
    body = json.loads(request.body)
    # submit validation request to recaptcha
    r = requests.post(
      'https://www.google.com/recaptcha/api/siteverify',
      data={
        'secret': settings.GR_SECRETKEY,
        'response': body['reCaptchaValue']
      }
    )

    if r.json()['success']:
        # Successfuly validated, send email
        fn = body['fname']
        ln = body['lname']
        em = body['emailaddr']
        ms = body['message']
        contact_democracylab_email(fn, ln, em ,ms)
        return HttpResponse(status=200)

    # Error while verifying the captcha, do not send the email
    return HttpResponse(status=401)

def robots(request):
    template = loader.get_template('robots.txt')
    context = {
        'PROTOCOL_DOMAIN': settings.PROTOCOL_DOMAIN,
        'DISALLOW_CRAWLING': settings.DISALLOW_CRAWLING
    }

    return HttpResponse(template.render(context, request))

def team(request):
    response = {
        'board_of_directors': settings.BOARD_OF_DIRECTORS
    }

    if settings.DLAB_PROJECT_ID is not None:
        project = Project.objects.get(id=settings.DLAB_PROJECT_ID)
        response['project'] = project.hydrate_to_json()

    return JsonResponse(response)


import os
import glob
import torch
from torch import nn
import gym

import pybullet_envs
import shutil
gym.logger.set_level(40)

def wrap_monitor(env):
    return gym.wrappers.Monitor(
        env, '/tmp/monitor', video_callable=lambda x: True, force=True)

def play_mp4():
    path = glob.glob(os.path.join('/tmp/monitor', '*.mp4'))[0]
    shutil.move(path, './movies/outset.mp4')

class SACActor(nn.Module):

    def __init__(self, state_shape, action_shape, std=0.0):
        super().__init__()

        self.net = nn.Sequential(
            nn.Linear(state_shape[0], 256),
            nn.ReLU(inplace=True),
            nn.Linear(256, 256),
            nn.ReLU(inplace=True),
            nn.Linear(256, 2 * action_shape[0]),
        )
        self.std = std

    def forward(self, states):
        with torch.no_grad():
            actions = torch.tanh(self.net(states).chunk(2, dim=1)[0])
            return actions.add_(torch.randn_like(actions) * self.std).clamp_(-1.0, 1.0)

    def load(self, path):
        self.load_state_dict(torch.load(path))

def run_expert(env_id, weight_path, std=0.0, device=torch.device('cuda')):
    env = wrap_monitor(gym.make(env_id))

    actor = SACActor(env.observation_space.shape, env.action_space.shape, std).to(device)
    actor.load(weight_path)

    state = env.reset()
    done = False

    while (not done):
        action = actor(torch.tensor(state, dtype=torch.float, device=device).unsqueeze_(0)).cpu().numpy()[0]
        state, _, done, _ = env.step(action)

    del env
    del actor
    play_mp4()

if __name__ == "__main__":
    run_expert(
        'HalfCheetahBulletEnv-v0',
        os.path.join('./logs/expert', 'expert.pth'),
        std=0.1
    )


# This is just a script to print json files neatly.

from pprint import pprint
import json
import argparse

#Read Args
parser = argparse.ArgumentParser()
parser.add_argument("filea", help="First Json.")
parser.add_argument("fileb", help="Second Json.")
args = parser.parse_args()

#Setup printer (just for debug output)
#pp = pprint.PrettyPrinter(indent=4)

with open(args.filea, 'r') as handle:
    filea = json.load(handle)

with open(args.fileb, 'r') as handle:
    fileb = json.load(handle)

print("PRINTING FILE A")
with open('fileaout.txt', 'w') as handle:
    pprint(filea,stream=handle)

print("PRINTING FILE B")
with open('filebout.txt', 'w') as handle:
    pprint(fileb,stream=handle)


from twilio.twiml.messaging_response import MessagingResponse
from flask import Flask, request, render_template
from hue_controller import HueController
from name_converter import clean_name
from data_writer import writeFile,colorPercent,mostRecentColors,numOfEachColor,invalidColors,firstEntryDate
from previousRedisColor import PreviousColorsRedis
from html_colors_generator import generate_colors_from_redis
from colors_redis import colorsRedis
import logging

logging.basicConfig(level=logging.INFO,filename="hue_log.log",
                    format="%(asctime)s:%(levelname)s:%(message)s"	)

app = Flask(__name__)
controller = HueController()
file = "data.csv"

@app.route('/', methods=['POST'])
def set_color():
    phone_number = request.values.get('From', None)
    color_name = request.values.get('Body', None)
    color_name = clean_name(color_name)

    if color_name == "black":
        response = MessagingResponse()
        response.message("Haha... please use a color that contains light.")
        return str(response)

    if color_name == "previous":
        prev_color = PreviousColorsRedis().getPreviousColor()
        response = MessagingResponse()
        response.message("The previous color was {}".format(clean_name(prev_color)))
        return str(response)

    if color_name == "colors":
        colors_link = 'http://localhost:5000/colors'  # the link of the end point for '/colors'
        response = MessagingResponse()
        response.message("{}".format(colors_link))
        return str(response)

    message = controller.set_color(color_name)
    percent = colorPercent(file,color_name)
    date = firstEntryDate(file)
    response = MessagingResponse()
    response.message(message + " This entry has been chosen " + str(percent) + "% of the time since " + date + "!")
    logging.info("Color " + color_name + " has been set by the phone number " + phone_number + ".")
    writeFile(file,str(phone_number), str(color_name), str(message))
    if colorsRedis().is_color(color_name):
        PreviousColorsRedis().updatePreviousColor(color_name)

    return str(response)

@app.route('/recents',methods=['GET'])
def get_most_recent():
    return mostRecentColors(file)
@app.route('/number',methods=['GET'])
def get_num_of_each():
    return numOfEachColor(file)
@app.route('/invalids',methods=['GET'])
def get_invalids():
    return invalidColors(file)

@app.route('/colors',methods=['GET'])
def get_colors():
    """return a html page that display the colors with their name"""
    colors = generate_colors_from_redis()
    return render_template('displaycolors.html', colors= colors.items())

if __name__ == '__main__':
    app.run()
    logging.info("Server has been stopped")


import csv
import flet as ft
from movie_class import Movie

WELCOME = "Welcome to KinoPaís! \nThe best place to find global movies! \nPlease choose one of the countries below and revieve a list of the top ten movies from that country!\n"
COUNTS = ["Africa", "Balkans", "Benelux", "Central America", "Canada", "Germany", "Greater China", "Iberia", "India", "Italy", "Japan", "Korea", "Mexico", "Middle East", "Russia", "Scandinavia", "South America", "United Kingdom", "United States"]

COUNTRIES = ''
COUNTS = sorted(COUNTS)
for x in COUNTS:
  if x == COUNTS[len(COUNTS)-1]:
    COUNTRIES = COUNTRIES + x + '.'
  else:
    COUNTRIES = COUNTRIES + x + ', '

for x in range(len(COUNTS)):
  COUNTS[x] = COUNTS[x].lower()

def main(page: ft.Page):
  global count
  count = 0
  page.horizontal_alignment = 'center'
  def btn_click(e):
    global count
    if not txt_country.value:
        txt_country.error_text = "Please enter a country"
        page.update()
    elif txt_country.value.lower() not in COUNTS:
        txt_country.error_text = "Invalid Entry. Please Try Again"
        page.update()
    else:
        #this is where we take the users input
        if count != 0:
          try:
            page.controls.pop(5)
          except:
            pass

        count = 1

        csv_filename = txt_country.value

        movie_list = []

        fp = open(csv_filename + ".csv", mode="r", encoding='latin 1')
        fp.readline()

        for line in fp:
          line = line.strip().split(',$,')
          film_title = line[0]
          release_year = int(line[1])
          director = line[2]
          cast = list(line[3])
          personal_rating = line[4]
          average_rating = float(line[5])
          url = line[6]

          movie_list.append(Movie(film_title, release_year, director, cast, personal_rating, average_rating, url))

        lv = ft.ListView(expand=1, spacing=10, padding=20, auto_scroll=True)

        #this is where we add the algorithms output
        for i in range(10):
          lv.controls.append(ft.Text(f"{movie_list[i]}",text_align='center'))
        page.add(lv)

  title =  ft.Row([ft.Text('KinoPaís', style=ft.TextThemeStyle.DISPLAY_LARGE)],alignment=ft.MainAxisAlignment.CENTER, wrap=True)
  welcome = ft.Row([ft.Text(WELCOME, style=ft.TextThemeStyle.TITLE_SMALL, text_align='center')],alignment=ft.MainAxisAlignment.CENTER, wrap=True)
  instructions = ft.Row([ft.Text(COUNTRIES, style=ft.TextThemeStyle.TITLE_SMALL, text_align='center')],alignment=ft.MainAxisAlignment.CENTER, wrap=True)

  txt_country = ft.TextField(label="Country")

  page.add(title, welcome, instructions,
  txt_country, ft.Row([ft.ElevatedButton("GO!", icon="MOVIE", bgcolor='BLACK', on_click=btn_click)], alignment=ft.MainAxisAlignment.CENTER))

if __name__ == "__main__":
    ft.app(target=main)


__author__ = "V.A. Sole - ESRF"
__contact__ = "sole@esrf.fr"
__license__ = "MIT"
__copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
import os
import sys
import numpy
import re
import logging
from PyMca5.PyMcaIO import specfile
from PyMca5.PyMcaIO import Fit2DChiFileParser
from PyMca5.PyMcaIO import APSMEDFileParser
from PyMca5.PyMcaIO import SRSFileParser
from PyMca5.PyMcaIO import BAXSCSVFileParser
from PyMca5.PyMcaIO import OlympusCSVFileParser
from PyMca5.PyMcaIO import ThermoEMSFileParser
from PyMca5.PyMcaIO import JcampFileParser
from PyMca5.PyMcaIO import BlissSpecFile

_logger = logging.getLogger(__name__)

try:
    from PyMca5.PyMcaIO import ArtaxFileParser
    SPX = True
except Exception:
    _logger.info("specfilewrapper cannot import ArtaxFileParser")
    SPX = False

if sys.version >= '2.6':
    def safe_str(bytesObject):
        try:
            return str(bytesObject, 'utf-8')
        except UnicodeDecodeError:
            try:
                return str(bytesObject, 'latin-1')
            except Exception:
                try:
                    return str(bytesObject, 'utf-16')
                except Exception:
                    return str(bytesObject)
else:
    def safe_str(*var, **kw):
        return str(var[0])

    #python 2.5 does not have bytes function
    def bytes(*var, **kw):
        return var[0]

def Specfile(filename):
    if BlissSpecFile.isBlissSpecFile(filename):
        return BlissSpecFile.BlissSpecFile(filename)
    if sys.version_info < (3, 0):
        f = open(filename)
    else:
        f = open(filename, 'r', errors="ignore")
    line0  = f.readline()
    if filename.upper().endswith('DTA'):
        #TwinMic single column file
        line = line0 * 1
        line = line.replace('\r','')
        line = line.replace('\n','')
        line = line.replace('\t',' ')
        s = line.split(' ')
        if len(s) == 2:
            if len(s[-1]) == 0:
                try:
                    float(s[0])
                    f.close()
                    output = specfilewrapper(filename, dta=True)
                    return output
                except Exception:
                    #try to read in other way
                    pass

    # this piece of code checks if we deal with a SPEC file
    # Prior to any data, all lines have to be either empty or starting
    # by the hash character.
    line = line0
    while(len(line)):
        if len(line) > 1:
            if line[0:2] == '#S':
                if ('#SIGNALTYPE' in line) or \
                   ('#SPECTRUM' in line):
                    line = ""
                break
            elif line[0] not in ['#', ' ', '\r']:
                line = ""
                break
        try:
            line = f.readline()
        except Exception:
            line = ""
            break
    f.close()
    # end of specfile identification
    amptek = False
    qxas   = False
    if len(line):
        #it is a Specfile
        _logger.debug("This looks as a specfile")
        output=specfile.Specfile(filename)
    elif SPX and ArtaxFileParser.isArtaxFile(filename):
        _logger.debug("This looks as an Artax file")
        output = ArtaxFileParser.ArtaxFileParser(filename)
    else:
        _logger.debug("this does not look as a specfile")
        if len(line0) > 7:
            if line0.startswith('$SPEC_ID') or\
               line0.startswith('$DATE_MEA') or\
               line0.startswith('$MEAS_TIM') or\
               line0.startswith('$Core_ID') or\
               line0.startswith('$Section_ID'):
                qxas = True
        if (not qxas) and line0.startswith('<<'):
                amptek = True
        if (not qxas) and (not amptek) and Fit2DChiFileParser.isFit2DChiFile(filename):
            return Fit2DChiFileParser.Fit2DChiFileParser(filename)
        if (not qxas) and (not amptek) and APSMEDFileParser.isAPSMEDFile(filename):
            return APSMEDFileParser.APSMEDFileParser(filename)
        if (not qxas) and (not amptek) and SRSFileParser.isSRSFile(filename):
            _logger.debug("SRSFileParser")
            return SRSFileParser.SRSFileParser(filename)
        if (not qxas) and (not amptek) and BAXSCSVFileParser.isBAXSCSVFile(filename):
            _logger.debug("BAXSCSVFileParser")
            return BAXSCSVFileParser.BAXSCSVFileParser(filename)
        if (not qxas) and (not amptek) and \
           OlympusCSVFileParser.isOlympusCSVFile(filename):
            _logger.debug("OlympusCSVFileParser")
            return OlympusCSVFileParser.OlympusCSVFileParser(filename)
        if (not qxas) and (not amptek) and \
           ThermoEMSFileParser.isThermoEMSFile(filename):
            _logger.debug("ThermoEMSFileParser")
            return ThermoEMSFileParser.ThermoEMSFileParser(filename)
        if (not qxas) and (not amptek) and \
           JcampFileParser.isJcampFile(filename):
            _logger.debug("JcampFileParser")
            return JcampFileParser.JcampFileParser(filename)
        output = specfilewrapper(filename, amptek=amptek, qxas=qxas)
    return output

class specfilewrapper(object):
    def __init__(self, filename, amptek=None, qxas=None, dta=None):
        if amptek is None:
            amptek = False
        if qxas is None:
            qxas = False
        if dta is None:
            dta = False
        self.amptek = amptek
        self.qxas = qxas
        self.dta = dta
        self.ketek = None
        self.header = []
        if self.dta:
            #TwinMic .dta files with only one spectrum
            if 0:
                f = open(filename, 'rb')
                raw_content = f.read()
                f.close()
                expr = r'([-+]?\d+)\t\r\n'
                self.data = [float(i) for i in re.split(expr,raw_content) if i != '']
                self.data = numpy.array(self.data, numpy.float32)
            else:
                self.data = numpy.fromfile(filename,
                                           dtype=numpy.float32,
                                           sep='\t\r\n')
            self.header = ['#S1 %s' % os.path.basename(filename)]
            self.data.shape = -1, 1
            self.scandata=[myscandata(self.data,'MCA','1.1',
                                      scanheader=self.header)]
            return
        if self.qxas:
            f = open(filename)
        else:
            f = BufferedFile(filename)
        line = f.readline()
        outdata = []
        ncol0 = -1
        nlines= 0
        if amptek:
            if sys.version < '3.0':
                while "<<DATA>>" not in line:
                    self.header.append(line.replace("\n",""))
                    line = f.readline()
            else:
                while bytes("<<DATA>>", 'utf-8') not in line:
                    self.header.append(safe_str(line.replace(bytes("\n", 'utf-8'),
                                                    bytes("", 'utf-8'))))
                    line = f.readline()
        elif qxas:
            line.replace("\n","")
            line.replace("\x00","")
            self._qxasHeader = {}
            self._qxasHeader['S'] = '#S1 '+ " Unlabelled Spectrum"
            while 1:
                self.header.append(line)
                if line.startswith('$SPEC_ID:'):
                    line = f.readline().replace("\n","")
                    line.replace("\x00","")
                    self.header.append(line)
                    self._qxasHeader['S'] = '#S1 '+ line
                if line.startswith('$DATE_MEA'):
                    line = f.readline().replace("\n","")
                    self.header.append(line)
                    self._qxasHeader['D'] = line
                if line.startswith('$MEAS_TIM'):
                    line = f.readline().replace("\n","")
                    self.header.append(line)
                    tmpList = [float(i) for i in line.split()]
                    if len(tmpList) == 1:
                        preset = tmpList[0]
                        elapsed = preset
                    else:
                        preset, elapsed = tmpList[0:2]
                    self._qxasHeader['@CTIME'] = ['#@CTIME %f %f %f' % (preset, preset, elapsed)]
                if line.startswith('$MCA_CAL'):
                    try:
                       line = f.readline().replace("\n","")
                       self.header.append(line)
                       if line.startswith('$'):
                           continue
                       line = f.readline().replace("\n","")
                       self.header.append(line)
                       if line.startswith('$'):
                           continue
                       coefficients = [float(i) for i in line.split()]
                       if len(coefficients) == 2:
                           coefficients.append(0.0)
                       self._qxasHeader['@CALIB']=  ['#@CALIB %f  %f  %f' %\
                                            (coefficients[0], coefficients[1], coefficients[2])]
                    except Exception:
                        pass
                if line.startswith('$DATA:'):
                    line = f.readline().replace("\n","")
                    self.header.append(line)
                    start, stop = [int(i) for i in line.split()]
                    self._qxasHeader['@CHANN'] = ['#@CHANN  %d  %d  %d  1' % (stop-start+1, start, stop)]
                    break
                line = f.readline().replace("\n","")
        if qxas:
            outdata = []
            line = f.readline().replace("\n","")
            while len(line):
                if line[0] == "$":
                    break
                outdata += [float(x) for x in line.split()]
                line = f.readline().replace("\n","")
            nlines = len(outdata)
            f.close()
            self.data = numpy.resize(numpy.array(outdata).astype(numpy.float64),(nlines,1))
        else:
            if sys.version < '3.0':
                line = line.replace(",","  ")
                line = line.replace(";","  ")
                line = line.replace("\t","  ")
                line = line.replace("\r","\n")
                line = line.replace('"',"")
                line = line.replace('\n\n',"\n")
            else:
                tmpBytes = bytes(" ",'utf-8')
                line = line.replace(bytes(",",'utf-8'), tmpBytes)
                line = line.replace(bytes(";",'utf-8'), tmpBytes)
                line = line.replace(bytes("\t",'utf-8'), tmpBytes)
                tmpBytes = bytes("\n",'utf-8')
                line = line.replace(bytes("\r","utf-8"), tmpBytes)
                line = line.replace(bytes('"',"utf-8"), bytes("", "utf-8"))
                line = line.replace(bytes('\n\n',"utf-8"), tmpBytes)
            while(len(line)):
                values = line.split()
                if len(values):
                    try:
                        reals = [float(x) for x in values]
                        ncols = len(reals)
                        if ncol0 < 0:ncol0 = ncols
                        if ncols == ncol0:
                            outdata.append(reals)
                            nlines += 1
                    except Exception:
                        if len(line) > 1:
                            if sys.version < '3.0':
                                self.header.append(line.replace("\n",""))
                            else:
                                self.header.append(safe_str(line.replace(\
                                                    bytes("\n",'utf-8'),\
                                                    bytes("", 'utf-8'))))
                else:
                    if len(line) > 1:
                        if sys.version < '3.0':
                            self.header.append(line.replace("\n",""))
                        else:
                            self.header.append(safe_str(line.replace(bytes("\n",'utf-8'),
                                                            bytes("", 'utf-8'))))
                line = f.readline()
                if sys.version < '3.0':
                    line = line.replace(",","  ")
                    line = line.replace(";","  ")
                    line = line.replace("\t","  ")
                    line = line.replace("\r","\n")
                    line = line.replace('"',"")
                    line = line.replace('\n\n',"\n")
                else:
                    tmpBytes = bytes(" ",'utf-8')
                    line = line.replace(bytes(",",'utf-8'), tmpBytes)
                    line = line.replace(bytes(";",'utf-8'), tmpBytes)
                    line = line.replace(bytes("\t",'utf-8'), tmpBytes)
                    tmpBytes = bytes("\n",'utf-8')
                    line = line.replace(bytes("\r","utf-8"), tmpBytes)
                    line = line.replace(bytes('"',"utf-8"), bytes("", "utf-8"))
                    line = line.replace(bytes('\n\n',"utf-8"), tmpBytes)
            f.close()
            self.data = numpy.resize(numpy.array(outdata).astype(numpy.float64),(nlines,ncol0))
        if self.amptek:
            self.scandata=[myscandata(self.data,'MCA','1.1',
                                      scanheader=self.header)]
        elif self.qxas:
            self.scandata=[myscandata(self.data,'MCA','1.1',
                                      scanheader=self.header,
                                      qxas=self._qxasHeader)]
        else:
            labels = None
            if len(self.header) == 1:
                if len(self.header[0]) > 0:
                    labels = self.header[0].split("  ")
                    if len(labels) != ncol0:
                        labels = None
            # check if it is a KETEK AXAS-D file
            ketek_keys = ["File Version = ",
                          "Livetime = ", "Live Time: = ",
                          "Realtime = ", "Real Time: = ",
                          "Input Count Rate = ", "ICR: = ",
                          "Output Count Rate = ", "OCR: = ",
                          "= KETEK"]
            ketek_counter = 0
            icr = None
            ocr = None
            live_time = None
            live_time = None
            real_time = None
            for line in self.header:
                for key in ketek_keys:
                    if key in line:
                        keylower = line.lower()
                        if keylower.startswith("livetime =") or keylower.startswith("live time: = "):
                            tokens = line.split(" = ")[-1].split()
                            if tokens[-1] == "s":
                                if "." in tokens[0]:
                                    live_time = float(tokens[0])
                                else:
                                    live_time = float(tokens[0] + "." + tokens[1])
                        elif keylower.startswith("realtime =") or keylower.startswith("real time: = "):
                            tokens = line.split(" = ")[-1].split()
                            if tokens[-1] == "s":
                                if "." in tokens[0]:
                                    real_time = float(tokens[0])
                                else:
                                    real_time = float(tokens[0] + "." + tokens[1])
                        elif keylower.startswith("input count rate = ") or keylower.startswith("icr: = "):
                            tokens = line.split(" = ")[-1].split()
                            if "." in tokens[0]:
                                icr = float(tokens[0])
                            else:
                                icr = float(tokens[0] + "." + tokens[1])
                        elif keylower.startswith("output count rate = ") or keylower.startswith("ocr: = "):
                            tokens = line.split(" = ")[-1].split()
                            if "." in tokens[0]:
                                ocr = float(tokens[0])
                            else:
                                ocr = float(tokens[0] + "." + tokens[1])
                        ketek_counter += 1
                        break
            if ketek_counter >= 4:
                self.ketek = 1
                if real_time and live_time:
                    self._ketekHeader = {}
                    self._ketekHeader['S'] = '#S1 '+ " Unlabelled Spectrum"
                    if ocr and icr:
                        live_time = real_time * ocr / icr
                        _logger.info("Taking live time =  real_time * ocr / icr")
                    else:
                        _logger.warning("Taking live time from file")
                    self._ketekHeader['@CTIME'] = ['#@CTIME %f %f %f' % (real_time,
                                                                         live_time,
                                                                         real_time)]
                else:
                    self._ketekHeader = None
                self.scandata=[myscandata(self.data,'MCA','1.1',
                                      fileheader=self.header,
                                      qxas=self._ketekHeader)]
            else:
                self.scandata=[myscandata(self.data,'SCAN','1.1',
                                      labels=labels,
                                      fileheader=self.header),
                               myscandata(self.data,'MCA','2.1',
                                      fileheader=self.header)]

    def list(self):
        if self.amptek or self.qxas or self.dta or self.ketek:
            return "1:1"
        else:
            return "1:2"

    def __getitem__(self,item):
        return self.scandata[item]

    def __len__(self):
        return self.scanno()

    def select(self,i):
        n = i.split(".")
        return self.__getitem__(int(n[0]) - 1)

    def scanno(self):
        if self.amptek or self.qxas or self.dta or self.ketek:
            return 1
        else:
            return 2

class myscandata(object):
    def __init__(self, data, scantype=None, identification=None,
                 scanheader=None, qxas=None, labels=None, fileheader=None):
        if identification is None:
            identification='1.1'
        if scantype is None:
            scantype='SCAN'
        self.qxas = qxas
        self.scanheader = scanheader
        if fileheader is None:
            fileheader = []
        self._fileheader = fileheader
        #print shape(data)
        (rows, cols) = numpy.shape(data)
        if scantype == 'SCAN':
            self.__data = numpy.zeros((rows, cols +1 ), numpy.float64)
            self.__data[:,0] = numpy.arange(rows) * 1.0
            self.__data[:,1:] = data * 1
            self.__cols = cols + 1
            self.labels = ['Point']
            if labels is None:
                for i in range(cols):
                    self.labels.append('Column %d'  % i)
            else:
                for label in labels:
                    self.labels.append('%s' % label)
        else:
            self.__data = data
            self.__cols = cols
            self.labels = []
        self.scantype = scantype
        self.rows = rows
        if scanheader is None:
            labels = '#L '
            for label in self.labels:
                labels += '  '+label
            if self.scantype == 'SCAN':
                self.scanheader = ['#S1 Unknown command',
                              '#N %d' % len(self.labels),
                              labels]
            else:
                self.scanheader = ['#S1 Unknown command']

        n = identification.split(".")
        self.__number = int(n[0])
        self.__order  = int(n[1])

    def alllabels(self):
        if self.scantype == 'SCAN':
            return self.labels
        else:
            return []

    def allmotorpos(self):
        return []

    def cols(self):
        return self.__cols

    def command(self):
        _logger.debug("command called")
        if self.qxas is not None:
            if 'S' in self.qxas:
                text = self.qxas['S']
        elif self.scanheader is not None:
            if len(self.scanheader):
                text = self.scanheader[0]
        return text

    def data(self):
        return numpy.transpose(self.__data)

    def datacol(self, col):
        # it is awful that starts at one ...
        if col <= 0:
            raise ValueError("Specfile column numberig starts at 1")
        return self.__data[:, col - 1]

    def dataline(self, line):
        # it is awful that starts at one ...
        if line <= 0:
            raise ValueError("Specfile line numberig starts at 1")
        return self.__data[line - 1,:]

    def date(self):
        text = 'sometime'
        if self.qxas is not None:
            if 'D' in self.qxas:
                return self.qxas['D']
        elif self.scanheader is not None:
            for line in self.scanheader:
                if 'START_TIME' in line:
                    text = "%s" % line
                    break
        return text

    def fileheader(self, key=''):
        # key is there for compatibility
        _logger.debug("file header called")
        return self._fileheader

    def header(self, key):
        if self.qxas is not None:
            if key in self.qxas:
                return self.qxas[key]
            elif key == "" or key == " ":
                return self.scanheader
        if key == 'S':
            return self.scanheader[0]
        elif key == 'N':
            return self.scanheader[-2]
        elif key == 'L':
            return self.scanheader[-1]
        elif key == '@CALIB':
            output = []
            if self.scanheader is None:
                return output
            if self.scanheader[0][0:2] == '<<':
                #amptek
                try:
                    amptekCalibrationLines = []
                    amptekInCalibrationLines = False
                    for line in self.scanheader:
                        if '<<CALIBRATION>>' in line:
                            amptekInCalibrationLines = True
                            continue
                        if line.startswith('<<'):
                            amptekInCalibrationLines = False
                            continue
                        if amptekInCalibrationLines and\
                           ('LABEL' not in line):
                            amptekCalibrationLines.append(line)
                    n = len(amptekCalibrationLines)
                    if n == 0 :
                        return output
                    if n == 1:
                        #one point calibration
                        x0,y0 = 0.0, 0.0
                        values = amptekCalibrationLines[0].split()
                        x1,y1 = map(float,values)
                        gain = (y1-y0)/(x1-x0)
                        zero = y0 - gain * x0
                    elif n == 2:
                        #two point calibration
                        values = amptekCalibrationLines[0].split()
                        x0,y0 = map(float,values)
                        values = amptekCalibrationLines[1].split()
                        x1,y1 = map(float,values)
                        gain = (y1-y0)/(x1-x0)
                        zero = y0 - gain * x0
                    else:
                        x = numpy.zeros((n,), numpy.float64)
                        y = numpy.zeros((n,), numpy.float64)
                        for i in range(n):
                            values = amptekCalibrationLines[i].split()
                            x[i], y[i] = map(float,values)
                        Sxy = numpy.dot(x, y.T)
                        Sxx = numpy.dot(x, x.T)
                        Sx  = x.sum()
                        Sy  = y.sum()
                        d = n * Sxx - Sx * Sx
                        zero = (Sxx * Sy - Sx * Sxy)/d
                        gain = (n * Sxy - Sx * Sy)/d
                    output = ['#@CALIB  %g  %g  0' % (zero, gain)]
                except Exception:
                    pass
            return output
        elif key == "" or key == " ":
            return self.scanheader
        else:
            return []

    def order(self):
        return self.__order

    def number(self):
        return self.__number

    def lines(self):
        if self.scantype == 'SCAN':
            return self.rows
        else:
            return 0

    def nbmca(self):
        if self.scantype == 'SCAN':
            return 0
        else:
            return self.__cols

    def mca(self,number):
        if number <= 0:
            raise ValueError("Specfile mca numberig starts at 1")
        return self.__data[:,number-1]

class BufferedFile(object):
    def __init__(self, filename):
        f = open(filename, 'rb')
        self.__buffer = f.read()
        f.close()
        if sys.version < '3.0':
            self.__buffer=self.__buffer.replace("\r", "\n")
            self.__buffer=self.__buffer.replace("\n\n", "\n")
            self.__buffer = self.__buffer.split("\n")
        else:
            tmp = bytes("\n", 'utf-8')
            self.__buffer=self.__buffer.replace(bytes("\r", 'utf-8'), tmp)
            self.__buffer=self.__buffer.replace(bytes("\n\n", 'utf-8'), tmp)
            self.__buffer = self.__buffer.split(tmp)
        self.__currentLine = 0

    if sys.version < '3.0':
        def readline(self):
            if self.__currentLine >= len(self.__buffer):
                return ""
            line = self.__buffer[self.__currentLine] + "\n"
            self.__currentLine += 1
            return line
    else:
        def readline(self):
            if self.__currentLine >= len(self.__buffer):
                return bytes("", 'utf-8')
            line = self.__buffer[self.__currentLine] + bytes("\n", 'utf-8')
            self.__currentLine += 1
            return line

    def close(self):
        self.__currentLine = 0
        return

if __name__ == "__main__":
    filename = sys.argv[1]
    print(filename)
    sf=Specfile(filename)
    sf.list()
    print(sf[0].alllabels())
    print(dir(sf[0]))