import sys

from django.contrib.auth.models import User
from django.db.models import Q
from .models import *

class SelfField:
   pass

class RelatedField:

	def __init__(self, search):
		super().__init__()
		self.search = search
		print(self.search,'\n')

	@property
	def search_class(self):
		""" returns search class object """
		if self.search:
			if type(self.search) == str:
				try:
					return getattr(sys.modules[__name__], self.search)
				except:
					pass
			else:
				return self.search

		return None

	def get_ql(self, field, query):
		q = list()
		for item in self.search_class().get_mql(query=query):
			for key, value in item.items():
				q.append({field+'__'+key : value})

		return q

class Search:
	# returns field name of self class
	def get_fields(self):
			return [field for field in dir(self) if not callable(getattr(self, field)) and not field.startswith('__')]

	def get_ql(self, field, query):
		""" Returns generated lookups for single word in a query for a field"""
		q = list()
		for item in query.split(' '): # split query for multiple word search
			q.append({ field + '__istartswith': item })
		return q

	def get_mql(self, query):
		fields = self.get_fields()
		mql = list()
		for field in fields:
			field_obj = getattr(self, field)

			if type(field_obj) == SelfField: # calls the mql generator of self model
				mql += self.get_ql(field=field, query=query)

			elif type(field_obj) == RelatedField: # calls the mql generator of related model
				mql += field_obj.get_ql(field=field, query=query)

		return mql

	def get_full_mql(self, query):
		q = None
		for item in self.get_mql(query=query):
			if not q:
				q = Q(**item)
			else:
				q = q | Q(**item)
		return q

	def search(self,query):
		mql = self.get_full_mql(query=query)
		return self.Meta.model.objects.filter(mql).distinct()

#
#------------------------------------------------------------
#

class UserSearch(Search):
	username = SelfField()
	first_name = SelfField()
	last_name = SelfField()

	class Meta:
		model = User

class CallTypeSearch(Search):
	name = SelfField()

	class Meta:
		model = CallType

class CallPurposeSearch(Search):
	name = SelfField()

	class Meta:
		model = CallPurpose

class CallStatusSearch(Search):
	name = SelfField()

	class Meta:
		model = CallStatus

class CommunicationMediumSearch(Search):
	name = SelfField()

	class Meta:
		model = CommunicationMedium

class CallSearch(Search):
	call_type = RelatedField(search=CallTypeSearch)
	purpose = RelatedField(search=CallPurposeSearch)
	status = RelatedField(search=CallStatusSearch)
	class Meta:
		model = Call

class DealSearch(Search):
	title = SelfField()

	class Meta:
		model = Deal

class EventSearch(Search):
	print('bikash saerch type')
	location = SelfField()
	subject = SelfField()
	class Meta:
		model = Event

class FeedSearch(Search):
	employee = RelatedField(search = UserSearch)
	date = SelfField()
	deal = RelatedField(search = DealSearch )

	class Meta:
		model = Feeds


# 15 Data points test.

import bluetooth
import csv
import time
import sys
import json

def writeToFile( data1 ):

	#path1 = r"C:\Users\Jonah\OneDrive - Cornell University\Fall 2017\ECE 4760\Final Project\Server\ "
	try:
		with open('data15.csv', 'w',  newline='') as f:

			writer = csv.writer(f, delimiter=',')
			writer.writerow(data1)
	except:
		print("Data not written")

def writeToFile1( data1 ):

	#path1 = r"C:\Users\Jonah\OneDrive - Cornell University\Fall 2017\ECE 4760\Final Project\Server\ "
	while True:
		try:
			with open('data15.txt', 'r+' ) as f:

				#writer = csv.writer(f, delimiter=',')
				#writer.writerow(data1)
				f.seek(0,0)
				f.truncate()
				json.dump( data1, f)
			break
		except:
			#print("Data not written")
			break
			print( "Trying again..")

# Mac address of the HC-06:
macAddress = '98:d3:31:fd:31:27'
port = 1

# Startup bluetooth connection:
print("Attempting to connect...")
s = bluetooth.BluetoothSocket(bluetooth.RFCOMM)
s.connect((macAddress,port))
print("Connected!" )
time.sleep(0.5)

decodedData = ""
jsonData = {}
jsonData['sensorData'] = []

try:

	while True:
		time.sleep(0.02)
		data = s.recv(1024)
		#print( "Data:  " + str(data))
		val = len(data)
		if (data[val-1] == 10):

			decodedData = decodedData + data[0:val-1].decode('utf-8', "ignore")
			#print( "Decoded Data: " + str(decodedData) )
			rayData = str(decodedData).split()
			#dataRay =[]
			#print( "Data: ", str(decodedData).split())

			for i in range(15):
				decodedData = decodedData
				#l =
				#p = rayData[0][0]
				num = rayData[i][:]
				jsonData['sensorData'].append({'Sensor' : 'Sensor' + str(i+1),  'R' : int(num)})
				#dataRay.append(num)

			timin = time.time()
			writeToFile1(jsonData)
			timEnd = time.time()
			sys.stdout.flush()
			print( "Json Data:" , jsonData)
			print("\n")
			print("Duration: " + str(timEnd-timin) + "\n")
			decodedData = ""
			jsonData = {}
			json.dumps(jsonData)
			jsonData['sensorData'] = []
			time.sleep(0.05)

		else:

			decodedData = decodedData + data.decode('utf-8', "ignore")
		#print("Data", data.decode('utf-8', "ignore"))

except Exception as e:
	print( "Exception:  " + str(e))
	pass

print("Disconnected")

s.close()


'''
473. Matchsticks to Square
https://leetcode.com/problems/matchsticks-to-square/
'''
from typing import List

from test_tool import assert_value

class Solution:
    def makesquare(self, matchsticks: List[int]) -> bool:
        def backtrack(idx):
            if idx == len(matchsticks):
                return not sum(edges)

            for e_idx in range(len(edges)):
                if not edges[e_idx]:
                    continue
                if edges[e_idx] < matchsticks[idx]:
                    continue
                edges[e_idx] -= matchsticks[idx]
                if backtrack(idx + 1):
                    return True
                edges[e_idx] += matchsticks[idx]
                if edges[e_idx] == edge:
                    break
            return False

        edge, resid = divmod(sum(matchsticks), 4)
        if resid:
            return False
        matchsticks.sort(reverse=True)
        if matchsticks[0] > edge:
            return False

        edges = [edge] * 4
        return backtrack(0)


from django.conf.urls import patterns, include, url

from django.contrib import admin
admin.autodiscover()

urlpatterns = patterns('',
    # Examples:
    # url(r'^$', 'ask.views.home', name='home'),
    # url(r'^blog/', include('blog.urls')),

    url(r'^$',
        'qa.views.base_pagin'),
    url(r'^popular\/$',
        'qa.views.base_popular'),
    url(r'^question\/(?P<slug>\d+)\/$',
        'qa.views.post_details'),
    url(r'^ask\/$',
        'qa.views.add_action'),
    url(r'^answer\/$',
        'qa.views.add_answer'),
)


# -*- coding: utf-8 -*-
"""
Created on Mon May 17 18:53:35 2021
@author: rringuet
"""
from glob import glob
import numpy as np
from time import perf_counter
from os.path import basename, isfile
from netCDF4 import Dataset

swmfie_varnames = {"X": ['x', 'km'], "Y": ['y', 'km'], "Z": ['z', 'km'],
                   # coordinates are ignored
                   "Theta": ['theta', "deg"], "Psi": ['psi', "deg"],
                   "Btilt_theta": ['theta_Btilt', "deg"],
                   "Btilt_psi": ['psi_Btilt', "deg"],
                   "SigmaH": ['Sigma_H', "S"], "SigmaP": ['Sigma_P', "S"],
                   "E-Flux": ['Phi_E', "W/m**2"], "Ave-E": ['E_avg', 'eV'],
                   "JR": ["j_R", "mA/m**2"], "PHI": ["Phi", "kV"],
                   "Ex": ["E_x", "mV/m"], "Ey": ["E_y", "mV/m"],
                   "Ez": ["E_z", "mV/m"],
                   "Jx": ["j_x", "mA/m**2"], "Jy": ["j_y", "mA/m**2"],
                   "Jz": ["j_z", "mA/m**2"],
                   "Ux": ['v_x', "km/s"], "Uy": ['v_y', "km/s"],
                   "Uz": ['v_z', "km/s"],
                   "JouleHeat": ['Q_Joule', "mW/m**2"],
                   "IonNumFlux": ['Phi_nion', "1/cm**2/s"],
                   "RT 1/B": ['Binv_RT', "1/T"],
                   "RT Rho": ['rho_RT', "amu/cm**3"],
                   "RT P": ['P_RT', "Pa"],
                   "conjugate dLat": ['dLat_star', "deg"],
                   "conjugate dLon": ['dlon_star', "deg"]}

'''
Documentation variables:
(ignored, given in R_E on a unit sphere)
"X":['x','km'],"Y":['y','km'],"Z":['z','km'],
"Theta":['theta',"deg"],"Psi":['psi',"deg"],     (used as coordinates)
"Btilt_theta":['theta_Btilt',"deg"], "Btilt_psi":['psi_Btilt',"deg"]
(added directly to object for documentation purposes)
'''

def convert_all(file_dir):
    '''Convert all files independent of the others.'''

    print('NetCDF version of data not found. Converting tec files in ' +
          f'{file_dir} to netCDF.')
    ftic, nfiles = perf_counter(), 0
    files = sorted(glob(file_dir + '*.tec'))  # want YYYYMMDD
    if len(files) == 0:
        print('No original files found.')
        return

    # convert and return
    for file in files:
        if not isfile(file.replace('.tec', '.nc')):
            coords, variables, var_units, theta_Btilt, psi_Btilt = \
                _read_SWMFIE(file, verbose=True)
            cdf_filename = _toCDF(file, coords, variables, var_units,
                                  theta_Btilt, psi_Btilt)
            nfiles += 1
            print(cdf_filename, ' converted.')
    print(f'{nfiles} files in {file_dir} now converted into ' +
          f'netCDF4 files in {perf_counter()-ftic:.6f}s.')
    return None

def read_SWMFIE_header(filename):
    '''parse header for a representative file'''

    # get file data
    file_object = open(filename, 'r')
    file_contents = file_object.read()
    file_object.close()

    # sort info with TITLE keyword
    title_lines = file_contents.split('TITLE=')[1].split('VARIABLES=')[0].\
        replace('\n', ',').replace('"', '').strip().strip(',')
    title, time, BtiltDeg_str = title_lines.split(',')
    # get time into 'YYYY-MM-DD HH:mm:SS' format, clean up BtiltDeg
    dts = time[:10]+' '+''.join([time[11:].split('-')[i]+':'
                                 for i in [0, 1]])+time[11:].split('-')[2]
    BtiltDeg = BtiltDeg_str.split('=')[1].split()

    # sort info with VARIABLE keyword
    variable_lines = file_contents.split('VARIABLES=')[1].split('ZONE')[0].\
        replace('\n', ',').replace('"', '').strip().strip(',')
    variable_list = [var.strip().split(']')[0].split(' [')[0] for var in
                     variable_lines.split(',')]  # units in dict above

    # sort info with ZONE keyword 'IonN N=0000241 T=0000:20:00'
    zone_lines = file_contents.split('ZONE T=')[1].split('\n')[0].strip(
        '"').strip()
    run_type1, N1, t = zone_lines.split(' ')
    N1 = int(N1.split('=')[1].strip())
    zone_lines = file_contents.split('ZONE T=')[2].split('\n')[0].strip(
        '"').strip()
    run_type2, N2, t = zone_lines.split(' ')
    N2 = int(N2.split('=')[1].strip())

    # sort coordinate info   'I=           91  J=          181  F=POINT'
    coord_info = file_contents.split(zone_lines)[1].split('\n')[1].strip()
    text, i, text, j, f = coord_info.split()
    i, j, f = int(i), int(j), f.split('=')[1]
    # if sometimes different, need another set for this

    # determine number of lines to skip for data sections
    line_test = [f in c for c in file_contents.split('\n')]
    skip1, skip2 = np.where(np.array(line_test, dtype=int) == 1)[0]

    # determine how many lines constitute one data list
    data_lines = file_contents.split('\n')[skip1+1:]
    num_test = [len(line) for line in data_lines]  # list of lengths of lines
    ndata_lines = np.diff(np.where(np.array(num_test) == num_test[0])[0])[0]

    # cleanup and return
    del file_contents
    header = {'title': title, 'time': dts, 'BtiltDeg': BtiltDeg,
              'variable_list': variable_list, 'run_type1': run_type1, 'N1': N1,
              'run_type2': run_type2, 'N2': N2, 'i': i, 'j': j, 'f': f,
              'skip1': skip1, 'skip2': skip2, 'ndata_lines': ndata_lines}
    return header

def read_SWMFIE_data(filename, header):
    '''read only data from file and return in labeled dictionary'''

    # get data from file using skip as number of lines to skip
    file_object = open(filename, 'r')
    file_contents = file_object.readlines()
    file_object.close()

    # Get theta_Btilt value from file
    title_line = file_contents[0].replace('\n',
                                          ',').replace('"',
                                                       '').strip().strip(',')
    theta_Btilt = float(title_line.split(',')[-1].strip().split()[1])

    # collect data into an array, one for each run_type
    file_data1 = file_contents[header['skip1']+1:header['skip2']-1]
    data1 = np.array([''.join(file_data1[i:i+header['ndata_lines']]).
                      strip('\n').split() for i in range(
                          0, len(file_data1), header['ndata_lines'])],
                     dtype=float)
    file_data2 = file_contents[header['skip2']+1:]
    data2 = np.array([''.join(file_data2[i:i+header['ndata_lines']]).
                      strip('\n').split() for i in range(
                          0, len(file_data2), header['ndata_lines'])],
                     dtype=float)

    # determine correct dimension sizes from theta and psi (3 and 4)
    # combine data and return in dict
    nLatA, nLon = np.unique(data1[:, 3]).size, np.unique(data1[:, 4]).size
    data = np.concatenate((np.reshape(data1, (nLon, nLatA,
                                              len(header['variable_list']))),
                           np.reshape(data2, (nLon, nLatA,
                                              len(header['variable_list'])))),
                          axis=1)[:, ::-1, :]
    # leave since grids are identical in N/S hemispheres

    return {swmfie_varnames[header['variable_list'][i]][0]:
            np.transpose(data[:, :, i], [1, 0])
            for i in range(len(header['variable_list']))}, theta_Btilt

def _read_SWMFIE(file, verbose=False):
    '''file_dir should be the complete path to the location of the files.'''
    t0 = perf_counter()

    # establish time attributes first for file searching
    file_datestr = basename(file)[3:11]
    # string_date = 'YYYY-MM-DD'
    string_date = file_datestr[:4]+'-'+file_datestr[4:6]+'-'+file_datestr[6:8]

    # get list of variables possible in these files using first file
    file_header = read_SWMFIE_header(file)
    file_varlist = file_header['variable_list']
    gvar_list = [swmfie_varnames[key][0] for key in file_varlist if key not in
                 ['X', 'Y', 'Z', 'Theta', 'Psi', 'Btilt_theta', 'Btilt_psi']]
    # avoid returning coordinates stored elsewhere (or ignored)

    # collect coordinates and intialize data storage from first file
    data, Btilt = read_SWMFIE_data(file, file_header)
    lon, lat0 = np.unique(data['psi']), np.unique(data['theta'])-90.
    lon_ge180 = list(np.where((lon >= 180.) & (lon < 360.))[0])
    lon_le180 = list(np.where(lon <= 180.)[0])
    lon_idx = lon_ge180 + lon_le180
    lon -= 180  # shifting longitude to be in range -180 to 180
    lat0 = np.insert(lat0, np.where(lat0 == 0.)[0], 0.)
    # interpolator requires unique ascending values
    # offset two zero values by 0.0001
    lat0[np.where(lat0 == 0.)[0]] = -0.0000001, 0.0000001
    lat = np.insert(lat0, 0, -90.)
    lat = np.append(lat, 90.)  # add values for poles for wrapping
    # data is 2D (lon, lat) with time, so no height
    theta_Btilt = Btilt   # to append more values later
    psi_Btilt = 0.   # this value is always zero

    # Store variable data and units for the file.
    # transpose (lat, lon) -> (lon, lat) and wrap longitude
    var_units = {value[0]: value[-1] for key, value in swmfie_varnames.items()
                 if value[0] in gvar_list}
    variables = {key: np.array(data[key]).T[lon_idx] for key in gvar_list}

    # wrap latitude for scalar variables (all to be treated as scalars)
    for var_key in variables:
        tmp = variables[var_key]
        new_shape = list(tmp.shape)
        new_shape[1] += 2  # one value on each end
        tmp2 = np.zeros(new_shape)
        tmp2[:, 1:-1] = tmp
        # put in top values
        top = np.mean(tmp2[:, 1], axis=0)  # single value
        new_top = np.broadcast_to(top, (new_shape[0]))
        tmp2[:, 0] = new_top
        # same for bottom, reusing variable names
        top = np.mean(tmp2[:, -2], axis=0)  # single value
        new_top = np.broadcast_to(top, (new_shape[0]))
        tmp2[:, -1] = new_top
        variables[var_key] = tmp2  # store result

    coords = {'lat': lat, 'lon': lon}
    return coords, variables, var_units, theta_Btilt, psi_Btilt
# removed files from return statement

def _toCDF(file, coords, variables, var_units, theta_Btilt, psi_Btilt):
    '''Write data to a netCDF4 file.'''

    # start new wrapped output object
    cdf_filename = file.replace('.tec', '.nc')
    data_out = Dataset(cdf_filename, 'w', format='NETCDF4')
    data_out.file = file
    data_out.model = 'SWMF_IE'
    for dim in coords.keys():
        units = 'deg'
        # create dimension
        new_dim = data_out.createDimension(dim, coords[dim].size)
        # create variable
        new_var = data_out.createVariable(dim, np.float64, dim)
        new_var[:] = coords[dim]  # store data for dimension in variable
        new_var.units = units

    # copy over variables to file
    for variable_name in variables.keys():
        if len(variables[variable_name].shape) == 2:
            new_var = data_out.createVariable(variable_name, np.float64,
                                              ('lon', 'lat'))
            new_data = variables[variable_name]
        else:
            continue
        new_var[:] = new_data  # store data in variable
        new_var.units = var_units[variable_name]
    data_out.close()
    return cdf_filename


from django.shortcuts import render

from django.http.response import JsonResponse
from django.http.response import HttpResponse
from rest_framework.parsers import JSONParser
from rest_framework import status

from tutorials.models import Tutorial
from tutorials.serializers import TutorialSerializer
from rest_framework.decorators import api_view
from tutorials.utils import  get_query,get_column_names,get_max_cropORpest_state,get_top5_cropORpest, get_location
import datetime
from collections import defaultdict
from django.db.models import Q
import time
from tutorials.codeUtil2 import predict_pest

@api_view(['GET','POST'])
def chatbot_general_query(request):
    if(request.method=='GET'):
        qname= request.GET.get('crop',None)
        if(qname!=None):
            return JsonResponse( get_top5_cropORpest('crop'), safe=False)

        qname= request.GET.get('pest',None)
        if (qname!=None):
            return JsonResponse( get_top5_cropORpest('pest'), safe=False)

        else:
            return HttpResponse("HTTP_400_BAD_REQUEST", status=status.HTTP_400_BAD_REQUEST)

@api_view(['GET','POST'])
def chatbot_specific_query(request):
    if(request.method=='GET'):
        qname= request.GET.get('crop',None)
        if(qname!=None):
            return JsonResponse( get_max_cropORpest_state('crop',qname), safe=False)

        qname= request.GET.get('pest',None)
        if (qname!=None):
            return JsonResponse( get_max_cropORpest_state('pest',qname), safe=False)

        else:
            return HttpResponse("HTTP_400_BAD_REQUEST", status=status.HTTP_400_BAD_REQUEST)

@api_view(['GET','POST'])
def column_names(request):
    if(request.method=='GET'):
        list1= ['crop','pest','state_name','district_name']
        for item in list1:
            str=request.GET.get(item,None)
            if (str!=None):
                str=item
                break;
        if(str==None):
            return HttpResponse("HTTP_400_BAD_REQUEST", status=status.HTTP_400_BAD_REQUEST)
        print("column",str)
        list_l1= get_column_names(str)
        list_l1=sorted(list_l1)
        return JsonResponse(list_l1,safe=False)

@api_view(['GET','POST'])
def district_of_state(request):
    if(request.method=='GET'):
        return HttpResponse("HTTP_400_BAD_REQUEST", status=status.HTTP_400_BAD_REQUEST)

@api_view(['GET','POST'])
def prediction(request):
    if request.method=='GET':
        stName = request.GET.get('state_name', None)
        pestName = request.GET.get('pest', None)

        if (stName == None or pestName == None):
                return HttpResponse("HTTP_400_BAD_REQUEST", status=status.HTTP_400_BAD_REQUEST)
        x_axis, y_axis, upper_bnd, lower_bnd = predict_pest(pestName, stName)
        if (x_axis == -1):
                return HttpResponse("No query for this combo in dataset", status=status.HTTP_400_BAD_REQUEST)
        dict1 = {'x_axis': x_axis, 'y_axis': y_axis, 'upper_bnd': upper_bnd, 'lower_bnd': lower_bnd}
        return JsonResponse(dict1,safe=False)

@api_view(['GET','POST'])
def user_location(request):
    if request.method=='GET':
        Latitude = request.GET.get('Latitude', None)
        Longitude = request.GET.get('Longitude', None)

        if (Latitude == None or Longitude == None):
                return HttpResponse("HTTP_400_BAD_REQUEST", status=status.HTTP_400_BAD_REQUEST)
        state_name = get_location(Latitude, Longitude)
        dict1 = {"state_name": state_name}
        return JsonResponse(dict1,safe=False)

@api_view(['GET','POST'])
def home_page(request):
    if request.method=='GET':
        dict={}
        dict['date__gte']= request.GET.get('date1', None)
        dict['date__lte']= request.GET.get('date2', None)
        dict['crop']=  request.GET.get('crop', None)
        dict['pest']=  request.GET.get('pest', None)
        dict['state_name']= request.GET.get('state_name', None)
        dict['district_name']= request.GET.get('district_name', None)

        dict2={}
        flag=0
        for item in dict.items():
            print(item[1])
            if (item[1]!="all"):        #important
                dict2[item[0]]=item[1]
            if (item[1]==None):
                flag=1
                break
        print(flag)
        if(flag==1):
            return HttpResponse("HTTP_400_BAD_REQUEST", status=status.HTTP_400_BAD_REQUEST)
        dict= get_query(dict2)
        return JsonResponse(dict,safe=False)

def dt_funct():
    dict=defaultdict(int)
    return dict

@api_view(['GET', 'POST'])
def tutorial_list_1(request):
    if request.method == 'GET':
        print("///////////////////////////////")
        check()
        title = request.GET.get('date1', None)
        title1 = request.GET.get('date2', None)
        print(title1)
        if title is not None:
            print("true")
            query_start_time = time.time()

            tutorials = Tutorial.objects.filter(Q(date__gte=title)&Q(date__lte=title1)).values_list('state_name','district_name','count')
            cnt1=0
            cnt2=0
            dict=defaultdict(dt_funct)
            for item in tutorials:
                print(item)
                dict[item[0]][item[1]]=dict[item[0]][item[1]]+item[2]
                cnt1=cnt1+1
                cnt2=cnt2+item[2]
            print("---Finished querying db in %s seconds ---" % (time.time() - query_start_time))
            print(cnt1)
            print(cnt2)
            return JsonResponse(dict,safe=False)
        else:
            tutorials= Tutorial.objects.all()
        tutorials_serializer = TutorialSerializer(tutorials,many=True)
        print(tutorials_serializer.data[0])
        return JsonResponse(tutorials_serializer,safe=False)

    if request.method == 'POST':
        tutorial_data = JSONParser().parse(request)
        print(tutorial_data['date'])
        tutorial_data['date'] = datetime.datetime.strptime(tutorial_data['date']+"T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ")
        print(tutorial_data['date'])
        tutorial_serializer = TutorialSerializer(data=tutorial_data)
        if tutorial_serializer.is_valid():
            print("inside")
            #return JsonResponse(tutorial_serializer.validated_data)
            return HttpResponse(tutorial_serializer.data['date'])
        else:
            print(tutorial_serializer.errors)
            return HttpResponse("yo mannn  ")

@api_view(['GET', 'PUT', 'DELETE'])
def tutorial_detail(request, pk):
    # find tutorial by pk (id)
    try:
        tutorial = Tutorial.objects.get(pk=pk)
    except Tutorial.DoesNotExist:
        return JsonResponse({'message': 'The tutorial does not exist'}, status=status.HTTP_404_NOT_FOUND)

    # GET / PUT / DELETE tutorial

@api_view(['GET'])
def tutorial_list_published(request):
    return JsonResponse({'message': 'The tutorial does not exist'}, status=status.HTTP_404_NOT_FOUND)

    # GET all published tutorials


import numpy as np
import pickle
from random import randrange
import networkx as nx

class Board:

    def get_page_rank(self):
        G = nx.from_numpy_matrix(self.board_2d)
        pr = nx.pagerank(G,0.001)
        return np.fromiter(pr.values(), dtype=float)

    def init_player_pos(self,num_players):

        init_stops = [13,26,29,34,50,53,91,94,103,112,117,132,138,141,155,174,197,198]
        self.player_pos = [0]*num_players
        for player_index in range(len(self.player_pos)):
            self.player_pos[player_index] = init_stops[randrange(len(init_stops))] -1
            init_stops.remove(self.player_pos[player_index] + 1)
            print("player%s initialized at position:  %s" %(player_index, self.player_pos[player_index] + 1))

    def set_player_pos(self,player,pos):
        print("player %s changed from pos %s \t  to %s " %(player, self.player_pos[player] + 1, pos + 1) )
        self.player_pos[player] = int(pos)

    def __init__(self, num_players):
        self.board = pickle.load(open("rides.pkl","rb"))
        self.N = self.board.shape[1]
        self.board_2d = self.board.sum(axis=0) > 0
        self.pagerank = self.get_page_rank()
        self.init_player_pos(num_players)


import subprocess
from flask import Flask, request

from model_provider import ModelProvider

app = Flask(__name__)

model_provider = ModelProvider()

@app.route('/')
def hello():
    command = "`echo hostname`"
    hostname = subprocess.check_output(['bash', '-c', command])
    return f'hello from docker container {hostname}'

@app.route('/api/model', methods=['POST'])
def set_model():
    if not request.data:
        return 'Empty content', 400
    if model_provider.save_model(str(request.data, 'utf-8')):
        return 'Model was received', 200
    return 'Error while submitting model', 500

@app.route('/api/prediction', methods=['POST'])
def prediction():
    log_dic = request.get_json()
    data = log_dic.get('log', None)
    if not data:
        return 'Invalid parameter', 400
    res = model_provider.get_prediction(data)
    if not res:
        return 'Prediction was failed, try submit a valid model', 500
    return {'result': res}

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=5000)


import os
from django.conf import settings

ALLOWED_HOSTS = ['b802-01.fsv.cvut.cz']

# Application definition

INSTALLED_APPS = settings.INSTALLED_APPS + ['users.apps.UsersConfig', 'django_python3_ldap', ]

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [os.path.join(settings.BASE_DIR, 'templates')],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

# define global static
STATICFILES_DIRS = (
    os.path.join(settings.BASE_DIR, 'static'),
)

LOGIN_REDIRECT_URL = 'home'
LOGOUT_REDIRECT_URL = 'home'

### CUSTOM USER MODEL ###
AUTH_USER_MODEL = 'users.CustomUser'

### LDAP AUTHENTICATION ###
AUTHENTICATION_BACKENDS = (
    'django_python3_ldap.auth.LDAPBackend',
#    'django.contrib.auth.backends.ModelBackend',
)

# The URL of the LDAP server.
LDAP_AUTH_URL = "ldap://127.0.0.1:389"

# Initiate TLS on connection
LDAP_AUTH_USE_TLS = True

# The LDAP search base for looking up users.
LDAP_AUTH_SEARCH_BASE = "ou=people,dc=gis,dc=lab"

# User model fields mapped to the LDAP attributes that represent them.
LDAP_AUTH_USER_FIELDS = {
    "username": "uid",
    "first_name": "givenName",
    "last_name": "sn",
    "email": "mail",
    "description": "description",
}

# A tuple of django model fields used to uniquely identify a user.
LDAP_AUTH_USER_LOOKUP_FIELDS = ("username",)

LDAP_AUTH_SYNC_USER_RELATIONS =   "web_console_project.ldap_auth.custom_sync_user_relations"

### LOGGING ###
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'handlers': {
        'console': {
            'level': 'INFO',
            'class': 'logging.StreamHandler'
        },
    },
    'loggers': {
        'django': {
            'handlers': ['console'],
            'level': 'INFO',
            'propagate': True
        }
    },
    "loggers": {
        "django_python3_ldap": {
            "handlers": ["console"],
            "level": "INFO",
        },
    }
}


import torch
from torch import cat, einsum, argmax, unsqueeze, FloatTensor
import torch.nn as nn
from torch.nn import init
from scipy.ndimage import zoom
from torch.nn.functional import softmax
from numpy import array, float32, expand_dims, average, sqrt, argwhere, zeros, shape, pad
from skimage import measure, morphology, io
import skimage
from skimage.color import rgba2rgb, rgb2gray
from os import makedirs
import matplotlib.pyplot as plt

class unetConv2(nn.Module):
    def __init__(self, in_size, out_size, is_batchnorm, n=2, ks=3, stride=1, padding=1):
        super(unetConv2, self).__init__()
        self.n = n
        self.ks = ks
        self.stride = stride
        self.padding = padding
        s = stride
        p = padding
        if is_batchnorm:
            for i in range(1, n + 1):
                conv = nn.Sequential(nn.Conv2d(in_size, out_size, ks, s, p),
                                     nn.BatchNorm2d(out_size),
                                     nn.ReLU(inplace=True), )
                setattr(self, 'conv%d' % i, conv)
                in_size = out_size

        else:
            for i in range(1, n + 1):
                conv = nn.Sequential(nn.Conv2d(in_size, out_size, ks, s, p),
                                     nn.ReLU(inplace=True), )
                setattr(self, 'conv%d' % i, conv)
                in_size = out_size

        for m in self.children():
            init_weights(m, init_type='kaiming')

    def forward(self, inputs):
        x = inputs
        for i in range(1, self.n + 1):
            conv = getattr(self, 'conv%d' % i)
            x = conv(x)
        return x

class unetUp(nn.Module):
    def __init__(self, in_size, out_size, is_deconv, n_concat=2):
        super(unetUp, self).__init__()
        self.conv = unetConv2(out_size*2, out_size, False)
        if is_deconv:
            self.up = nn.ConvTranspose2d(in_size, out_size, kernel_size=4, stride=2, padding=1)
        else:
            self.up = nn.UpsamplingBilinear2d(scale_factor=2)
        for m in self.children():
            if m.__class__.__name__.find('unetConv2') != -1: continue
            init_weights(m, init_type='kaiming')

    def forward(self, inputs0, *input):
        outputs0 = self.up(inputs0)
        for i in range(len(input)):
            outputs0 = cat([outputs0, input[i]], 1)
        return self.conv(outputs0)

def weights_init_normal(m):
    classname = m.__class__.__name__
    if classname.find('Conv') != -1:
        init.normal_(m.weight.data, 0.0, 0.02)
    elif classname.find('Linear') != -1:
        init.normal_(m.weight.data, 0.0, 0.02)
    elif classname.find('BatchNorm') != -1:
        init.normal_(m.weight.data, 1.0, 0.02)
        init.constant_(m.bias.data, 0.0)

def weights_init_xavier(m):
    classname = m.__class__.__name__
    if classname.find('Conv') != -1:
        init.xavier_normal_(m.weight.data, gain=1)
    elif classname.find('Linear') != -1:
        init.xavier_normal_(m.weight.data, gain=1)
    elif classname.find('BatchNorm') != -1:
        init.normal_(m.weight.data, 1.0, 0.02)
        init.constant_(m.bias.data, 0.0)

def weights_init_kaiming(m):
    classname = m.__class__.__name__
    if classname.find('Conv') != -1:
        init.kaiming_normal_(m.weight.data, a=0, mode='fan_in')
    elif classname.find('Linear') != -1:
        init.kaiming_normal_(m.weight.data, a=0, mode='fan_in')
    elif classname.find('BatchNorm') != -1:
        init.normal_(m.weight.data, 1.0, 0.02)
        init.constant_(m.bias.data, 0.0)

def weights_init_orthogonal(m):
    classname = m.__class__.__name__
    if classname.find('Conv') != -1:
        init.orthogonal_(m.weight.data, gain=1)
    elif classname.find('Linear') != -1:
        init.orthogonal_(m.weight.data, gain=1)
    elif classname.find('BatchNorm') != -1:
        init.normal_(m.weight.data, 1.0, 0.02)
        init.constant_(m.bias.data, 0.0)

def init_weights(net, init_type='normal'):
    if init_type == 'normal':
        net.apply(weights_init_normal)
    elif init_type == 'xavier':
        net.apply(weights_init_xavier)
    elif init_type == 'kaiming':
        net.apply(weights_init_kaiming)
    elif init_type == 'orthogonal':
        net.apply(weights_init_orthogonal)
    else:
        raise NotImplementedError('initialization method [%s] is not implemented' % init_type)

class UNet(nn.Module):
    def __init__(self, in_channels=1, n_classes=3, feature_scale=4, is_deconv=True, is_batchnorm=True):
        super(UNet, self).__init__()
        self.is_deconv = is_deconv
        self.in_channels = in_channels
        self.is_batchnorm = is_batchnorm
        self.feature_scale = feature_scale
        filters = [64, 128, 256, 512, 1024]

        # downsampling
        self.conv1 = unetConv2(self.in_channels, filters[0], self.is_batchnorm)
        self.maxpool1 = nn.MaxPool2d(kernel_size=2)
        self.conv2 = unetConv2(filters[0], filters[1], self.is_batchnorm)
        self.maxpool2 = nn.MaxPool2d(kernel_size=2)
        self.conv3 = unetConv2(filters[1], filters[2], self.is_batchnorm)
        self.maxpool3 = nn.MaxPool2d(kernel_size=2)
        self.conv4 = unetConv2(filters[2], filters[3], self.is_batchnorm)
        self.maxpool4 = nn.MaxPool2d(kernel_size=2)
        self.center = unetConv2(filters[3], filters[4], self.is_batchnorm)

        # upsampling
        self.up_concat4 = unetUp(filters[4], filters[3], self.is_deconv)
        self.up_concat3 = unetUp(filters[3], filters[2], self.is_deconv)
        self.up_concat2 = unetUp(filters[2], filters[1], self.is_deconv)
        self.up_concat1 = unetUp(filters[1], filters[0], self.is_deconv)
        self.outconv1 = nn.Conv2d(filters[0], n_classes, 1, padding=0)

        # initialise weights
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                init_weights(m, init_type='kaiming')
            elif isinstance(m, nn.BatchNorm2d):
                init_weights(m, init_type='kaiming')

    def dotProduct(self,seg,cls):
        B, N, H, W = seg.size()
        seg = seg.view(B, N, H * W)
        final = einsum("ijk,ij->ijk", [seg, cls])
        final = final.view(B, N, H, W)
        return final

    def forward(self, inputs):
        conv1 = self.conv1(inputs)  # 16*512*1024
        maxpool1 = self.maxpool1(conv1)  # 16*256*512

        conv2 = self.conv2(maxpool1)  # 32*256*512
        maxpool2 = self.maxpool2(conv2)  # 32*128*256

        conv3 = self.conv3(maxpool2)  # 64*128*256
        maxpool3 = self.maxpool3(conv3)  # 64*64*128

        conv4 = self.conv4(maxpool3)  # 128*64*128
        maxpool4 = self.maxpool4(conv4)  # 128*32*64

        center = self.center(maxpool4)  # 256*32*64

        up4 = self.up_concat4(center, conv4)  # 128*64*128
        up3 = self.up_concat3(up4, conv3)  # 64*128*256
        up2 = self.up_concat2(up3, conv2)  # 32*256*512
        up1 = self.up_concat1(up2, conv1)  # 16*512*1024
        d1 = self.outconv1(up1)  # 256
        return d1

class pub(nn.Module):

    def __init__(self, in_channels, out_channels, batch_norm=True):
        super(pub, self).__init__()
        inter_channels = int(in_channels/2 if in_channels > out_channels else out_channels/2)
        layers = [
                    nn.Conv3d(in_channels, inter_channels, 3, stride=1, padding=1),
                    nn.ReLU(True),
                    nn.Conv3d(inter_channels, out_channels, 3, stride=1, padding=1),
                    nn.ReLU(True)
                 ]
        if batch_norm:
            layers.insert(1, nn.BatchNorm3d(inter_channels))
            layers.insert(len(layers)-1, nn.BatchNorm3d(out_channels))
        self.pub = nn.Sequential(*layers)

    def forward(self, x):
        return self.pub(x)

class unet3dDown(nn.Module):

    def __init__(self, in_channels, out_channels, batch_norm=True):
        super(unet3dDown, self).__init__()
        self.pub = pub(in_channels, out_channels, batch_norm)
        self.pool = nn.MaxPool3d(2, stride=2)

    def forward(self, x):
        x = self.pool(x)
        x = self.pub(x)
        return x

class unet3dUp(nn.Module):
    def __init__(self, in_channels, out_channels, batch_norm=True, sample=True):
        super(unet3dUp, self).__init__()
        self.pub = pub(int(in_channels/2+in_channels), out_channels, batch_norm)
        if sample:
            self.sample = nn.Upsample(scale_factor=2, mode='nearest')
        else:
            self.sample = nn.ConvTranspose3d(in_channels, in_channels, 2, stride=2)

    def forward(self, x, x1):
        x = self.sample(x)
        x = torch.cat((x, x1), dim=1)
        x = self.pub(x)
        return x

class unet3d(nn.Module):
    def __init__(self, init_channels=1, class_nums=3, batch_norm=True, sample=True):
        super(unet3d, self).__init__()
        self.down1 = pub(init_channels, 64, batch_norm)
        self.down2 = unet3dDown(64, 128, batch_norm)
        self.down3 = unet3dDown(128, 256, batch_norm)
        self.down4 = unet3dDown(256, 512, batch_norm)
        self.up3 = unet3dUp(512, 256, batch_norm, sample)
        self.up2 = unet3dUp(256, 128, batch_norm, sample)
        self.up1 = unet3dUp(128, 64, batch_norm, sample)
        self.con_last = nn.Conv3d(64, class_nums, 1)
        self.sigmoid = nn.Sigmoid()

    def forward(self, x):
        x1 = self.down1(x)
        x2 = self.down2(x1)
        x3 = self.down3(x2)
        x4 = self.down4(x3)
        x = self.up3(x4, x3)
        x = self.up2(x, x2)
        x = self.up1(x, x1)
        x = self.con_last(x)
        return x

    def _initialize_weights(self):
        for m in self.modules():
            if isinstance(m, nn.Conv3d):
                nn.init.kaiming_uniform(m.weight.data)
                if m.bias is not None:
                    m.bias.data.zero_()
            elif isinstance(m, nn.BatchNorm3d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()

def common_mitotic_cell_regions_3d(img):
    img_shape = shape(img)
    images_cells = zeros(img_shape)
    images_mitotic_cells = zeros(img_shape)
    locations = argwhere(img == 2)
    locations_mitotic = argwhere(img == 1)
    for temp in locations:
        images_cells[temp[0], temp[1], temp[2]] = 1
    for temp in locations_mitotic:
        images_mitotic_cells[temp[0], temp[1], temp[2]] = 1
    return images_cells, images_mitotic_cells

def measure_region(mask, connectivity=1):
    label_img, num = measure.label(mask, return_num=True, connectivity=connectivity)
    return label_img, num

def distance_between_points(point1, point2):
    if len(point2) != len(point1):
        print('Error, the length of two points is different')
    else:
        temp = 0
        for i in range(len(point1)):
            d_j = point1[i] - point2[i]
            temp += float(d_j ** 2)
        return sqrt(temp)

def remove_overlapped_cells(all_cells, threshold=4):
    tracked_index = []
    new_all_cells = []
    for i, cell in enumerate(all_cells):
        if i not in tracked_index:
            temp_i = []
            temp_i.append(cell)
            for j, cell_1 in enumerate(all_cells):
                if j not in tracked_index:
                    if i != j:
                        dist = distance_between_points(cell, cell_1)
                        if dist <= threshold:
                            temp_i.append(cell_1)
                            tracked_index.append(j)
            temp_i = average(array(temp_i), 0)
            new_all_cells.append([temp_i[0], temp_i[1], temp_i[2]])
    return new_all_cells

# def remove_overlapped_cells_x(all_cells, threshold=3, threshold2=5):
#     tracked_index = []
#     new_all_cells = []
#     for i, cell in enumerate(all_cells):
#         if i not in tracked_index:
#             temp_i = []
#             temp_i.append(cell)
#             for j, cell_1 in enumerate(all_cells):
#                 if j not in tracked_index:
#                     if i != j:
#                         dist = abs(cell[1] - cell_1[1])
#                         if dist <= threshold:
#                             if abs(cell[2] - cell_1[2]) < threshold2:
#                                 temp_i.append(cell_1)
#                                 tracked_index.append(j)
#             temp_i = average(array(temp_i), 0)
#             new_all_cells.append([temp_i[0], temp_i[1], temp_i[2]])
#     return new_all_cells

def detect_middle_slice(model, path, idx, slice, device):
    makedirs(f'{path}/images', exist_ok=True)
    slice = slice[0:848, 0:480]
    plt.imsave(r'{0}/images/{1}_middle.png'.format(path, idx), slice)
    data = io.imread(r'{0}/images/{1}_middle.png'.format(path, idx))
    data = rgba2rgb(data)
    data = rgb2gray(data)
    data = expand_dims(data, -1)
    data = data.transpose((2, 0, 1))
    data = expand_dims(data, 0)
    slice_torch = torch.FloatTensor(data)
    pred = model(slice_torch.to(device))
    pred_softmax = softmax(pred, dim=1)
    pred_mask = argmax(pred_softmax, dim=1)
    pred_mask_numpy = pred_mask[0].cpu().data.numpy()
    plt.imsave(r'{0}/images/{1}_middle_mask.png'.format(path, idx), pred_mask_numpy)
    pred_mask_final = morphology.remove_small_objects(array(pred_mask_numpy, dtype=bool), 0.5)
    return pred_mask_final

def pad_image(volume, height=1024, width=512):
    if shape(volume)[1] != height or shape(volume)[2] != width:
        if shape(volume)[1] != height:
            if shape(volume)[2] != width:
                padding_h = int((height-shape(volume)[1])/2)
                padding_x = int((width -shape(volume)[2]) / 2)
                volume = pad(volume, ((0, 0),
                                      (padding_h, height-shape(volume)[1]-padding_h),
                                      (padding_x, width-shape(volume)[2]-padding_x)),
                             'constant')
            else:
                padding_h = int((height - shape(volume)[1]) / 2)
                padding_x = 0
                volume = pad(volume, ((0, 0),
                                      (padding_h, height - shape(volume)[1] - padding_h),
                                      (0, 0)),
                             'constant')
        else:
            padding_x = int((width - shape(volume)[2]) / 2)
            padding_h = 0
            volume = pad(volume, ((0, 0),
                                  (0, 0),
                                  (padding_x, width - shape(volume)[2] - padding_x)),
                         'constant')
    else:
        padding_h = 0
        padding_x = 0

    return volume, padding_h, padding_x

def pred_img(data, model, device):
    data = expand_dims(data, -1)
    data = data.transpose((2, 0, 1))
    data = expand_dims(data, 0)
    slice_torch = torch.FloatTensor(data)
    pred = model(slice_torch.to(device))
    pred_softmax = softmax(pred, dim=1)
    pred_mask = argmax(pred_softmax, dim=1)
    return pred_mask[0].cpu().data.numpy()

def test_model(model, path, idx, imgs, device, whether_transpose):
    h_ratio = shape(imgs)[-2] // 1024
    w_ratio = shape(imgs)[-1] // 512
    temp_masks = []
    common_centroids = []
    mitotic_centroids = []

    if h_ratio > 1 and w_ratio > 1:
        volume = imgs[idx][:, 0:h_ratio * 1024, 0:w_ratio * 512]
        padding_h = padding_x = 0
        for k in range(len(volume)):
            for i in range(h_ratio):
                for j in range(w_ratio):
                    temp = volume[k][i * 1024:(i + 1) * 1024, j * 512:(j + 1) * 512]
                    temp = temp / 255
                    temp_mask = pred_img(temp, model, device)
                    temp_masks.append(temp_mask)
        pred_masks = zeros((len(volume), h_ratio * 1024, w_ratio * 512))
        for k in range(len(volume)):
            for i in range(h_ratio):
                for j in range(w_ratio):
                    pred_masks[k, i * 1024:(i + 1) * 1024, j * 512:(j + 1) * 512] = temp_masks[k * h_ratio * w_ratio + i * w_ratio + j]
    elif h_ratio > 1 and w_ratio <=1:
        volume = imgs[idx][:, 0:h_ratio * 1024, 0:512]
        volume, padding_h, padding_x = pad_image(volume, h_ratio*1024, 512)
        for k in range(len(volume)):
            for i in range(h_ratio):
                temp = volume[k][i * 1024:(i + 1) * 1024, 0:512]
                temp = temp / 255
                temp_mask = pred_img(temp, model, device)
                temp_masks.append(temp_mask)
        pred_masks = zeros((len(volume), h_ratio * 1024, 512))
        for k in range(len(volume)):
            for i in range(h_ratio):
                pred_masks[k, i * 1024:(i + 1) * 1024, 512] = temp_masks[k * h_ratio+i]
    elif h_ratio<=1 and w_ratio >1:
        volume = imgs[idx][:, 0:1024, 0:w_ratio * 512]
        volume, padding_h, padding_x = pad_image(volume, 1024, w_ratio * 512)
        for k in range(len(volume)):
            for j in range(w_ratio):
                temp = volume[k][0:1024, j * 512:(j + 1) * 512]
                temp = temp / 255
                temp_mask = pred_img(temp, model, device)
                temp_masks.append(temp_mask)
        pred_masks = zeros((len(volume), 1024, w_ratio * 512))
        for k in range(len(volume)):
            for j in range(w_ratio):
                pred_masks[k, 1024,  j * 512:(j + 1) * 512] = temp_masks[k * w_ratio + j]
    else:
        volume = imgs[idx][:, 0:1024, 0:512]
        volume, padding_h, padding_x = pad_image(volume)
        for k in range(len(volume)):
            temp = volume[k]
            temp = temp/255
            temp_mask = pred_img(temp, model, device)
            temp_masks.append(temp_mask)
        pred_masks = zeros((len(volume), 1024, 512))
        for k in range(len(volume)):
            pred_masks[k] = temp_masks[k]

    p_c, p_m = common_mitotic_cell_regions_3d(pred_masks)
    p_c = morphology.remove_small_objects(array(p_c, dtype=bool), 1)  # 3.5
    p_m = morphology.remove_small_objects(array(p_m, dtype=bool), 3)  # 50
    region_c, _ = measure_region(p_c)
    region_m, _ = measure_region(p_m)
    common_regions = measure.regionprops(region_c)
    mitotic_regions = measure.regionprops(region_m)

    for props in common_regions:
        z0, y0, x0 = props.centroid
        if whether_transpose:
            common_centroids.append([z0, x0 - padding_x, y0 - padding_h])
        else:
            common_centroids.append([z0, y0 - padding_h, x0 - padding_x])

    for props in mitotic_regions:
        z0, y0, x0 = props.centroid
        if whether_transpose:
            mitotic_centroids.append([z0, x0 - padding_x, y0 - padding_h])
        else:
            mitotic_centroids.append([z0, y0-padding_h, x0-padding_x])
    return remove_overlapped_cells(common_centroids), remove_overlapped_cells(mitotic_centroids)

def test_3d(model, path, idx, imgs, device, whether_transpose):
    h_ratio = shape(imgs)[-2] // 1024
    w_ratio = shape(imgs)[-1] // 512
    temp_masks = []
    common_centroids = []
    mitotic_centroids = []

    if h_ratio > 1 and w_ratio > 1:
        volume = imgs[idx][:, 0:h_ratio * 1024, 0:w_ratio * 512]
        padding_h = padding_x = 0
        for i in range(h_ratio):
            for j in range(w_ratio):
                temp = volume[:, i * 1024:(i + 1) * 1024, j * 512:(j + 1) * 512]
                temp = temp / 255
                temp_mask = pred_img(temp, model, device)
                temp_masks.append(temp_mask)
        pred_masks = zeros((len(volume), h_ratio * 1024, w_ratio * 512))
        for k in range(len(volume)):
            for i in range(h_ratio):
                for j in range(w_ratio):
                    pred_masks[k, i * 1024:(i + 1) * 1024, j * 512:(j + 1) * 512] = temp_masks[
                        k * h_ratio * w_ratio + i * w_ratio + j]
    elif h_ratio > 1 and w_ratio <= 1:
        volume = imgs[idx][:, 0:h_ratio * 1024, 0:512]
        volume, padding_h, padding_x = pad_image(volume, h_ratio * 1024, 512)
        for k in range(len(volume)):
            for i in range(h_ratio):
                temp = volume[k][i * 1024:(i + 1) * 1024, 0:512]
                temp = temp / 255
                temp_mask = pred_img(temp, model, device)
                temp_masks.append(temp_mask)
        pred_masks = zeros((len(volume), h_ratio * 1024, 512))
        for k in range(len(volume)):
            for i in range(h_ratio):
                pred_masks[k, i * 1024:(i + 1) * 1024, 512] = temp_masks[k * h_ratio + i]

    if shape(imgs)[-2] > 1100 or shape(imgs)[-1] > 600:
        if shape(imgs)[-2] > 1100 and shape(imgs)[-1] > 600:
            resize_parameter_h = shape(imgs)[-2]/1100
            resize_parameter_w = shape(imgs)[-1]/600
            crop_h = int(1100 * resize_parameter_h)
            crop_w = int(600 * resize_parameter_w)
            volume = imgs[idx][:, 0:crop_h, 0:crop_w]
            volume = zoom(volume, [1, 1/resize_parameter_h, 1/resize_parameter_w])
            volume = volume[:, 0:1024, 0:512]
        elif shape(imgs)[-2] > 1024:
            resize_parameter_h = shape(imgs)[-2] / 1024
            resize_parameter_w = 1
            crop_h = int(1024 * resize_parameter_h)
            volume = imgs[idx][:, 0:crop_h, :]
            volume = zoom(volume, [1, 1 / resize_parameter_h, 1])
            volume = volume[:, 0:1024, 0:512]
        else:
            resize_parameter_h = 1
            resize_parameter_w = shape(imgs)[-1] / 512
            crop_w = int(512 * resize_parameter_w)
            volume = imgs[idx][:, :, 0:crop_w]
            volume = zoom(volume, [1, 1, 1 / resize_parameter_w])
            volume = volume[:, 0:1024, 0:512]
    else:
        volume = imgs[idx][:, 0:1024, 0:512]
        resize_parameter_h = 1
        resize_parameter_w = 1

    if shape(volume)[1] != 1024 or shape(volume)[2] != 512:
        if shape(volume)[1] != 1024:
            if shape(volume)[2] != 512:
                padding_h = int((1024 - shape(volume)[1]) / 2)
                padding_x = int((512 - shape(volume)[2]) / 2)
                volume = pad(volume, ((0, 0),
                                      (padding_h, 1024 - shape(volume)[1] - padding_h),
                                      (padding_x, 512 - shape(volume)[2] - padding_x)),
                             'constant')
            else:
                padding_h = int((1024 - shape(volume)[1]) / 2)
                padding_x = 0
                volume = pad(volume, ((0, 0),
                                      (padding_h, 1024 - shape(volume)[1] - padding_h),
                                      (0, 0)),
                             'constant')
        else:
            padding_x = int((512 - shape(volume)[2]) / 2)
            padding_h = 0
            volume = pad(volume, ((0, 0),
                                  (0, 0),
                                  (padding_x, 512 - shape(volume)[2] - padding_x)),
                         'constant')
    else:
        padding_h = 0
        padding_x = 0

    z_number = len(volume)
    start_slice = int((z_number-24)/2)+1
    end_slice = int((z_number-24)/2)+25
    volume = volume[start_slice:end_slice, :, :]
    volume = zoom(volume, (1, 0.5, 0.5))
    volume_torch = torch.unsqueeze(torch.unsqueeze(torch.FloatTensor(volume / 255), 0), 0)
    pred = model(volume_torch.to(device))
    pred_softmax = softmax(pred, dim=1)
    pred_mask = torch.argmax(pred_softmax, dim=1)
    pred_mask_cpu = pred_mask[0].cpu().data.numpy()

    p_c, p_m = common_mitotic_cell_regions_3d(pred_mask_cpu)
    p_c = morphology.remove_small_objects(array(p_c, dtype=bool), 1)  # 3.5
    p_m = morphology.remove_small_objects(array(p_m, dtype=bool), 6)  # 50
    region_c, _ = measure_region(p_c)
    region_m, _ = measure_region(p_m)
    common_regions = measure.regionprops(region_c)
    mitotic_regions = measure.regionprops(region_m)

    mitotic_centroids = []
    common_centroids = []
    for props in common_regions:
        z0, y0, x0 = props.centroid
        common_centroids.append([z0 + start_slice, (y0 * 2 - padding_h)*resize_parameter_h, (x0 * 2 -padding_x)*resize_parameter_w])

    for props in mitotic_regions:
        z0, y0, x0 = props.centroid
        mitotic_centroids.append([z0 + start_slice, (y0 * 2 - padding_h)*resize_parameter_h, (x0 * 2 - padding_x)*resize_parameter_w])
    return common_centroids, mitotic_centroids


"""
Flavor fixtures.

@author: schipiga@mirantis.com
"""

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import pytest

from mcp_tests.nova.steps import FlavorSteps
from mcp_tests.utils import generate_ids

__all__ = [
    'create_flavor',
    'flavor',
    'flavor_steps'
]

@pytest.fixture
def flavor_steps(nova_client):
    """Fixture to get flavor steps."""
    return FlavorSteps(nova_client.flavors)

@pytest.yield_fixture
def create_flavor(flavor_steps):
    """Fixture to create flavor with options.

    Can be called several times during test.
    """
    flavors = []

    def _create_flavor(flavor_name):
        flavor = flavor_steps.create_flavor(flavor_name)
        flavors.append(flavor)
        return flavor

    yield _create_flavor

    for flavor in flavors:
        flavor_steps.delete_flavor(flavor)

@pytest.fixture
def flavor(create_flavor):
    """Fixture to create flavor with default options before test."""
    flavor_name = next(generate_ids('flavor'))
    return create_flavor(flavor_name)


import os
import sys

from PyQt5.QtWidgets import QApplication, QWidget
from PyQt5 import QtCore
from PyQt5 import QtGui

current_directory = os.path.dirname(os.path.abspath(__file__))
parent_directory = os.path.abspath(os.path.join(current_directory, os.pardir))
resource_directory = os.path.join(current_directory, "resources")
sys.path.append(os.path.join(parent_directory, "core"))

from game import Game
from unit import Directions

class Aquarium(QWidget):
    score_changed = QtCore.pyqtSignal()

    def __init__(self, parent):
        super(Aquarium, self).__init__()
        self.game = Game((self.size().width(), self.size().height()))
        self.score = self.game.score
        self.score_changed.emit()
        self.paused = False
        self.load_pictures()
        self.timer = QtCore.QTimer(self)
        self.timer.timeout.connect(self.action)
        self.timer.start(40)

    def resizeEvent(self, event):
        self.game.constrain((event.size().width(), event.size().height()))

    def paintEvent(self, event):
        canvas = QtGui.QPainter()
        canvas.begin(self)
        canvas.setPen(QtCore.Qt.NoPen)
        canvas.drawPixmap(0, 0, self.background.scaled(self.size().width(),
                          self.size().height()))
        for alien in self.game.aliens:
            self.draw_alien(canvas, alien)
        for fish in self.game.fishes:
            self.draw_fish(canvas, fish)
        for coin in self.game.coins:
            self.draw_coin(canvas, coin)
        for food in self.game.food:
            self.draw_food(canvas, food)

    @QtCore.pyqtSlot()
    def spawn_fish(self):
        self.game.spawn_fish()

    @QtCore.pyqtSlot()
    def upgrade_weapon(self):
        self.game.upgrade_weapon()

    @QtCore.pyqtSlot()
    def upgrade_food(self):
        self.game.upgrade_food()

    @QtCore.pyqtSlot()
    def pause(self):
        self.paused = True

    @QtCore.pyqtSlot()
    def unpause(self):
        self.paused = False

    def mousePressEvent(self, event):
        if event.button() == QtCore.Qt.LeftButton:
            self.game.mouse_press(event.x(), event.y())

    def load_pictures(self):
        # Alien
        self.alien_images = {
            'lion': QtGui.QPixmap(
                os.path.join(resource_directory, "alien.png")),
            'blue': QtGui.QPixmap(
                os.path.join(resource_directory, "alien2.png"))}
        self.alien_images_mirrored = {}.fromkeys(self.alien_images)
        self.fill_mirrored(self.alien_images, self.alien_images_mirrored)

        # Fish
        self.fish_images = {
            'swim': QtGui.QPixmap(
                os.path.join(resource_directory, "fish_swim.png")),
            'eat': QtGui.QPixmap(
                os.path.join(resource_directory, "fish_eat.png")),
            'turn': QtGui.QPixmap(
                os.path.join(resource_directory, "fish_turn.png")),
            'hungry_die': QtGui.QPixmap(
                os.path.join(resource_directory, "fish_die.png")),
            'hungry_swim': QtGui.QPixmap(
                os.path.join(resource_directory, "hungry_swim.png")),
            'hungry_eat': QtGui.QPixmap(
                os.path.join(resource_directory, "hungry_eat.png")),
            'hungry_turn': QtGui.QPixmap(
                os.path.join(resource_directory, "hungry_turn.png"))}
        self.fish_images_mirrored = {}.fromkeys(self.fish_images)
        self.fill_mirrored(self.fish_images, self.fish_images_mirrored)

        # Food
        self.food_image = QtGui.QPixmap(
            os.path.join(resource_directory, "food.png"))

        # Coin
        self.coin_image = QtGui.QPixmap(
            os.path.join(resource_directory, "coin.png"))

        # Background
        self.background = QtGui.QPixmap(
            os.path.join(resource_directory, "background.png"))

    def fill_mirrored(self, normal_images, mirrored):
        for key, value in normal_images.items():
            mirror = value.toImage()
            mirror = mirror.mirrored(True, False)
            mirrored[key] = QtGui.QPixmap().fromImage(mirror)

    def action(self):
        """ Incorporates all objects' actions and calls the repaint event."""
        if not self.paused:
            self.game.actions()
            if self.score != self.game.score:
                self.score = self.game.score
                self.score_changed.emit()
            self.repaint()

    def draw_alien(self, canvas, alien):
        if alien.mirrored:
            image = self.alien_images_mirrored[alien.kind]
        else:
            image = self.alien_images[alien.kind]
        if alien.state == 'swim':
            state = 0
        else:
            state = 160
        canvas.drawPixmap(alien.x, alien.y, image,
                          alien.frame, state, 160, 160)

    def draw_fish(self, canvas, fish):
        state = fish.state
        if fish.hungry:
            state = 'hungry_' + state
        if fish.mirrored:
            image = self.fish_images_mirrored[state]
        else:
            image = self.fish_images[state]
        canvas.drawPixmap(fish.x, fish.y, image,
                          fish.frame, fish.size * 80, 80, 80)

    def draw_coin(self, canvas, coin):
        canvas.drawPixmap(coin.x, coin.y, self.coin_image,
                          coin.frame, coin.worth * 72, 72, 72)

    def draw_food(self, canvas, food):
        canvas.drawPixmap(food.x, food.y, self.food_image,
                          food.frame, 0, 40, 40)


import conllu
from IPython import embed
import numpy as np
import networkx as nx
import random

def mean_d_dist(tree):
    return np.mean([abs(wi-hi) for wi, hi in enumerate(tree, start=1) if hi != 0])

def mean_h_dist(tree):
    tree_1 = nx.DiGraph()
    tree_1.add_edges_from([(x,i) for i,x in enumerate(tree, start=1)])
    return np.mean([len(nx.shortest_path(tree_1, 0, n)) for n in tree_1 if n != 0])

def read_file(file):
    language = '<' + file.split('/')[-1].split('_')[0] + '>'
    labels = set([])
    dataset = []
    with open(file) as f:
        parsed_file = conllu.parse_incr(f)
        for i, sentence in enumerate(parsed_file):
            ws, hs, rs, ps = list(zip(*[(s['form'], s['head'], s['deprel'], s['upostag']) for
                                        s in sentence if type(s['id']) is int]))
            # ignore sentences with <= 3 words
            if len(ws) <= 3:
                continue

            for l in rs:
                labels.add(l)

            dataset.append({'forms': list(map(lambda x: x.lower(), ws)),
                            'heads': list(hs),
                            'deprels': list(rs),
                            'upos': list(ps),
                            'len': len(ws),
                            'mhd': mean_d_dist(list(hs)),
                            'mdd': mean_h_dist(list(hs)),
                            'random': random.random(),
                            'language': language})
    return dataset, labels


#!/usr/bin/env python
# coding=utf-8
# This file is an adaptation of run_glue.py from The HuggingFace Inc. team
# which is licensed under the Apache License, Version 2.0 (the "License");
# https://github.com/huggingface/transformers/blob/main/examples/pytorch/text-classification/run_glue.py
# This code is used to fine-tune the library models for n-way classification.
# You can also adapt this script on your own text classification task. Pointers for this are left as comments.

import logging
import os
import random
import sys
from dataclasses import dataclass, field
from typing import Optional, List
from glob import glob
import pandas as pd
import json
from shutil import move
from os import path

import numpy as np
from datasets import load_dataset, load_metric, Dataset, Features, Value, ClassLabel
from numpy.lib.function_base import append
from primeqa.text_classification.processors.postprocessors.text_classifier import TextClassifierPostProcessor
from primeqa.text_classification.processors.preprocessors.text_classifier import TextClassifierPreProcessor
from primeqa.boolqa.processors.dataset.mrc2dataset import create_dataset_from_run_mrc_output
from primeqa.text_classification.metrics.classification import Classification
from primeqa.mrc.data_models.eval_prediction_with_processing import EvalPredictionWithProcessing
from primeqa.text_classification.trainers.nway import NWayTrainer
from primeqa.mrc.run_mrc_utils import object_reference # TODO this should be a utils file

import transformers
from transformers import (
    AutoConfig,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    DataCollatorWithPadding,
    EvalPrediction,
    HfArgumentParser,
    PretrainedConfig,
    Trainer,
    TrainingArguments,
    default_data_collator,
    set_seed,
)
from transformers.trainer_utils import get_last_checkpoint, is_main_process

# Will error if the minimal version of Transformers is not installed. Remove at your own risks.
#check_min_version("4.6.0")

logger = logging.getLogger(__name__)

@dataclass
class DataTrainingArguments:
    """
    Arguments pertaining to what data we are going to input our model for training and eval.

    Using `HfArgumentParser` we can turn this class
    into argparse arguments to be able to specify them on
    the command line.
    """

    task_name: str = field(
        default=None,
        metadata={"help": "The name of the task to train on: "},
    )
    max_seq_length: int = field(
        default=128,
        metadata={
            "help": "The maximum total input sequence length after tokenization. Sequences longer "
            "than this will be truncated, sequences shorter will be padded."
        },
    )
    overwrite_cache: bool = field(
        default=False, metadata={"help": "Overwrite the cached preprocessed datasets or not."}
    )
    pad_to_max_length: bool = field(
        default=True,
        metadata={
            "help": "Whether to pad all samples to `max_seq_length`. "
            "If False, will pad the samples dynamically when batching to the maximum length in the batch."
        },
    )
    max_train_samples: Optional[int] = field(
        default=None,
        metadata={
            "help": "For debugging purposes or quicker training, truncate the number of training examples to this "
            "value if set."
        },
    )
    max_eval_samples: Optional[int] = field(
        default=None,
        metadata={
            "help": "For debugging purposes or quicker training, truncate the number of evaluation examples to this "
            "value if set."
        },
    )
    max_predict_samples: Optional[int] = field(
        default=None,
        metadata={
            "help": "For debugging purposes or quicker training, truncate the number of prediction examples to this "
            "value if set."
        },
    )
    train_file: Optional[str] = field(
        default=None, metadata={"help": "A csv or a json file containing the training data."}
    )
    validation_file: Optional[str] = field(
        default=None, metadata={"help": "A csv or a json file containing the validation data."}
    )
    test_file: Optional[str] = field(default=None, metadata={"help": "A csv or a json file containing the test data."})
    balanced: Optional[bool] = field(
        default=False,
        metadata={
            "help": "balance the data if true "
        },
    )
    label_list: List[str] = field(
        default=None,
        metadata={"help": "the labels used by the classifier (order is significant)"}
    )
    sentence2_key: str = field(
        default=None,
        metadata={"help": "the field in the input dataset to use as sentence2" }
    )
    drop_label: str = field(
        default=None,
        metadata={"help": "dropping label 'no_answer' converts ternary classifier into binary"}
    )
    sentence1_key: str = field(
        default="question",
        metadata={"help": "the field in the input dataset to use as sentence1" }
    )
    example_id_key: str = field(
        default="example_id",
        metadata={"help": "a field that can be used as a unique identifier of input records"}
    )
    language_key: str = field(
        default=None,
        metadata={"help": "a field that can be used as a unique identifier of input records"}
    )
    output_label_prefix: str = field(
        default=None,
        metadata={"help": "a prefix used in the output file eval_predictions.json to distinguish fields created by this invocation of the classifier"}
    )

    def __post_init__(self):
        if self.train_file is None and (self.validation_file is None and self.test_file is None):
            raise ValueError("Need a training and test/validation file.")
        else:
            if self.train_file is not None:
                train_extension = self.train_file.split(".")[-1]
                assert train_extension in ["csv", "json","tsv", "jsonl"], "`train_file` should be a csv, tsv or a json(l) file."
            if self.validation_file is not None:
                validation_extension = self.validation_file.split(".")[-1]
                if self.train_file is not None:
                    assert train_extension == validation_extension
                else:
                    assert validation_extension in ["csv", "json", "tsv", "jsonl"], "`validation_file` should be a csv, tsv or a json(l) file."
            if self.test_file is not None:
                test_extension = self.test_file.split(".")[-1]
                assert test_extension in ["csv", "json", "tsv", "jsonl"], "`test_file` should be a csv, tsv or a json(l) file."

@dataclass
class ModelArguments:
    """
    Arguments pertaining to which model/config/tokenizer we are going to fine-tune from.
    """

    model_name_or_path: str = field(
        metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}
    )
    config_name: Optional[str] = field(
        default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"}
    )
    tokenizer_name: Optional[str] = field(
        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
    )
    cache_dir: Optional[str] = field(
        default=None,
        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
    )
    use_fast_tokenizer: bool = field(
        default=True,
        metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."},
    )
    model_revision: str = field(
        default="main",
        metadata={"help": "The specific model version to use (can be a branch name, tag name or commit id)."},
    )
    use_auth_token: bool = field(
        default=False,
        metadata={
            "help": "Will use the token generated when running `transformers-cli login` (necessary to use this script "
            "with private models)."
        },
    )

@dataclass
class TaskArguments:
    """
    Task specific arguments.
    """
    preprocessor: object_reference = field(
        default=TextClassifierPreProcessor,
        metadata={"help": "The name of the preprocessor to use.",
                  "choices": [TextClassifierPreProcessor]
                  }
    )
    postprocessor: object_reference = field(
        default=TextClassifierPostProcessor,
        metadata={"help": "The name of the postprocessor to use.",
                  "choices": [TextClassifierPostProcessor]
                  }
    )
    do_mrc_pipeline: bool = field(
        default=False,
        metadata={
            "help": "if true, then the predict dataset will be loaded from json format produced by run_mrc.py"
        }
    )

def create_a_backup_file_if_file_exists(original_file: str):
    if path.isfile(original_file):
        backup_file = '%s.bak' % original_file
        logging.debug('Found a pre-existing file at location %s.  Backing it up to %s' %
                      (original_file, backup_file))
        move(original_file, backup_file)

def save_to_json_file(obj_to_save, out_file_path, with_backup=True, indent=4, sort_keys=True):
    if with_backup:
        create_a_backup_file_if_file_exists(out_file_path)

    logging.debug('Writing %s as json to file: %s' % (type(obj_to_save), out_file_path))
    with open(out_file_path, 'w') as outfile:
        json.dump(obj_to_save, outfile, indent=indent, sort_keys=sort_keys)

def restrict_labels(dataset : Dataset, sentence1_key : str, label_list : List[str]) -> Dataset:
    ''' discard instances whose label is not in the label list '''
    examples_with_label = []
    for example in dataset:
        if example['label'] in label_list and example[sentence1_key] != None:
            examples_with_label.append(example)
    df = pd.DataFrame(examples_with_label)
    return Dataset.from_pandas(df)
# TODO - something like this is more idiomatic for pandas?
# df_all=datasets['train'].to_pandas()
# mask=~df_all[data_args.sentence1_key].isna() & df_all['label'].isin(data_args.label_list)
# df=df_all[mask]

def balance_dataset(data_args, datasets):
    '''balance the number of labels in a dataset'''
    examples_by_label = {}

    for label in data_args.label_list:
        examples_by_label[label] = []

    for example in datasets['train']:
        examples_by_label[example['label']].append(example)

        # min class size is smaller than dataset size
    min_count = len(datasets['train'])

    for label in examples_by_label:
        if len(examples_by_label[label]) < min_count:
            min_count = len(examples_by_label[label])

    balanced_examples = []

    if data_args.max_train_samples is not None and data_args.max_train_samples < min_count * len(examples_by_label):
        min_count = int(data_args.max_train_samples / len(examples_by_label))

    for label in examples_by_label:
        def seed():
          return 0.1
            # should we have something like this? its possible the data is sorted in some manner (eg alphabetically)
        random.shuffle(examples_by_label[label], seed)
        balanced_examples.extend(examples_by_label[label][:min_count])
    random.shuffle(balanced_examples)
    df = pd.DataFrame(balanced_examples)
    logger.info("balanced (down sampling) dataset to " + str(min_count) + " per class.")
    return Dataset.from_pandas(df)

def main(raw_args):
    # See all possible arguments in src/transformers/training_args.py
    # or by passing the --help flag to this script.
    # We now keep distinct sets of args, for a cleaner separation of concerns.
    parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments, TaskArguments))
    if len(raw_args) == 2 and raw_args[1].endswith(".json"):
        # If we pass only one argument to the script and it's the path to a json file,
        # let's parse it to get our arguments.
        model_args, data_args, training_args, task_args = parser.parse_json_file(json_file=os.path.abspath(raw_args[1]))
    elif len(raw_args) == 1:
        model_args, data_args, training_args, task_args = parser.parse_dict(raw_args[0])
    else:
        model_args, data_args, training_args, task_args = parser.parse_args_into_dataclasses()

    # Detecting last checkpoint.
    last_checkpoint = None
    if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir:
        last_checkpoint = get_last_checkpoint(training_args.output_dir)
        if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0:
            raise ValueError(
                f"Output directory ({training_args.output_dir}) already exists and is not empty. "
                "Use --overwrite_output_dir to overcome."
            )
        elif last_checkpoint is not None and training_args.resume_from_checkpoint is None:
            logger.info(
                f"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change "
                "the `--output_dir` or add `--overwrite_output_dir` to train from scratch."
            )

    # Setup logging
    logging.basicConfig(
        format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
        datefmt="%m/%d/%Y %H:%M:%S",
        handlers=[logging.StreamHandler(sys.stdout)],
    )
    logger.setLevel(logging.INFO if is_main_process(training_args.local_rank) else logging.WARN)

    # Log on each process the small summary:
    logger.warning(
        f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}"
        + f"distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16}"
    )
    # Set the verbosity to info of the Transformers logger (on main process only):
    if is_main_process(training_args.local_rank):
        transformers.utils.logging.set_verbosity_info()
        transformers.utils.logging.enable_default_handler()
        transformers.utils.logging.enable_explicit_format()
    logger.info(f"Training/evaluation parameters {training_args}")

    # Set seed before initializing model.
    set_seed(training_args.seed)

    # Get the datasets: Provide your own CSV/JSON training and evaluation files (see below)
    # This script will use as labels the column called 'label' and as pair of sentences the
    # sentences in columns called 'sentence1' and 'sentence2' if such column exists or the first two columns not named
    # label if at least two columns are provided.
    #
    # If the CSVs/JSONs contain only one non-label column, the script does single sentence classification on this
    # single column. You can easily tweak this behavior (see below)
    #
    # In distributed training, the load_dataset function guarantee that only one local process can concurrently
    # download the dataset.

    # Loading a dataset from your local files.
    # CSV/JSON training and evaluation files are needed.

    # Possible options:
    # Train, Train+Eval, Train+Predict, Eval, Predict
    data_files = {}
    non_label_column_names = None
    if training_args.do_train:
        if data_args.train_file is None:
            raise ValueError("Missing train file for `do_train`.")
        train_files = [infile for infile in glob(data_args.train_file)]
        data_files["train"] = train_files

    if training_args.do_eval:
        if data_args.validation_file is None:
            raise ValueError("Missing validation file for `do_eval`.")
        validation_files = [infile for infile in glob(data_args.validation_file)]
        data_files["validation"] = validation_files
    # Get the test dataset: you can provide your own CSV/JSON test file (see below)
    if training_args.do_predict:
        if data_args.test_file is None:
            raise ValueError("Missing test file for `do_predict`.")
        test_files = [infile for infile in glob(data_args.test_file)]
        data_files["test"] = test_files

    for key in data_files.keys():
        logger.info(f"load a local file for {key}: {data_files[key]}")

    cache_dir = None
    if not data_args.overwrite_cache:
        cache_dir=model_args.cache_dir

    features_input={
        data_args.example_id_key: Value('string'),
        data_args.sentence1_key: Value('string'),
        'label': Value('string')
    }
    if data_args.language_key is not None:
        features_input[ data_args.language_key ] = Value('string')
    if data_args.sentence2_key is not None:
        features_input[ data_args.sentence2_key ] = Value('string')
    features = Features( features_input )

    if task_args.do_mrc_pipeline:
        datasets={'test':create_dataset_from_run_mrc_output(data_args.test_file, unpack=False)}
    else:
        if data_args.train_file is not None and data_args.train_file.endswith(".csv") or data_args.test_file is not None and data_args.test_file.endswith(".csv"):
            # Loading a dataset from local csv files
            datasets = load_dataset("csv", data_files=data_files, cache_dir=cache_dir, delimiter=",", features=features)
        elif (data_args.train_file is not None and data_args.train_file.endswith(".tsv")) or (data_args.validation_file is not None \
        and data_args.validation_file.endswith(".tsv")) or (data_args.test_file is not None and data_args.test_file.endswith(".tsv")):
            # Loading a dataset from local csv files
            datasets = load_dataset("csv", data_files=data_files, cache_dir=cache_dir, delimiter="\t", features=features)
        else:
            # Loading a dataset from local json files
            datasets = load_dataset("json", data_files=data_files, cache_dir=cache_dir, features=features)

    # Labels - load from file
    num_labels = len(data_args.label_list)

    logger.info("The following labels are being used, all other instances will be discarded.")
    logger.info(data_args.label_list)

    # discard instances whose label is not in the label list
    if training_args.do_train:
        logger.info("Discard training instance not in label list, if any.")
        datasets['train']=restrict_labels(datasets['train'], data_args.sentence1_key, data_args.label_list)
        if len(datasets['train']) == 0:
            raise ValueError("No training data left with labels provided in label list")

    if training_args.do_eval:
        logger.info("Discard eval instance not in label list, if any.")
        datasets['validation']=restrict_labels(datasets['validation'], data_args.sentence1_key, data_args.label_list)
        if len(datasets['validation']) == 0:
            raise ValueError("No validation data left with labels provided in label list")

    # balance the training dataset
    if training_args.do_train and data_args.balanced:
        datasets['train'] = balance_dataset(data_args, datasets)

    # See more about loading any type of standard or custom dataset at
    # https://huggingface.co/docs/datasets/loading_datasets.html.

    # Load pretrained model and tokenizer
    #
    # In distributed training, the .from_pretrained methods guarantee that only one local process can concurrently
    # download model & vocab.
    config = AutoConfig.from_pretrained(
        model_args.config_name if model_args.config_name else model_args.model_name_or_path,
        num_labels=num_labels,
        finetuning_task=data_args.task_name,
        cache_dir=model_args.cache_dir,
        revision=model_args.model_revision,
        use_auth_token=True if model_args.use_auth_token else None,
    )
    tokenizer = AutoTokenizer.from_pretrained(
        model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path,
        cache_dir=model_args.cache_dir,
        use_fast=model_args.use_fast_tokenizer,
        revision=model_args.model_revision,
        use_auth_token=True if model_args.use_auth_token else None,
    )
    model = AutoModelForSequenceClassification.from_pretrained(
        model_args.model_name_or_path,
        from_tf=bool(".ckpt" in model_args.model_name_or_path),
        config=config,
        cache_dir=model_args.cache_dir,
        revision=model_args.model_revision,
        use_auth_token=True if model_args.use_auth_token else None,
    )

    # Padding strategy
    if data_args.pad_to_max_length:
        padding = "max_length"
    else:
        # We will pad later, dynamically at batch creation, to the max sequence length in each batch
        padding = False

    if data_args.max_seq_length > tokenizer.model_max_length:
        logger.warning(
            f"The max_seq_length passed ({data_args.max_seq_length}) is larger than the maximum length for the"
            f"model ({tokenizer.model_max_length}). Using max_seq_length={tokenizer.model_max_length}."
        )
    max_seq_length = min(data_args.max_seq_length, tokenizer.model_max_length)

    preprocessor_class = task_args.preprocessor
    preprocessor = preprocessor_class(
        example_id_key=data_args.example_id_key,
        sentence1_key=data_args.sentence1_key,
        sentence2_key=data_args.sentence2_key,
        language_key=data_args.language_key,
        tokenizer=tokenizer,
        load_from_cache_file=not data_args.overwrite_cache,
        max_seq_len=max_seq_length,
        padding=padding,
        label_list=data_args.label_list
    )

    if training_args.do_train:
        if "train" not in datasets:
            raise ValueError("--do_train requires a train dataset")
        train_examples = datasets["train"]
        max_train_samples = (
            data_args.max_train_samples if data_args.max_train_samples is not None else len(train_examples)
        )
        if max_train_samples is not None and len(train_examples) > max_train_samples:
            train_examples = train_examples.select(range(max_train_samples))
        train_examples, train_dataset = preprocessor.process_train(train_examples)

    if training_args.do_eval:
        if "validation" not in datasets and "validation_matched" not in datasets:
            raise ValueError("--do_eval requires a validation dataset")
        eval_examples = datasets["validation"]

        max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_examples)

        if max_eval_samples is not None:
            eval_examples = eval_examples.select(range(max_eval_samples))
        eval_examples, eval_dataset = preprocessor.process_eval(eval_examples)

    if training_args.do_predict or data_args.test_file is not None:
        if "test" not in datasets and "test_matched" not in datasets:
            raise ValueError("--do_predict requires a test dataset")
        predict_examples = datasets["test"]
        if data_args.max_predict_samples is not None:
            predict_examples = predict_examples.add_faiss_indexselect(range(data_args.max_predict_samples))

        predict_examples, predict_dataset = preprocessor.process_eval(predict_examples)

    # Log a few random samples from the training set:
    if training_args.do_train:
        for index in random.sample(range(len(train_dataset)), 3):
            logger.info(f"Sample {index} of the training set: {train_dataset[index]}.")

    # Get the metric function
    #if data_args.task_name is not None:
    eval_metrics = Classification()
    def compute_metrics(p: EvalPredictionWithProcessing):
        return eval_metrics.compute(predictions=p.processed_predictions, references=p.label_ids)

    # Data collator will default to DataCollatorWithPadding, so we change it if we already did the padding.
    if data_args.pad_to_max_length:
        data_collator = default_data_collator
    elif training_args.fp16:
        data_collator = DataCollatorWithPadding(tokenizer, pad_to_multiple_of=8)
    else:
        data_collator = None

    postprocessor_class = task_args.postprocessor
    postprocessor = postprocessor_class(
        k=10,
        drop_label=data_args.drop_label,
        label_list = data_args.label_list,
        id_key=data_args.example_id_key,
        output_label_prefix=data_args.output_label_prefix
    )

    training_args.metric_for_best_model='eval_all_avg_f1'

    # Initialize our Trainer
    trainer = NWayTrainer(
        model=model,
        args=training_args,
        train_dataset=train_dataset if training_args.do_train else None,
        eval_dataset=eval_dataset if training_args.do_eval else None,
        eval_examples=eval_examples if training_args.do_eval else None,
        compute_metrics=compute_metrics,
        tokenizer=tokenizer,
        data_collator=data_collator,
        post_process_function=postprocessor.process if task_args.do_mrc_pipeline else postprocessor.process_references_and_predictions,
    )

    # Training
    if training_args.do_train:
        checkpoint = None
        if training_args.resume_from_checkpoint is not None:
            checkpoint = training_args.resume_from_checkpoint
        elif last_checkpoint is not None:
            checkpoint = last_checkpoint
        train_result = trainer.train(resume_from_checkpoint=checkpoint)
        metrics = train_result.metrics
        metrics["train_samples"] = min(max_train_samples, len(train_dataset))

        trainer.save_model()  # Saves the tokenizer too for easy upload
        trainer.log_metrics("train", metrics)
        trainer.save_metrics("train", metrics)
        trainer.save_state()

    # Evaluation - have gold answers
    if training_args.do_eval:
        logger.info("*** Evaluate ***")
        metrics = trainer.evaluate()
        metrics["eval_samples"] = min(max_eval_samples, len(eval_examples))

        trainer.log_metrics("eval", metrics)
        trainer.save_metrics("eval", metrics)

    # Predict on unlabeled data
    if training_args.do_predict or task_args.do_mrc_pipeline:
        logger.info("*** Predict ***")
        predict_dataset=predict_dataset.remove_columns('label')
        predictions = trainer.predict(predict_dataset, predict_examples)
        # TODO I don't think process_reference_and_predictions should be called here

        with open(os.path.join(training_args.output_dir, 'predictions.json'), 'w') as f:
            json.dump(predictions.predictions, f, indent=4)
        with open(os.path.join(training_args.output_dir, 'predictions_processed.json'), 'w') as f:
            json.dump(predictions.processed_predictions, f, indent=4)

def _mp_fn(index):
    # For xla_spawn (TPUs)
    main()

if __name__ == "__main__":
    main(sys.argv)


import csv
import time
import random
import string
import numpy as np
from datetime import datetime
from datetime import timedelta
from dateutil.relativedelta import relativedelta

from ...helpers import get_file_lines, str_time_prop
from ...AbstractTableTransformer import AbstractTableTransformer

class ClientTableTransformer(AbstractTableTransformer):
    def __init__(self, settings, tableName):
        super().__init__(settings, tableName)

        self.clientsIdCounter = 0
        self.allClients = []

        # helpers
        self._date_format = self.settings["date_format"]
        self._date_format_out = self.settings["date_format_out"]

    def __get_ageGroup(self, birthDate):
        date = datetime.strptime(birthDate, self._date_format)
        age = relativedelta(datetime.now(), date).years

        age_group = "Unknown"
        if age <= 25:
            age_group = "15-25"
        elif age <= 35:
            age_group = "26-35"
        elif age <= 45:
            age_group = "36-45"
        elif age <= 55:
            age_group = "46-55"
        elif age <= 65:
            age_group = "56-65"
        elif age <= 75:
            age_group = "66-75"
        elif age <= 85:
            age_group = "76-85"
        elif age >= 86:
            age_group = "86-95"

        return age_group

    def __get_gender(self, gender):
        return gender

    def __get_married(self, married):
        status = "Unknown"

        if married == "married":
            status = "true"
        elif married == "single":
            status = "false"

        return status

    def __get_children(self, children):
        return children

    def __asserts(self, data):
        pass

    def __check_for_repeaters(self, data):
        for i in self.allClients:
            if data["AGE_GROUP"] == i["AGE_GROUP"] and data["GENDER"] == i["GENDER"] and data["MARRIED"] == i["MARRIED"] and data["CHILDREN"] == i["CHILDREN"]:
                data = i
                break
        else:
            data["CLIENTID"] = self.clientsIdCounter
            self.allClients.append(data)
            self.clientsIdCounter += 1
        return data

    def transform(self, src_data):
        # to collection
        data = {
            "CLIENTID": None,
            "AGE_GROUP": self.__get_ageGroup(src_data["client_birthDate"]),
            "GENDER": self.__get_gender(src_data["client_gender"]),
            "MARRIED": self.__get_married(src_data["client_married"]),
            "CHILDREN": self.__get_children(src_data["client_children"])
        }

        self.__asserts(data)
        data = self.__check_for_repeaters(data)
        self.__asserts(data)

        return data

    def table_write(self, data):
        self._writer.writerow([
            data["CLIENTID"],
            data["AGE_GROUP"],
            data["GENDER"],
            data["MARRIED"],
            data["CHILDREN"]
        ])

    def table_write_all(self):
        for item in self.allClients:
            self.table_write(item)