text:
import unittest
import math
from kdt import *
from kdt import pyCombBLAS as pcb
import numpy as np

class SpParVecTests(unittest.TestCase):
    def initializeSpParVec(self, length, i, v=1):
        """
        Initialize a ParVec instance with values equal to one or the input value.
        """
        ret = SpParVec(length)
        for ind in range(len(i)):
	    if type(v) != int and type(v) != float:
		ret[i[ind]] = v[ind]
            else:
                ret[i[ind]] = v

        return ret

    def initializeParVec(self, length, i, v=1):
        """
        Initialize a ParVec instance with values equal to one or the input value.
        """
        ret = ParVec(length, 0)
        for ind in range(len(i)):
	    if type(v) != int and type(v) != float:
		ret[i[ind]] = v[ind]
            else:
                ret[i[ind]] = v

        return ret

class ConstructorTests(SpParVecTests):
    def test_SpParVec_simple(self):
	sz = 25
        i = [0, 2, 4, 6, 8, 10]
        vec = self.initializeSpParVec(sz, i)
        expI = [1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
                 0, 0, 0, 0, 0]
	self.assertEqual(sz, len(vec))
        for ind in range(sz):
            self.assertEqual(expI[ind],vec[ind])

    def test_SpParVec_zeros(self):
	sz = 25
        vec = SpParVec(sz)
        expI = 0
	self.assertEqual(sz, len(vec))
        for ind in range(sz):
            self.assertEqual(expI,vec[ind])

    def test_SpParVec_ones(self):
	sz = 25
        vec = SpParVec.ones(sz)
        expI = 1
	self.assertEqual(sz, len(vec))
        for ind in range(sz):
            self.assertEqual(expI,vec[ind])

    def test_SpParVec_range_simple(self):
	sz = 25
        vec = SpParVec.range(sz)
	self.assertEqual(sz, len(vec))
        for ind in range(sz):
            self.assertEqual(ind,vec[ind])

    def test_SpParVec_range_offset(self):
	sz = 25
	offset = -13
        vec = SpParVec.range(offset, sz+offset)
        expI = 1
	self.assertEqual(sz, len(vec))
        for ind in range(sz):
            self.assertEqual(ind,vec[ind]-offset)

    def test_SpParVec_set37(self):
	sz = 25
        vec = SpParVec.ones(sz)
        i = [0, 2,  4,  6,  8, 10]
        v = [4, 8, 12, 16, 20, 24]
        vec = self.initializeSpParVec(sz, i, v)
	scalar = 37
	vec.set(scalar)
	expVec = [37, 0, 37, 0, 37, 0, 37, 0, 37, 0, 37, 0, 0, 0, 0, 0, 0, 0,
		0, 0, 0, 0, 0, 0, 0]
	self.assertEqual(sz, len(vec))
        for ind in range(sz):
            self.assertEqual(expVec[ind],vec[ind])

    def test_SpParVec_spOnes(self):
	sz = 25
        i = [0, 2,  4,  6,  8, 10]
        v = [4, 8, 12, 16, 20, 24]
        vec = self.initializeSpParVec(sz, i, v)
        vec.spOnes()
	expVec = [1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0,
		0, 0, 0, 0, 0, 0, 0]
	self.assertEqual(sz, len(vec))
        for ind in range(sz):
            self.assertEqual(expVec[ind],vec[ind])

    def test_SpParVec_toBool(self):
	sz = 25
        i = [0, 2,  4,  6,  8, 10]
        v = [4, 8, 12, 16, 20, 24]
        vec = self.initializeSpParVec(sz, i, v)
        vec.toBool()
	expVec = [1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0,
		0, 0, 0, 0, 0, 0, 0]
	self.assertEqual(sz, len(vec))
        for ind in range(sz):
            self.assertEqual(expVec[ind],vec[ind])

class BuiltInTests(SpParVecTests):
    def test_add_constant(self):
	sz = 25
        i = [0, 2, 4, 6, 8, 10]
        vec = self.initializeSpParVec(sz, i)
        vec2 = vec + 3.07
        expI = [4.07, 0, 4.07, 0, 4.07, 0, 4.07, 0, 4.07, 0,
		4.07, 0, 0, 0, 0, 0, 0, 0, 0, 0,
		0, 0, 0, 0, 0, 0]
	self.assertEqual(sz, len(vec2))
	self.assertEqual(len(i), vec2.nnn())
        for ind in range(sz):
	    self.assertEqual(expI[ind], vec2[ind])

    def test_add_vector(self):
	sz = 25
        i1 = [0, 2, 4, 6, 8, 10]
        v1 = [0, 4, 16, 36, 64, 100]
        vec1 = self.initializeSpParVec(sz, i1, v1)
        i2 = [0, 3, 6, 9, 12, 15, 18]
        v2 = [1, 27, 216, 729, 1728, 3375, 5832]
        vec2 = self.initializeSpParVec(sz, i2, v2)
        vec3 = vec1 + vec2
        expI = [1, 0, 4, 27, 16, 0, 252, 0, 64, 729, 100, 0, 1728, 0, 0, 3375,
		0, 0, 5832, 0, 0, 0, 0, 0, 0,]
	self.assertEqual(sz, len(vec3))
	self.assertEqual(len(i1)+len(i2)-2,vec3.nnn())
        for ind in range(sz):
	    self.assertEqual(expI[ind], vec3[ind])

    def test_iadd_constant(self):
	sz = 25
        i = [0, 2, 4, 6, 8, 10]
        vec = self.initializeSpParVec(sz, i)
        vec += 3.07
        expI = [4.07, 0, 4.07, 0, 4.07, 0, 4.07, 0, 4.07, 0,
		4.07, 0, 0, 0, 0, 0, 0, 0, 0, 0,
		0, 0, 0, 0, 0, 0]
	self.assertEqual(sz, len(vec))
	self.assertEqual(len(i), vec.nnn())
        for ind in range(sz):
	    self.assertEqual(expI[ind], vec[ind])

    def test_iadd_vector(self):
	sz = 25
        i1 = [0, 2, 4, 6, 8, 10]
        v1 = [0, 4, 16, 36, 64, 100]
        vec1 = self.initializeSpParVec(sz, i1, v1)
        i2 = [0, 3, 6, 9, 12, 15, 18]
        v2 = [1, 27, 216, 729, 1728, 3375, 5832]
        vec2 = self.initializeSpParVec(sz, i2, v2)
        vec1 += vec2
        expI = [1, 0, 4, 27, 16, 0, 252, 0, 64, 729, 100, 0, 1728, 0, 0, 3375,
		0, 0, 5832, 0, 0, 0, 0, 0, 0,]
	self.assertEqual(sz, len(vec1))
	self.assertEqual(len(i1)+len(i2)-2,vec1.nnn())
        for ind in range(sz):
	    self.assertEqual(expI[ind], vec1[ind])

    def test_sub_constant(self):
	sz = 25
        i = [0, 2, 4, 6, 8, 10]
        vec = self.initializeSpParVec(sz, i)
        vec2 = vec - 3.07
        expI = [-2.07, 0, -2.07, 0, -2.07, 0, -2.07, 0, -2.07,
		0, -2.07, 0, 0, 0, 0, 0, 0, 0,
		0, 0, 0, 0, 0, 0, 0, 0]
	self.assertEqual(sz, len(vec2))
	self.assertEqual(vec.nnn(), len(i))
	self.assertEqual(vec.nnn(), vec2.nnn())
        for ind in range(sz):
	    self.assertEqual(expI[ind], vec2[ind])

    def test_sub_vector(self):
	sz = 25
        i1 = [0, 2, 4, 6, 8, 10]
        v1 = [0, 4, 16, 36, 64, 100]
        vec1 = self.initializeSpParVec(sz, i1, v1)
        i2 = [0, 3, 6, 9, 12, 15, 18]
        v2 = [1, 27, 216, 729, 1728, 3375, 5832]
        vec2 = self.initializeSpParVec(sz, i2, v2)
        vec3 = vec1 - vec2
        expI = [-1, 0, 4, -27, 16, 0, -180, 0, 64, -729, 100, 0, -1728, 0, 0,
		-3375, 0, 0, -5832, 0, 0, 0, 0, 0, 0,]
	self.assertEqual(sz, len(vec3))
	self.assertEqual(len(i1)+len(i2)-2,vec3.nnn())
        for ind in range(sz):
	    self.assertEqual(expI[ind], vec3[ind])

    def test_isub_constant(self):
	sz = 25
        i = [0, 2, 4, 6, 8, 10]
        vec = self.initializeSpParVec(sz, i)
        vec -= 3.07
        expI = [-2.07, 0, -2.07, 0, -2.07, 0, -2.07, 0, -2.07,
		0, -2.07, 0, 0, 0, 0, 0, 0, 0,
		0, 0, 0, 0, 0, 0, 0, 0]
	self.assertEqual(sz, len(vec))
	self.assertEqual(vec.nnn(), len(i))
        for ind in range(sz):
	    self.assertEqual(expI[ind], vec[ind])

    def test_isub_vector(self):
	sz = 25
        i1 = [0, 2, 4, 6, 8, 10]
        v1 = [0, 4, 16, 36, 64, 100]
        vec1 = self.initializeSpParVec(sz, i1, v1)
        i2 = [0, 3, 6, 9, 12, 15, 18]
        v2 = [1, 27, 216, 729, 1728, 3375, 5832]
        vec2 = self.initializeSpParVec(sz, i2, v2)
        vec1 -= vec2
        expI = [-1, 0, 4, -27, 16, 0, -180, 0, 64, -729, 100, 0, -1728, 0, 0,
		-3375, 0, 0, -5832, 0, 0, 0, 0, 0, 0,]
	self.assertEqual(sz, len(vec1))
	self.assertEqual(len(i1)+len(i2)-2,vec1.nnn())
        for ind in range(sz):
	    self.assertEqual(expI[ind], vec1[ind])

    def test_negate(self):
	sz = 25
        i1 = [0, 2, 4, 6, 8, 10]
        v1 = [0, 4, 16, 36, 64, 100]
        vec1 = self.initializeSpParVec(sz, i1, v1)
        vec3 = -vec1
        expI = [0, 0, -4, 0, -16, 0, -36, 0, -64, 0, -100, 0, 0, 0, 0,
		0, 0, 0, 0, 0, 0, 0, 0, 0, 0,]
	self.assertEqual(sz, len(vec3))
	self.assertEqual(vec1.nnn(),vec3.nnn())
        for ind in range(sz):
	    self.assertEqual(expI[ind], vec3[ind])

    def test_mul_constant(self):
	sz = 25
        i = [0, 2, 4, 6, 8, 10]
        vec = self.initializeSpParVec(sz, i)
        vec2 = vec * 3.07
        expI = [3.07, 0, 3.07, 0, 3.07, 0, 3.07, 0, 3.07, 0,
		3.07, 0, 0, 0, 0, 0, 0, 0, 0, 0,
		0, 0, 0, 0, 0, 0]
	self.assertEqual(sz, len(vec2))
	self.assertEqual(vec.nnn(), vec2.nnn())
        for ind in range(sz):
	    self.assertEqual(expI[ind], vec2[ind])

    def test_mul_vector(self):
	sz = 25
        i1 = [0, 2, 4, 6, 8, 10, 12]
        v1 = [1, 4, 16, 36, 64, 100, 144]
        vec1 = self.initializeSpParVec(sz, i1, v1)
	vec2 = ParVec(sz)
	vec2[0] = 1
	vec2[3] = 3
	vec2[6] = 6
	vec2[9] = 9
	vec2[12] = 12
	vec2[15] = 15
	vec2[18] = 18
        vec3 = vec1 * vec2
        expI = [1, 0, 0, 0, 0, 0, 216, 0, 0, 0, 0, 0, 1728, 0, 0, 0, 0, 0, 0, 0,
		0, 0, 0, 0, 0,]
	self.assertEqual(sz, len(vec3))
	self.assertEqual(vec3.nnn(), 3)
        for ind in range(sz):
	    self.assertEqual(expI[ind], vec3[ind])

    def test_div_constant(self):
	sz = 25
        i = [0, 2, 4, 6, 8, 10, 12]
        v1 = [1, 4, 16, 36, 64, 100, 144]
        vec = self.initializeSpParVec(sz, i, v1)
        vec2 = vec / .50
        expI = [2, 0, 8, 0, 32, 0, 72, 0, 128, 0,
		200, 0, 288, 0, 0, 0, 0, 0, 0,
		0, 0, 0, 0, 0, 0]
	self.assertEqual(sz, len(vec2))
	self.assertEqual(vec.nnn(), vec2.nnn())
        for ind in range(sz):
	    self.assertEqual(expI[ind], vec2[ind])

    def test_div_vector(self):
	sz = 25
        i1 = [0, 2, 4, 6, 8, 10, 12]
        v1 = [1, 4, 16, 36, 64, 100, 144]
        vec1 = self.initializeSpParVec(sz, i1, v1)
        i2 = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18,
		19, 20, 21, 22, 23, 24]
        v2 = [1, 1, 1, 3, 1, 1, 6, 1, 1, 9, 1, 1, 12, 1, 1, 15,
		1, 1, 18, 1, 1, 1, 1, 1, 1]
        vec2 = self.initializeSpParVec(sz, i2, v2)
	self.assertRaises(NotImplementedError,SpParVec.__div__,vec1,vec2)
        #expI = [1, 0, 4, 0, 16, 0, 6, 0, 64, 0, 100, 0, 12, 0, 0, 0, 0, 0, 0, 0,
	#	0, 0, 0, 0, 0,]
	#self.assertEqual(sz, len(vec3))
	#self.assertEqual(vec.nnn(), vec3.nnn())
        #for ind in range(sz):
	#    self.assertAlmostEqual(expI[ind], vec3[ind])

    def test_mod_constant(self):
	sz = 25
        i = [0, 2, 4, 6, 8, 10, 12]
        v1 = [1, 4, 16, 36, 64, 100, 144]
        vec = self.initializeSpParVec(sz, i, v1)
        vec2 = vec % 5
        expI = [1, 0, 4, 0, 1, 0, 1, 0, 4, 0,
		0, 0, 4, 0, 0, 0, 0, 0, 0,
		0, 0, 0, 0, 0, 0]
	self.assertEqual(sz, len(vec2))
	self.assertEqual(vec.nnn(), vec2.nnn())
        for ind in range(sz):
	    self.assertEqual(expI[ind], vec2[ind])

    def test_mod_vector(self):
	sz = 25
        i1 = [0, 2, 4, 6, 8, 10, 12]
        v1 = [1, 4, 16, 36, 64, 100, 144]
        vec1 = self.initializeSpParVec(sz, i1, v1)
        i2 = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18,
		19, 20, 21, 22, 23, 24]
        v2 = [3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2,
		3, 2, 3, 2, 3]
        vec2 = self.initializeSpParVec(sz, i2, v2)
	self.assertRaises(NotImplementedError,SpParVec.__mod__,vec1,vec2)
        #expI = [1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	#	0, 0, 0, 0, 0,]
	#self.assertEqual(sz, len(vec3))
	#self.assertEqual(vec.nnn(), vec3.nnn())
        #for ind in range(sz):
	#    self.assertAlmostEqual(expI[ind], vec3[ind])

    def test_and_vector(self):
	sz = 25
        i1 = [0, 2, 4, 6, 8, 10, 12, 14, 16, 18]
        v1 = [0, 1, 1, 1, 1,  1,  1,  1,  1,  1]
        vec1 = self.initializeSpParVec(sz, i1, v1)
        i2 = [0, 3, 6, 9, 12, 15, 18]
        v2 = [1, 1, 1, 1,  1,  1,  1]
        vec2 = self.initializeSpParVec(sz, i2, v2)
        vec3 = vec1 & vec2
        expI = [0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1,
		0, 0, 0, 0, 0, 0, 0]
	self.assertEqual(sz, len(vec3))
	self.assertEqual(3,vec3.nnn())
        for ind in range(sz):
	    self.assertEqual(expI[ind], vec3[ind])

    def test_xor_vector(self):
	sz = 25
        i1 = [0, 2, 4, 6, 8, 10, 12, 14, 16, 18]
        v1 = [0, 1, 1, 1, 1,  1,  1,  1,  1,  1]
        vec1 = self.initializeSpParVec(sz, i1, v1)
        i2 = [0, 3, 6, 9, 12, 15, 18]
        v2 = [1, 1, 1, 1,  1,  1,  1]
        vec2 = self.initializeSpParVec(sz, i2, v2)
        vec3 = vec1 ^ vec2
        expI = [0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1,
		0, 0, 0, 0, 0, 0, 0]
	self.assertEqual(sz, len(vec3))
	self.assertEqual(3,vec3.nnn())
        for ind in range(sz):
	    self.assertEqual(expI[ind], vec3[ind])

    def test_indexing_RHS_SpParVec_scalar(self):
	sz = 25
        vec1 = SpParVec.range(int(-math.floor(sz/2.0)),int(math.ceil(sz/2.0)))
	ndx = 9
        value = vec1[ndx]
	self.assertEqual(-3, value)

    def test_indexing_RHS_SpParVec_scalar_outofbounds(self):
	sz = 25
        vec1 = SpParVec.range(int(-math.floor(sz/2.0)),int(math.ceil(sz/2.0)))
	ndx = 7777
	self.assertRaises(IndexError, SpParVec.__getitem__, vec1, ndx)
        #value = vec1[ndx]
	#self.assertEqual(-3, value)

    def test_indexing_RHS_SpParVec_scalar_outofbounds2(self):
	sz = 25
        vec1 = SpParVec.range(int(-math.floor(sz/2.0)),int(math.ceil(sz/2.0)))
	ndx = -333
	self.assertRaises(IndexError, SpParVec.__getitem__, vec1, ndx)
        #value = vec1[ndx]
	#self.assertEqual(-3, value)

    def test_indexing_RHS_SpParVec_SpParVec(self):
	sz = 25
        vec1 = SpParVec.range(int(-math.floor(sz/2.0)),int(math.ceil(sz/2.0)))
	ndxLen = 4
        ndxI = [0, 1, 2,  3]
        ndxV = [1, 4, 9, 16]
        ndx = self.initializeSpParVec(ndxLen, ndxI, ndxV)
	self.assertRaises(KeyError, SpParVec.__getitem__, vec1, ndx)
        #expI = [-11, -8, -3, 4]
	#self.assertEqual(ndxLen, len(vec3))
        #for ind in range(ndxLen):
	#    self.assertEqual(expI[ind], vec3[ind])

    def test_indexing_RHS_SpParVec_ParVec(self):
	sz = 25
        vec1 = SpParVec.range(int(-math.floor(sz/2.0)),int(math.ceil(sz/2.0)))
	ndxLen = 4
	ndx = ParVec(4)
	ndx[0] = 1
	ndx[1] = 4
	ndx[2] = 9
	ndx[3] = 16
        vec3 = vec1[ndx]
        expI = [-11, -8, -3, 4]
	self.assertEqual(ndxLen, len(vec3))
        for ind in range(ndxLen):
	    self.assertEqual(expI[ind], vec3[ind])

    def test_indexing_RHS_SpParVec_booleanSpParVec(self):
	sz = 25
        vec1 = SpParVec.range(int(-math.floor(sz/2.0)),int(math.ceil(sz/2.0)))
	ndxLen = 25
	ndxTrue = 4
        ndxI = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17,
		18, 19, 20, 21, 22, 23, 24]
        ndxV = [0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0,
		0, 0, 0, 0, 0]
        ndx = self.initializeSpParVec(ndxLen, ndxI, ndxV)
	self.assertRaises(KeyError, SpParVec.__getitem__, vec1, ndx)
        #vec3 = vec1[ndx]
        #expI = [1,16,81,256]
	#self.assertEqual(ndxTrue, len(vec3))
        #for ind in range(ndxTrue):
	#    self.assertEqual(expI[ind], vec3[ind])

    def test_indexing_LHS_SpParVec_booleanParVec_scalar(self):
	sz = 18
        vec1 = SpParVec.range(int(-math.floor(sz/2.0)),int(math.ceil(sz/2.0)))
	ndxLen = 18
	ndx = ParVec(sz)
	ndx[1] = 1
	ndx[4] = 1
	ndx[9] = 1
	ndx[16] = 1
        vec1[ndx] = 77
        expI = [-9, 77, -7, -6, 77, -4, -3, -2, -1, 77, 1, 2, 3, 4, 5, 6, 77, 8]
	self.assertEqual(sz, len(vec1))
        for ind in range(ndxLen):
	    self.assertEqual(expI[ind], vec1[ind])

    def test_indexing_LHS_SpParVec_nonbooleanParVec_scalar(self):
	sz = 18
        vec1 = SpParVec.range(int(-math.floor(sz/2.0)),int(math.ceil(sz/2.0)))
	ndxLen = 18
	ndx = ParVec(sz)
	ndx[1] = 7
	ndx[4] = 11
	ndx[9] = 1
	ndx[16] = 5
	self.assertRaises(KeyError, SpParVec.__setitem__, vec1, ndx, 77)
        #vec1[ndx] = 77
        #expI = [-9, 77, -7, -6, 77, -4, -3, -2, -1, 77, 1, 2, 3, 4, 5, 6, 77, 8]
	#self.assertEqual(sz, len(vec1))
        #for ind in range(ndxLen):
	#    self.assertEqual(expI[ind], vec1[ind])

    def test_indexing_LHS_SpParVec_scalar_scalar(self):
	sz = 18
        vec1 = SpParVec.range(int(-math.floor(sz/2.0)),int(math.ceil(sz/2.0)))
        ndx = 11
        value = 777
        vec1[ndx] = value
        expI = [-9, -8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 777, 3, 4, 5, 6, 7, 8]
	self.assertEqual(sz, len(vec1))
	self.assertEqual(len(expI), len(vec1))
        for ind in range(sz):
	    self.assertEqual(expI[ind], vec1[ind])

    def test_indexing_LHS_SpParVec_booleanSpParVec_scalar(self):
	sz = 18
        vec1 = SpParVec.range(int(-math.floor(sz/2.0)),int(math.ceil(sz/2.0)))
	ndxLen = 18
	ndxTrue = 4
        ndxI = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]
        ndxV = [0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0]
        ndx = self.initializeSpParVec(ndxLen, ndxI, ndxV)
        value = 777
	self.assertRaises(KeyError,SpParVec.__setitem__,vec1,ndx, value)
        #vec1[ndx] = value
        #expI = [0, 777, 4, 9, 777, 25, 36, 49, 64, 777, 100, 121, 144, 169,
	#	196, 225, 777, 289]
	#self.assertEqual(ndxLen, len(vec1))
        #for ind in range(ndxLen):
	#    self.assertEqual(expI[ind], vec1[ind])

    def test_indexing_LHS_SpParVec_booleanSpParVec_SpParVec(self):
	sz = 18
        vec1 = SpParVec.range(int(-math.floor(sz/2.0)),int(math.ceil(sz/2.0)))
	ndxLen = 18
	ndxTrue = 4
        ndxI = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]
        ndxV = [0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0]
        ndx = self.initializeSpParVec(ndxLen, ndxI, ndxV)
        valueV = [0, 1, 7, 7, 0.25, 0, 0, 0, 0, 0.111, 0, 0, 0, 0, 0, 0, 0.0625,
		0, 0, 0, 0, 7, 0, 0, 0]
        value = self.initializeSpParVec(ndxLen, ndxI, valueV)
	self.assertRaises(KeyError,SpParVec.__setitem__,vec1,ndx, value)
        #vec1[ndx] = value
        #expI = [0, 1, 4, 9, 0.25, 25, 36, 49, 64, 0.111, 100, 121, 144, 169,
	#	196, 225, 0.0625, 289]
	#self.assertEqual(ndxLen, len(vec1))
        #for ind in range(ndxLen):
	#    self.assertEqual(expI[ind], vec1[ind])

    def test_indexing_LHS_SpParVec_ParVec(self):
	sz = 18
        vec1 = SpParVec.range(int(-math.floor(sz/2.0)),int(math.ceil(sz/2.0)))
	ndxLen = 4
        ndxI = [0, 1, 2, 3]
        ndxV = [1, 4, 9, 16]
        ndx = self.initializeSpParVec(ndxLen, ndxI, ndxV)
        valueV = [1, 0.25, 0.111, 0.0625]
        value = self.initializeSpParVec(ndxLen, ndxI, valueV)
	self.assertRaises(IndexError,SpParVec.__setitem__,vec1,ndx, value)
        #vec1[ndx] = value
        #expI = [0, 1, 4, 9, 0.25, 25, 36, 49, 64, 0.111, 100, 121, 144, 169,
	#	196, 225, 0.0625, 289]
	#self.assertEqual(ndxLen, len(vec1))
        #for ind in range(ndxLen):
	#    self.assertEqual(expI[ind], vec1[ind])

    def test_eq_scalar(self):
	sz = 18
        vec1 = SpParVec.range(sz)
        scalar = 8
	eq8 = vec1 == scalar
        expV = [0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]
	self.assertEqual(sz, len(vec1))
	self.assertEqual(len(eq8), len(vec1))
        for ind in range(sz):
	    self.assertEqual(expV[ind], eq8[ind])

    def test_eq_vector(self):
	sz = 18
        vec1 = SpParVec.range(sz)
        vec2 = SpParVec.ones(sz)*4
	eq4 = vec1 == vec2
        expV = [0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
	self.assertEqual(sz, len(vec1))
	self.assertEqual(len(eq4), len(vec1))
	self.assertEqual(len(eq4), len(vec2))
        for ind in range(sz):
	    self.assertEqual(expV[ind], eq4[ind])

    def test_ne_scalar(self):
	sz = 18
        vec1 = SpParVec.range(sz)
        scalar = 4
	ne4 = vec1 != scalar
        expV = [1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
	self.assertEqual(sz, len(vec1))
	self.assertEqual(len(ne4), len(vec1))
        for ind in range(sz):
	    self.assertEqual(expV[ind], ne4[ind])

    def test_ne_vector(self):
	sz = 18
        vec1 = SpParVec.range(sz)
        vec2 = SpParVec.ones(sz)*4
	ne4 = vec1 != vec2
        expV = [1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
	self.assertEqual(sz, len(vec1))
	self.assertEqual(len(ne4), len(vec1))
	self.assertEqual(len(ne4), len(vec2))
        for ind in range(sz):
	    self.assertEqual(expV[ind], ne4[ind])

    def test_ge_scalar(self):
	sz = 18
        vec1 = SpParVec.range(sz)
        scalar = 4
	ge4 = vec1 >= scalar
        expV = [0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
	self.assertEqual(sz, len(vec1))
	self.assertEqual(len(ge4), len(vec1))
        for ind in range(sz):
	    self.assertEqual(expV[ind], ge4[ind])

    def test_ge_vector(self):
	sz = 18
        vec1 = SpParVec.range(sz)
        vec2 = SpParVec.ones(sz)*4
	ge4 = vec1 >= vec2
        expV = [0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
	self.assertEqual(sz, len(vec1))
	self.assertEqual(len(ge4), len(vec1))
	self.assertEqual(len(ge4), len(vec2))
        for ind in range(sz):
	    self.assertEqual(expV[ind], ge4[ind])

    def test_gt_scalar(self):
	sz = 18
        vec1 = SpParVec.range(sz)
        scalar = 4
	gt4 = vec1 > scalar
        expV = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
	self.assertEqual(sz, len(vec1))
	self.assertEqual(len(gt4), len(vec1))
        for ind in range(sz):
	    self.assertEqual(expV[ind], gt4[ind])

    def test_gt_vector(self):
	sz = 18
        vec1 = SpParVec.range(sz)
        vec2 = SpParVec.ones(sz)*4
	gt4 = vec1 > vec2
        expV = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
	self.assertEqual(sz, len(vec1))
	self.assertEqual(len(gt4), len(vec1))
	self.assertEqual(len(gt4), len(vec2))
        for ind in range(sz):
	    self.assertEqual(expV[ind], gt4[ind])

    def test_le_scalar(self):
	sz = 18
        vec1 = SpParVec.range(sz)
        scalar = 4
	le4 = vec1 <= scalar
        expV = [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
	self.assertEqual(sz, len(vec1))
	self.assertEqual(len(le4), len(vec1))
        for ind in range(sz):
	    self.assertEqual(expV[ind], le4[ind])

    def test_le_vector(self):
	sz = 18
        vec1 = SpParVec.range(sz)
        vec2 = SpParVec.ones(sz)*4
	le4 = vec1 <= vec2
        expV = [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
	self.assertEqual(sz, len(vec1))
	self.assertEqual(len(le4), len(vec1))
	self.assertEqual(len(le4), len(vec2))
        for ind in range(sz):
	    self.assertEqual(expV[ind], le4[ind])

    def test_lt_scalar(self):
	sz = 18
        vec1 = SpParVec.range(sz)
        scalar = 4
	lt4 = vec1 < scalar
        expV = [1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
	self.assertEqual(sz, len(vec1))
	self.assertEqual(len(lt4), len(vec1))
        for ind in range(sz):
	    self.assertEqual(expV[ind], lt4[ind])

    def test_lt_vector(self):
	sz = 18
        vec1 = SpParVec.range(sz)
        vec2 = SpParVec.ones(sz)*4
	lt4 = vec1 < vec2
        expV = [1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
	self.assertEqual(sz, len(vec1))
	self.assertEqual(len(lt4), len(vec1))
	self.assertEqual(len(lt4), len(vec2))
        for ind in range(sz):
	    self.assertEqual(expV[ind], lt4[ind])

class GeneralPurposeTests(SpParVecTests):
    def test_all_all_nonnull_all_true(self):
	sz = 10
	vec = SpParVec.ones(sz)
	res = vec.all()
	self.assertEqual(True, res)

    def test_all_all_nonnull_one_false(self):
	sz = 10
	vec = SpParVec.ones(sz)
	vec[4] = False
	res = vec.all()
	self.assertEqual(False, res)

    def test_all_all_nonnull_one_true(self):
	sz = 10
	vec = SpParVec.ones(sz)-1
	vec[4] = True
	res = vec.all()
	self.assertEqual(False, res)

    def test_all_all_nonnull_all_false(self):
	sz = 10
	vec = SpParVec.ones(sz)-1
	res = vec.all()
	self.assertEqual(False, res)

    def test_all_one_nonnull_one_false(self):
	sz = 10
	vec = SpParVec(sz)
	vec[4] = 0
	res = vec.all()
	self.assertEqual(False, res)

    def test_all_one_nonnull_one_true(self):
	sz = 10
	vec = SpParVec(sz)
	vec[4] = 1
	res = vec.all()
	self.assertEqual(True, res)

    def test_all_three_nonnull_one_true(self):
	sz = 10
	vec = SpParVec(sz)
	vec[0] = 0
	vec[2] = 0
	vec[4] = 1
	res = vec.all()
	self.assertEqual(False, res)

    def test_all_three_nonnull_three_true(self):
	sz = 10
	vec = SpParVec(sz)
	vec[0] = 1
	vec[2] = 1
	vec[4] = 1
	res = vec.all()
	self.assertEqual(True, res)

    def test_any_all_true(self):
	sz = 10
	vec = SpParVec.ones(sz)
	res = vec.any()
	self.assertEqual(True, res)

    def test_any_one_false(self):
	sz = 10
	vec = SpParVec.ones(sz)
	vec[4] = False
	res = vec.any()
	self.assertEqual(True, res)

    def test_any_one_true(self):
	sz = 10
	vec = SpParVec.ones(sz)-1
	vec[4] = True
	res = vec.any()
	self.assertEqual(True, res)

    def test_any_all_false(self):
	sz = 10
	vec = SpParVec.ones(sz)-1
	res = vec.any()
	self.assertEqual(False, res)

    def test_sum_nulls(self):
	sz = 10
	vec = SpParVec(sz)
	res = vec.sum()
	self.assertEqual(0, res)

    def test_sum_ones(self):
	sz = 10
	vec = SpParVec.ones(sz)
	res = vec.sum()
	self.assertEqual(sz, res)

    def test_sum_range(self):
	sz = 10
	vec = SpParVec.range(sz)
	res = vec.sum()
	self.assertEqual((sz*(sz-1))/2, res)

    def test_sum_range2(self):
	sz = 11
	vec = SpParVec.range(-(sz/2), (sz/2)+1)
	res = vec.sum()
	self.assertEqual(0, res)

    def test_sum_fixed(self):
	sz = 11
	vec = SpParVec(sz)
	vec[2] = 23
	vec[4] = 9
	vec[5] = -32
	res = vec.sum()
	self.assertEqual(0, res)

    def test_abs_range(self):
	sz = 11
	vec = SpParVec.range(-(sz/2), (sz/2)+1)
	res = abs(vec)
	self.assertEqual(sz, len(res))
	expectedRes = [5, 4, 3, 2, 1, 0, 1, 2, 3, 4, 5]
        for ind in range(sz):
	    self.assertEqual(expectedRes[ind], res[ind])

class MixedDenseSparseVecTests(SpParVecTests):
    def test_add_sparse_dense(self):
	sz = 25
        i = [0, 2, 4, 6, 8, 10]
        v = [0, 4, 8,12,16, 20]
        vec = self.initializeSpParVec(sz, i, v)
        i2 = [0, 1,   3,  5, 7]
        v2 = [0,-2, 0.1,777, 0]
	vec2 = self.initializeParVec(sz, i2, v2)
        vec3 = vec + vec2
        expI = [0, -2, 4, 0.1, 8, 777, 12, 0, 16, 0, 20, 0, 0, 0, 0, 0, 0,
		0, 0, 0, 0, 0, 0, 0, 0]
	self.assertEqual(sz, len(vec3))
	# odd behavior here; element 0 in vec1 and element 7 in vec2 become
	# nulls in vec3 (!)
        for ind in range(sz):
	    self.assertEqual(expI[ind], vec3[ind])

    def test_add_dense_sparse(self):
	sz = 25
        i = [0, 2, 4, 6, 8, 10]
        v = [0, 4, 8,12,16, 20]
        vec = self.initializeSpParVec(sz, i, v)
        i2 = [0, 1,   3,  5, 7]
        v2 = [0,-2, 0.1,777, 0]
	vec2 = self.initializeParVec(sz, i2, v2)
        vec3 = vec2 + vec
        expI = [0, -2, 4, 0.1, 8, 777, 12, 0, 16, 0, 20, 0, 0, 0, 0, 0, 0,
		0, 0, 0, 0, 0, 0, 0, 0]
	self.assertEqual(sz, len(vec3))
	#self.assertEqual(len(i)+len(i2)-1, vec3.nnn())
        for ind in range(sz):
	    self.assertEqual(expI[ind], vec3[ind])

    def test_subtract_sparse_dense(self):
	sz = 25
        i = [0, 2, 4, 6, 8, 10]
        v = [0, 4, 8,12,16, 20]
        vec = self.initializeSpParVec(sz, i, v)
        i2 = [0, 1,   3,  5, 7]
        v2 = [0,-2, 0.1,777, 0]
	vec2 = self.initializeParVec(sz, i2, v2)
        vec3 = vec - vec2
        expI = [0, 2, 4, -0.1, 8, -777, 12, 0, 16, 0, 20, 0, 0, 0, 0, 0, 0,
		0, 0, 0, 0, 0, 0, 0, 0]
	self.assertEqual(sz, len(vec3))
	# odd behavior here; element 0 in vec1 and element 7 in vec2 become
	# nulls in vec3 (!)
        for ind in range(sz):
	    self.assertEqual(expI[ind], vec3[ind])

    def test_isubtract_sparse_dense(self):
	sz = 25
        i = [0, 2, 4, 6, 8, 10]
        v = [0, 4, 8,12,16, 20]
        vec = self.initializeSpParVec(sz, i, v)
        i2 = [0, 1,   3,  5, 7]
        v2 = [0,-2, 0.1,777, 0]
	vec2 = self.initializeParVec(sz, i2, v2)
	vec3 = vec.copy()
        vec3 -= vec2
        expI = [0, 2, 4, -0.1, 8, -777, 12, 0, 16, 0, 20, 0, 0, 0, 0, 0, 0,
		0, 0, 0, 0, 0, 0, 0, 0]
	self.assertEqual(sz, len(vec3))
	# odd behavior here; element 0 in vec1 and element 7 in vec2 become
	# nulls in vec3 (!)
        for ind in range(sz):
	    self.assertEqual(expI[ind], vec3[ind])

    def test_subtract_dense_sparse(self):
	sz = 25
        i = [0, 2, 4, 6, 8, 10]
        v = [0, 4, 8,12,16, 20]
        vec = self.initializeSpParVec(sz, i, v)
        i2 = [0, 1,   3,  5, 7]
        v2 = [0,-2, 0.1,777, 0]
	vec2 = self.initializeParVec(sz, i2, v2)
        vec3 = vec2 - vec
        expI = [0, -2, -4, 0.1, -8, 777, -12, 0, -16, 0, -20, 0, 0, 0, 0, 0, 0,
		0, 0, 0, 0, 0, 0, 0, 0]
	self.assertEqual(sz, len(vec3))
	#self.assertEqual(len(i)+len(i2)-1, vec3.nnn())
        for ind in range(sz):
	    self.assertEqual(expI[ind], vec3[ind])

class ApplyReduceTests(SpParVecTests):
    def test_apply(self):
	def ge0lt5(x):
		return x>=0 and x<5
	sz = 25
        i = [0, 2, 4, 6, 8, 10]
        v = [0, 4, 8,12,16, 20]
        vec = self.initializeSpParVec(sz, i, v)
        vec.apply(ge0lt5)
        vecExpected = [1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
	self.assertEqual(sz, len(vec))
        for ind in range(sz):
	    self.assertEqual(vecExpected[ind], vec[ind])

    def test_apply_pcbabs(self):
	sz = 25
        i = [0, 2,  4,   6, 8, 10]
        v = [0, -4, 8, -12,16, 20]
        vec = self.initializeSpParVec(sz, i, v)
        vec.apply(pcb.abs())
        vecExpected = [0, 0, 4, 0, 8, 0, 12, 0, 16, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
	self.assertEqual(sz, len(vec))
        for ind in range(sz):
	    self.assertEqual(vecExpected[ind], vec[ind])

    def test_count(self):
	sz = 25
        i = [0, 2, 4, 6, 8, 10]
        v = [0, 4, 8,12,16, 20]
        vec = self.initializeSpParVec(sz, i, v)
        ct = vec.count()
        ctExpected = 6
	self.assertEqual(ctExpected, ct)

    def test_reduce_default_op(self):
	sz = 25
        i = [0, 2, 4, 6, 8, 10]
        v = [0, 4, 8,12,16, 20]
        vec = self.initializeSpParVec(sz, i, v)
        red = vec.reduce()
        redExpected = 60
	self.assertEqual(redExpected, red)

    def test_reduce_max(self):
	sz = 25
        i = [0, 2, 4, 6, 8, 10]
        v = [0, 4, 8,12,16, 20]
        vec = self.initializeSpParVec(sz, i, v)
        red = vec.reduce(SpParVec.op_max)
        redExpected = 20
	self.assertEqual(redExpected, red)

def runTests(verbosity = 1):
    testSuite = suite()
    unittest.TextTestRunner(verbosity=verbosity).run(testSuite)

def suite():
    suite = unittest.TestSuite()
    suite.addTests(unittest.TestLoader().loadTestsFromTestCase(ConstructorTests))
    suite.addTests(unittest.TestLoader().loadTestsFromTestCase(BuiltInTests))
    suite.addTests(unittest.TestLoader().loadTestsFromTestCase(GeneralPurposeTests))
    suite.addTests(unittest.TestLoader().loadTestsFromTestCase(MixedDenseSparseVecTests))
    suite.addTests(unittest.TestLoader().loadTestsFromTestCase(ApplyReduceTests))
    return suite

if __name__ == '__main__':
    runTests()

max_stars_repo_name: shoaibkamil/OLD-kdt-specializer

max_issues_repo_name: shoaibkamil/OLD-kdt-specializer

max_forks_repo_name: shoaibkamil/OLD-kdt-specializer


text:
import numpy as np
import visgeom as vg
import matplotlib
import matplotlib.pyplot as plt
from pylie import SO3, SE3

# Define the pose of a camera.
T_w_c = SE3((SO3.from_roll_pitch_yaw(5*np.pi/4, 0, np.pi/2), np.array([[2, 2, 2]]).T))

# Perturbation.
xsi_vec = np.array([[2, 0, 0, 0, 0, 0]]).T

# Perform the perturbation on the right (locally).
T_r = T_w_c @ SE3.Exp(xsi_vec)

# Perform the perturbation on the left (globally).
T_l = SE3.Exp(xsi_vec) @ T_w_c

# We can transform the tangent vector between local and global frames using the adjoint matrix.
xsi_vec_w = T_w_c.adjoint() @ xsi_vec               # When xsi_vec is used on the right side.
xsi_vec_c = T_w_c.inverse().adjoint() @ xsi_vec     # When xsi_vec is used on the left side.
print('xsi_vec_w: ', xsi_vec_w.flatten())
print('xsi_vec_c: ', xsi_vec_c.flatten())

# Visualise the perturbations:
# Use Qt 5 backend in visualisation, use latex.
matplotlib.use('qt5agg')

# Create figure and axis.
fig = plt.figure()
ax = plt.axes(projection='3d')
ax.set_xlabel('x')
ax.set_ylabel('y')
ax.set_zlabel('z')

# Plot frames.
to_right = np.hstack((T_w_c.translation, T_r.translation))
ax.plot(to_right[0, :], to_right[1, :], to_right[2, :], 'k:')
to_left = np.hstack((T_w_c.translation, T_l.translation))
ax.plot(to_left[0, :], to_left[1, :], to_left[2, :], 'k:')
vg.plot_pose(ax, SE3().to_tuple(), text='$\mathcal{F}_w$')
vg.plot_pose(ax, T_w_c.to_tuple(), text='$\mathcal{F}_c$')
vg.plot_pose(ax, T_r.to_tuple(), text='$\mathbf{T}_{wc} Exp(xi)$')
vg.plot_pose(ax, T_l.to_tuple(), text='$Exp(xi) \mathbf{T}_{wc}$')

# Show figure.
vg.plot.axis_equal(ax)
plt.show()

ext: py

max_stars_repo_name: martiege/pylie

max_issues_repo_name: martiege/pylie

max_forks_repo_name: martiege/pylie

reason: import numpy


text:
import numpy as np
import pandas as pd
import scipy.sparse
import sparse
import sklearn
from sklearn.ensemble import RandomForestRegressor
from collections import Counter
import sys, os

import smooth_rf

def _rosenbrock(vec, a=1, b=100):
    """
    Returns Rosenbrock function, a non-convex function, whose global minimum
    is at (a,a^2).

    f(x,y) = (a-x)^2 + b(y-x^2)^2

    Arguments:
    ----------
    vec : array (2,)
        2d array containing (x,y)
    a : scalar
    b : scalar

    Returns:
    --------
    rosenbrock : scalar
        rosenbrock value f(vec[0],vec[1] | a=a, b=b)

    Notes:
        https://en.wikipedia.org/wiki/Rosenbrock_function

    """
    return (a - vec[0])**2 + b * (vec[1] - vec[0]**2)**2
def _d_rosenbrock(vec, a=1, b=100):
    """
    Returns derivative of Rosenbrock function, a non-convex function, whose
    global minimum is at (a,a^2).

    f(x,y) = (a-x)^2 + b(y-x^2)^2
    grad f(x,y) = ( -2*(a-x) + 4bx(y-x^2), 2b(y-x^2) )

    Arguments:
    ----------
    vec : array
        2d array containing (x,y)
    a : scalar
    b : scalar

    Returns:
    --------
    grad : array (2,)
        gradient of rosenbrock grad f(vec[0],vec[1] | a=a, b=b)

    Notes:
        https://en.wikipedia.org/wiki/Rosenbrock_function
    """
    grad = np.array([-2 * (a - vec[0]) - 4 * b * (vec[1]-vec[0]**2) * vec[0],
                    2 * b * (vec[1] - vec[0]**2)])
    return grad

def test_adam_step():
    """
    test for adam step

    similar to
    https://github.com/pytorch/pytorch/blob/master/test/optim/test.py
    """
    d_rosenbrock_lamb = lambda lamb: _d_rosenbrock(lamb,a = 1, b = 100)

    params = [{"alpha": 1e-4},
              {"alpha": 1e-4, "beta_1": 0.92},
              {"alpha": 1e-4, "beta_1": 0.92, "beta_2": 0.96},
              {"alpha": 1e-4, "beta_1": 0.92, "beta_2": 0.96, "eps": 1e-3}]

    for idx, param in enumerate(params):
        #print(param)
        #print("------")
        lamb = np.array([1.5,1.5])
        iv = None
        for step_idx in range(50000):
            lamb, iv = smooth_rf.adam_step(_d_rosenbrock, lamb,
                                 internal_values = iv, **param)
            if step_idx % 10000 == 0:
                pass
                #print('{:.8f}\t{:.8f}\t'.format(lamb[0], lamb[1]))

        if idx > 1:
            assert np.all(np.abs(lamb - np.array([1,1])) < 1e-3), \
                "adam step doesn't seem to coverge close enough to global " +\
                "optimal"

reason: import numpy,import scipy


text:
# Copyright 2020 Evgenij Grigorev evgengrmit@icloud.com

import keras
from keras.models import Sequential
from keras.layers import Dense
from sklearn.model_selection import train_test_split
import numpy as np
from detector import Preprocess as pp
from models import DatasetHandler as dh

class KerasModel:
    def __init__(self):
        self._preprocess = pp.Preprocess()
        self._model = keras.models.load_model('models/Saving/keras_model')
        self._train_size = 0.7
        self._metrics = [0.05013519152998924, 0.9858478903770447, 0.997950553894043]

    @property
    def model(self):
        return self._model

    @staticmethod
    def build_model(n_features, n_classes, num_layers=3):
        model = Sequential()
        delta = np.power(n_features / n_classes, 1 / num_layers)
        n_2 = np.int(4 * n_features / delta)
        n_3 = np.int(3 * n_2 / delta)
        model.add(Dense(n_2, input_dim=n_features, activation='relu'))
        model.add(Dense(n_3, activation='relu'))
        model.add(Dense(n_classes, activation='sigmoid'))
        return model

    def set_new_model(self, keras_model=""):
        if keras_model == '':
            raise IOError("No path to model")
        self._model = keras.models.load_model(keras_model)

    def set_pool(self, path_to_dataset=''):
        if path_to_dataset != '':
            self._preprocess.set_dataset(path_to_dataset)
            self._x, self._y = self._preprocess.process_data_for_neural_with_label()

    def get_predict_with_label(self, path_to_data=''):
        if path_to_data == '':
            raise IOError("No path to data")
        self._preprocess.set_dataset(path_to_data)
        self._x, self._y = self._preprocess.process_data_for_neural_with_label()
        return dh.massive(self._model.predict_classes(self._x))

    def relearn_model(self, path_to_dataset='', test_size=0.2):
        if path_to_dataset == '':
            raise IOError("No path to dataset")
        self.set_pool(path_to_dataset=path_to_dataset)
        self._train_size = 1 - test_size
        x_train, x_test, y_train, y_test = train_test_split(self._x, self._y, train_size=self._train_size)
        n_features = self._x.shape[1]
        n_cl = 1  # 2^1 =2
        self._model = KerasModel.build_model(n_features=n_features, n_classes=n_cl)
        self._model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy', 'AUC'])
        self._model.fit(x_train, y_train, validation_data=(x_test, y_test), epochs=450, batch_size=10000)
        self._metrics = self._model.evaluate(x_test, y_test)

    def get_test_log_loss(self):
        return self._metrics[0]

    def get_test_accuracy(self):
        return self._metrics[1]

    def get_test_auc(self):
        return self._metrics[2]

    def get_predict_unknown(self, path_to_data=''):
        if path_to_data == '':
            raise IOError("No path to data")
        self._preprocess.set_dataset(path_to_data)
        x = self._preprocess.get_data_for_predict_neural()
        return dh.massive(self._model.predict_classes(x))

max_stars_repo_name: Evgengrmit/course-project

max_issues_repo_name: Evgengrmit/course-project

max_forks_repo_name: Evgengrmit/course-project


text:
# Copyright (c) 2021, Michael Boyle
# See LICENSE file for details: <https://github.com/moble/spherical/blob/master/LICENSE>

### NOTE: The functions in this file are intended purely for inclusion in the Grid class.  In
### particular, they assume that the first argument, `self` is an instance of Grid.  They should
### probably not be used outside of that class.

def modes(self, ell_max=None, **kwargs):
    """Return mode weights of function decomposed into SWSHs

    The output array has one less dimension than this object; rather than the last
    two axes giving the values on the two-dimensional grid, the last axis gives the
    mode weights.

    Parameters
    ----------
    ell_max : {None, int}, optional
        Maximum ell value in the output.  If None, the result will have enough ell
        values to express the data on the grid without aliasing: (max(n_phi,
        n_theta) - 1) // 2.
    **kwargs : Any
        Additional keyword arguments are passed through to the Modes constructor on
        output

    """
    raise NotImplementedError()
    # import copy
    # import numpy as np
    # import spinsfast
    # from .. import Modes
    # ell_max = ell_max or (max(n_phi, n_theta) - 1) // 2
    # metadata = copy.copy
    # return Modes(spinsfast.map2salm(self.view(np.ndarray), self.spin_weight, ell_max),
    #              spin_weight=self.spin_weight, ell_min=0, ell_max=ell_max, **metadata)

def _check_broadcasting(self, array, reverse=False):
    """Test whether or not the given array can broadcast against this object"""
    import numpy as np

    if isinstance(array, type(self)):
        try:
            if reverse:
                np.broadcast(array, self)
            else:
                np.broadcast(self, array)
        except ValueError:
            return False
        else:
            return True
    else:
        if np.ndim(array) > np.ndim(self)-2:
            raise ValueError(f"Cannot broadcast array of {np.ndim(array)} dimensions against {type(self).__name__} "
                             f"object of fewer ({np.ndim(self)-2}) non-grid dimensions.\n"
                             "This is to ensure that scalars do not operate on individual "
                             "grid values; they must operate on all simultaneously.\n"
                             "If that is the case and you still want to broadcast, add more "
                             "dimensions before this object's first dimension.")
        try:
            if reverse:
                np.broadcast(array, self[..., 0, 0])
            else:
                np.broadcast(self[..., 0, 0], array)
        except ValueError:
            return False
        else:
            return True

max_stars_repo_name: moble/spherical

max_issues_repo_name: moble/spherical

max_forks_repo_name: moble/spherical


text:
from utils import *
import numpy as np
from sklearn.model_selection import KFold
from skorch import NeuralNetClassifier
from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score

data = {
    'NA': loadData("data/no_anomaly/"),
    'A': loadData("data/anomaly/")
}

device = 'cuda'

seeds = range(33)

cv = KFold(3, shuffle=True, random_state=1)

events = np.array(list(data["A"].keys()))

accAll = []
f1All = []
prAll = []
recAll = []

k=0
for a,b in cv.split(events):

    k += 1

    accs = []
    f1s = []
    prs = []
    recs = []

    for s in seeds:

        clf = NeuralNetClassifier(
            RNNClassifier,
            verbose=0,
            iterator_train__shuffle=False,
            batch_size=1000,
            optimizer=torch.optim.Adam,
            train_split=None,
            module__input_dim = 13,
            device = device,
            lr = 0.1,
            max_epochs = 50,
            module__dropout = 0.25,
            module__num_layers = 1,
            module__num_units = 32,
            module__seed = s,
            optimizer__weight_decay = 0,
        )

        e_train = list(events[a])
        e_test = list(events[b])

        X_train = []
        Y_train = []
        X_test = []
        Y_test = []

        for e in events:

            X_0 = data["A"][e][:60]
            X_1 = data["NA"][e][:60]

            if e in e_test:
                X_test.append(X_0)
                X_test.append(X_1)
                Y_test += [0,1]
            if e in e_train:
                X_train.append(X_0)
                X_train.append(X_1)
                Y_train += [0,1]

        X_train = np.array(X_train)
        Y_train = np.array(Y_train)
        X_test = np.array(X_test)
        Y_test = np.array(Y_test)

        # Train

        clf.fit(X_train, Y_train)

        # Test

        pred_test = clf.predict(X_test)
        accs.append(accuracy_score(Y_test, pred_test))
        f1s.append(f1_score(Y_test, pred_test, zero_division=0))
        prs.append(precision_score(Y_test, pred_test, zero_division=0))
        recs.append(recall_score(Y_test, pred_test, zero_division=0))

    print("########")
    print("Fold",k)
    print("Accuracy: mean:%.2f, std:%.2f"%(np.mean(accs), np.std(accs)))
    print("F1 score: mean:%.2f, std:%.2f"%(np.mean(f1s), np.std(f1s)))
    print("Precision: mean:%.2f, std:%.2f"%(np.mean(prs), np.std(prs)))
    print("Recall: mean:%.2f, std:%.2f"%(np.mean(recs), np.std(recs)))

    accAll += accs
    f1All += f1s
    prAll += prs
    recAll += recs

print("########")
print("All")
print("Accuracy: mean:%.2f, std:%.2f"%(np.mean(accAll), np.std(accAll)))
print("F1 score: mean:%.2f, std:%.2f"%(np.mean(f1All), np.std(f1All)))
print("Precision: mean:%.2f, std:%.2f"%(np.mean(prAll), np.std(prAll)))
print("Recall: mean:%.2f, std:%.2f"%(np.mean(recAll), np.std(recAll)))


text:
import tensorflow as tf
import sys
import os
import configparser

dir_path = os.path.dirname(os.path.realpath(__file__))
sys.path.append(os.path.join(dir_path, "../"))
from utils import convrnn
from tensorflow.contrib.rnn import LSTMStateTuple

from tensorflow.contrib.layers import batch_norm, flatten
from tensorflow.contrib.framework import arg_scope

MODEL_GRAPH_NAME = "graph.meta"
MODEL_CFG_FILENAME = "params.ini"
ADVANCED_SUMMARY_COLLECTION_NAME = "advanced_summaries"

tf.app.flags.DEFINE_string("modelfolder", None, "target location of graph on disk")

## hyper parameters ##
tf.app.flags.DEFINE_string("kernel", "(1,3,3)", "kernel of convolutions")
tf.app.flags.DEFINE_string("classkernel", "(3,3)", "kernelsize of final classification convolution")
tf.app.flags.DEFINE_string("cnn_activation", "leaky_relu",
                           "activation function for convolutional layers ('relu' or 'leaky_relu' [default])")

tf.app.flags.DEFINE_boolean("bidirectional", True, "Bidirectional Convolutional RNN")
tf.app.flags.DEFINE_integer("convrnn_compression_filters", -1,
                            "number of convrnn compression filters or (default) -1 for no compression")
tf.app.flags.DEFINE_string("convcell", "gru", "Convolutional RNN cell architecture ('gru' (default) or 'lstm')")
tf.app.flags.DEFINE_string("convrnn_kernel", "(3,3)", "kernelsize of recurrent convolution. default (3,3)")
tf.app.flags.DEFINE_integer("convrnn_filters", 24, "number of convolutional filters in ConvLSTM/ConvGRU layer")
tf.app.flags.DEFINE_float("recurrent_dropout_i", 1.,
                          "input keep probability for recurrent dropout (default no dropout -> 1.)")
tf.app.flags.DEFINE_float("recurrent_dropout_c", 1.,
                          "state keep probability for recurrent dropout (default no dropout -> 1.)")
tf.app.flags.DEFINE_integer("convrnn_layers", 1, "number of convolutional recurrent layers")
tf.app.flags.DEFINE_boolean("peephole", False,
                            "use peephole connections at convrnn layer. only for lstm (default False)")
tf.app.flags.DEFINE_boolean("convrnn_normalize", True, "normalize with batchnorm at convrnn layer (default True)")
tf.app.flags.DEFINE_string("aggr_strat", "state",
                           "aggregation strategie to reduce temporal dimension (either default 'state' or 'sum_output' or 'avg_output')")

tf.app.flags.DEFINE_float("learning_rate", 0.01, "Adam learning rate")
tf.app.flags.DEFINE_float("beta1", 0.9, "Adam beta1")
tf.app.flags.DEFINE_float("beta2", 0.999, "Adam beta2")
tf.app.flags.DEFINE_float("epsilon", 0.9, "Adam epsilon")

## expected data format ##
tf.app.flags.DEFINE_string("expected_datatypes",
                           "(tf.float32, tf.float32, tf.float32, tf.float32, tf.float32, tf.uint8)",
                           "expected datatypes")
tf.app.flags.DEFINE_integer("pix10m", 24, "number of 10m pixels")
tf.app.flags.DEFINE_integer("num_bands_10m", 4, "number of bands in 10 meter resolution (4)")
tf.app.flags.DEFINE_integer("num_bands_20m", 4, "number of bands in 20 meter resolution (6)")
tf.app.flags.DEFINE_integer("num_bands_60m", 3, "number of bands in 20 meter resolution (3)")
tf.app.flags.DEFINE_integer("num_classes", 13,
                            "number of classes not counting unknown class -> e.g. 0:uk,1:a,2:b,3:c,4:d -> num_classes 4")

## performance ##
tf.app.flags.DEFINE_boolean("swap_memory", True, "Swap memory between GPU and CPU for recurrent layers")

FLAGS = tf.app.flags.FLAGS

class Model():
    def __init__(self):

        # used to feed training/testing or evaluation iterator as string representation
        self.iterator_handle = tf.placeholder(tf.string, shape=[], name="data_iterator_handle")
        # requires info to data dimensions
        # self.output_shapes = tuple(output_shapes)
        # self.output_types = tuple(output_types)

        # changes behavior of batch normalization
        self.is_train = tf.placeholder_with_default(tf.cast(True, tf.bool), shape=(), name="is_train")

        def parse(var, dtype):
            if type(var) == dtype:
                return var
            else:
                return eval(var)

        # see FLAGS for documentation
        self.kernelsize = parse(FLAGS.kernel, tuple)
        self.num_classes = parse(FLAGS.num_classes, int)

    def build_graph(self, modelpath=None):

        # input pipeline
        print("building input pipeline...")

        # input pipeline
        with tf.name_scope("input"):
            (x, sequence_lengths), (alllabels,) = self.input()
            self.alllabels = alllabels

        print("building inference...")
        self.logits = self.inference(input=(x, sequence_lengths))

        # reduce label size to same shape like logits (important for downsampling)
        b, w, h, d = self.logits.get_shape()

        # take first label -> assume labels do not change over timeseries
        # first_labelmap = alllabels[:, 0]
        first_labelmap = alllabels

        # create one-hot labelmap from 0-num_classes
        labels = tf.one_hot(first_labelmap, self.num_classes + 1)

        # mask out class 0 -> unknown
        unknown_mask = tf.cast(labels[:, :, :, 0], tf.bool)
        not_unknown_mask = tf.logical_not(unknown_mask)

        # keep rest of labels
        self.labels = labels[:, :, :, 1:]

        # mask out all classes labeled unknown (labelid=0)
        # not_unknown_mask = tf.not_equal(first_labelmap, tf.constant(0, dtype=tf.int32))

        # self.labels = self.reshape_labels(labels=first_labelmap, width=w, height=h)
        # self.labels = tf.one_hot(tf.reduce_mean(alllabels, axis=1), self.num_classes)

        print("building loss...")
        self.loss = self.loss(logits=self.logits, labels=self.labels, mask=not_unknown_mask, name="loss")

        print("building optimizer...")
        self.train_op = self.optimize(self.loss, name="train_op")

        print("building metrics and summaries...")
        self.metrics(logits=self.logits, labels=self.labels, mask=not_unknown_mask)

        self.summary_op = self.summary()

        n_trainparams = count_params(collector_fun=tf.trainable_variables)
        print("graph completed with {} trainable parameters".format(n_trainparams))

        if modelpath is not None:
            path = os.path.join(modelpath, MODEL_GRAPH_NAME)
            print("writing meta graph to {}".format(path))
            tf.train.export_meta_graph(filename=path)

            # write FLAGS to file
            cfg = configparser.ConfigParser()

            # TF 1.4
            # cfg["model"] = {"model": os.path.basename(__file__)}
            # cfg["flags"] = FLAGS.__dict__["__flags"]

            # TF 1.7
            flags_dict = dict()
            for name in FLAGS:
                flags_dict[name] = str(FLAGS[name].value)

            cfg["flags"] = flags_dict  # FLAGS.__flags #  broke tensorflow=1.5

            path = os.path.join(modelpath, MODEL_CFG_FILENAME)
            print("writing parameters to {}".format(path))
            with open(path, 'w') as configfile:
                cfg.write(configfile)

            # write operation names to file for easier debugging
            path = os.path.join(modelpath, "nodes.txt")
            print("writing nodes list to {}".format(path))
            names = [n.name for n in tf.get_default_graph().as_graph_def().node]
            with open(path, 'w') as f:
                f.write("\n".join(names))

    def input(self):

        # output_shapes = tf.placeholder_with_default(self.output_shapes, [6], name="data_shapes")
        # os = [tf.TensorShape(s) for s in self.output_shapes]

        iterator = tf.data.Iterator.from_string_handle(self.iterator_handle,
                                                       output_types=eval(FLAGS.expected_datatypes))

        with tf.name_scope("raw"):
            x10, x20, x60, doy, year, labels = iterator.get_next()

            self.x10 = tf.cast(x10, tf.float32, name="x10")
            self.x20 = tf.cast(x20, tf.float32, name="x20")
            self.x60 = tf.cast(x60, tf.float32, name="x60")
            self.doy = tf.cast(doy, tf.float32, name="doy")
            self.year = tf.cast(year, tf.float32, name="year")
            self.y = tf.cast(labels, tf.int32, name="y")

        # self.x10.set_shape([2,46,24,24,5])

        # integer sequence lenths per batch for dynamic_rnn masking
        self.seq = sequence_lengths = tf.reduce_sum(tf.cast(self.x10[:, :, 0, 0, 0] > 0, tf.int32), axis=1,
                                                    name="sequence_lengths")

        def resize(tensor, new_height, new_width):
            b = tf.shape(tensor)[0]
            t = tf.shape(tensor)[1]
            h = tf.shape(tensor)[2]
            w = tf.shape(tensor)[3]
            d = tf.shape(tensor)[4]

            # stack batch on times to fit 4D requirement of resize_tensor
            stacked_tensor = tf.reshape(tensor, [b * t, h, w, d])
            reshaped_stacked_tensor = tf.image.resize_images(stacked_tensor, size=(new_height, new_width))
            return tf.reshape(reshaped_stacked_tensor, [b, t, new_height, new_width, d])

        def expand3x(vector):
            vector = tf.expand_dims(vector, -1)
            vector = tf.expand_dims(vector, -1)
            vector = tf.expand_dims(vector, -1)
            return vector

        with tf.name_scope("reshaped"):
            b = tf.shape(self.x10)[0]
            t = tf.shape(self.x10)[1]
            px = tf.shape(self.x10)[2]

            # b,t,w,h,d = self.x10.shape()

            x20 = tf.identity(resize(self.x20, px, px), name="x20")
            x60 = tf.identity(resize(self.x60, px, px), name="x60")

            tf.add_to_collection(ADVANCED_SUMMARY_COLLECTION_NAME, tf.identity(self.x10, name="x10"))
            tf.add_to_collection(ADVANCED_SUMMARY_COLLECTION_NAME, x20)
            tf.add_to_collection(ADVANCED_SUMMARY_COLLECTION_NAME, x60)

            # expand
            doymat = tf.multiply(expand3x(self.doy), tf.ones((b, t, px, px, 1)), name="doy")
            yearmat = self.yearmat = tf.multiply(expand3x(self.year), tf.ones((b, t, px, px, 1)), name="year")

            x = tf.concat((self.x10, x20, x60, doymat, yearmat), axis=-1, name="x")
            tf.add_to_collection(ADVANCED_SUMMARY_COLLECTION_NAME, x)

            # set depth of x for convolutions
            depth = FLAGS.num_bands_10m + FLAGS.num_bands_20m + FLAGS.num_bands_60m + 2  # doy and year

            # dynamic shapes. Fill for debugging

            x.set_shape([None, None, FLAGS.pix10m, FLAGS.pix10m, depth])
            self.y.set_shape((None, FLAGS.pix10m, FLAGS.pix10m))

        return (x, sequence_lengths), (self.y,)

    def inference(self, input):

        x, sequence_lengths = input

        rnn_output_list = list()
        rnn_state_list = list()

        x_rnn = x
        for j in range(1, FLAGS.convrnn_layers + 1):
            convrnn_kernel = eval(FLAGS.convrnn_kernel)
            x_rnn, state = convrnn_layer(input=x_rnn, is_train=self.is_train, filter=FLAGS.convrnn_filters,
                                         kernel=convrnn_kernel,
                                         bidirectional=FLAGS.bidirectional, convcell=FLAGS.convcell,
                                         sequence_lengths=sequence_lengths, scope="convrnn" + str(j))
            rnn_output_list.append(x_rnn)
            rnn_state_list.append(state)

        # concat outputs from cnns and rnns in a dense scheme
        x = tf.concat(rnn_output_list, axis=-1)

        # take concatenated states of last rnn block (might contain multiple conrnn layers)
        state = tf.concat(rnn_state_list, axis=-1)

        # use the cell state as featuremap for the classification step
        # cell state has dimensions (b,h,w,d) -> classification strategy
        if FLAGS.aggr_strat == 'state':
            class_input = state  # shape (b,h,w,d)
            classkernel = eval(FLAGS.classkernel)
            logits = conv_bn_relu(input=class_input, is_train=self.is_train, filter=self.num_classes,
                                  kernel=classkernel, dilation_rate=(1, 1), conv_fun=tf.layers.conv2d,
                                  var_scope="class")

        elif (FLAGS.aggr_strat == 'avg_output') or (FLAGS.aggr_strat == 'sum_output'):
            # last rnn output at each time t
            class_input = x_rnn  # shape (b,t,h,w,d)

            kernel = (1, FLAGS.classkernel[0], FLAGS.classkernel[1])

            # logits for each single timeframe
            logits = conv_bn_relu(input=class_input, is_train=self.is_train, filter=self.num_classes, kernel=kernel,
                                  dilation_rate=(1, 1, 1), conv_fun=tf.layers.conv3d, var_scope="class")

            if FLAGS.aggr_strat == 'avg_output':
                # average logit scores at each observation
                # (b,t,h,w,d) -> (b,h,w,d)
                logits = tf.reduce_mean(logits, axis=1)
            elif FLAGS.aggr_strat == 'sum_output':
                # summarize logit scores at each observation
                # the softmax normalization later will normalize logits again
                # (b,t,h,w,d) -> (b,h,w,d)
                logits = tf.reduce_sum(logits, axis=1)

        else:
            raise ValueError("please provide valid aggr_strat flag ('state' or 'avg_output' or 'sum_output')")

        tf.add_to_collection(ADVANCED_SUMMARY_COLLECTION_NAME, logits)

        return logits

    def loss(self, logits, labels, mask, name):

        loss_per_px = tf.nn.softmax_cross_entropy_with_logits_v2(labels=labels, logits=logits)

        # loss_per_px = tf.boolean_mask(loss_per_px, unknown_mask, name="masked_loss_per_px")
        _ = tf.identity(loss_per_px, name="loss_per_px")
        _ = tf.identity(mask, name="mask_per_px")

        lpp = tf.boolean_mask(loss_per_px, mask)

        return tf.reduce_mean(lpp, name=name)

    def optimize(self, loss, name):

        self.global_step = tf.Variable(0, name='global_step', trainable=False)
        self.samples_seen = tf.Variable(0, name='samples_seen', trainable=False)

        batchsize = tf.shape(self.x10)[0]
        samples_seen_increment_op = tf.assign(self.samples_seen, self.samples_seen + batchsize)

        lr = tf.placeholder_with_default(FLAGS.learning_rate, shape=(), name="learning_rate")
        beta1 = tf.placeholder_with_default(FLAGS.beta1, shape=(), name="beta1")
        beta2 = tf.placeholder_with_default(FLAGS.beta2, shape=(), name="beta2")

        with tf.control_dependencies([samples_seen_increment_op]):  # execute this every time global step is incremented
            optimizer = tf.train.AdamOptimizer(learning_rate=lr, beta1=beta1, beta2=beta2, epsilon=FLAGS.epsilon)
            return optimizer.minimize(loss, global_step=self.global_step, name=name)

    def metrics(self, logits, labels, mask):
        self.prediction_scores = tf.nn.softmax(logits=logits, name="prediction_scores")
        self.predictions = tf.argmax(self.prediction_scores, 3, name="predictions")

        targets = tf.argmax(labels, 3, name="targets")
        # correctly_predicted = tf.equal(self.predictions, targets, name="correctly_predicted")
        correctly_predicted = tf.equal(tf.boolean_mask(self.predictions, mask), tf.boolean_mask(targets, mask),
                                       name="correctly_predicted")
        self.overall_accuracy = tf.reduce_mean(tf.cast(correctly_predicted, tf.float32), name="overall_accuracy")

    def summary(self):
        """
        minimial summaries for training @ monitoring
        """

        tf.summary.scalar("oa", self.overall_accuracy)
        tf.summary.scalar("xe", self.loss)
        ## histograms

        return tf.summary.merge_all()

def convrnn_layer(input, filter, is_train=True, kernel=FLAGS.convrnn_kernel, sequence_lengths=None, bidirectional=True,
                  convcell='gru', scope="convrnn"):
    with tf.variable_scope(scope):

        x = input

        px = x.get_shape()[3]

        if FLAGS.convrnn_compression_filters > 0:
            x = conv_bn_relu(input=x, is_train=is_train, filter=FLAGS.convrnn_compression_filters, kernel=(1, 1, 1),
                             dilation_rate=(1, 1, 1), var_scope="comp")

        tf.add_to_collection(ADVANCED_SUMMARY_COLLECTION_NAME, tf.identity(input, "input"))

        if convcell == 'gru':
            cell = convrnn.ConvGRUCell((px, px), filter, kernel, activation=tf.nn.tanh,
                                       normalize=FLAGS.convrnn_normalize)

            # tf.Variable(tf.zeros((b,h,w)), validate_shape=False, trainable=False, name="zero_state")
            # zero_state = tf.Variable(tf.zeros((None, None, None), tf.float32), trainable=False)
        elif convcell == 'lstm':
            cell = convrnn.ConvLSTMCell((px, px), filter, kernel, activation=tf.nn.tanh,
                                        normalize=FLAGS.convrnn_normalize, peephole=FLAGS.peephole)
        else:
            raise ValueError("convcell argument {} not valid either 'gru' or 'lstm'".format(convcell))

        ## add dropout wrapper to cell
        cell = tf.contrib.rnn.DropoutWrapper(cell, input_keep_prob=FLAGS.recurrent_dropout_i,
                                             state_keep_prob=FLAGS.recurrent_dropout_i)

        if bidirectional:
            outputs, final_states = tf.nn.bidirectional_dynamic_rnn(cell_fw=cell, cell_bw=cell, inputs=x,
                                                                    sequence_length=sequence_lengths,
                                                                    dtype=tf.float32, time_major=False,
                                                                    swap_memory=FLAGS.swap_memory)

            concat_outputs = tf.concat(outputs, -1)
            if convcell == 'gru':
                concat_final_state = tf.concat(final_states, -1)
            elif convcell == 'lstm':
                fw_final, bw_final = final_states
                concat_final_state = LSTMStateTuple(
                    c=tf.concat((fw_final.c, bw_final.c), -1),
                    h=tf.concat((fw_final.h, bw_final.h), -1)
                )
                # else:
                #    concat_final_state = tf.concat((fw_final,bw_final),-1)
        else:
            concat_outputs, concat_final_state = tf.nn.dynamic_rnn(cell=cell, inputs=input,
                                                                   sequence_length=sequence_lengths,
                                                                   dtype=tf.float32, time_major=False)

        if convcell == 'lstm':
            concat_final_state = concat_final_state.c

        tf.add_to_collection(ADVANCED_SUMMARY_COLLECTION_NAME, tf.identity(concat_outputs, name="outputs"))
        tf.add_to_collection(ADVANCED_SUMMARY_COLLECTION_NAME, tf.identity(concat_final_state, name="final_states"))

        return concat_outputs, concat_final_state

def conv_bn_relu(var_scope="name_scope", is_train=True, **kwargs):
    with tf.variable_scope(var_scope):

        if FLAGS.cnn_activation == 'relu':
            activation_function = tf.nn.relu
        elif FLAGS.cnn_activation == 'leaky_relu':
            activation_function = tf.nn.leaky_relu
        else:
            raise ValueError("please provide valid 'cnn_activation' FLAG. either 'relu' or 'leaky_relu'")

        x = conv_layer(**kwargs)
        x = Batch_Normalization(x, is_train, "bn")
        x = activation_function(x)

        tf.add_to_collection(ADVANCED_SUMMARY_COLLECTION_NAME, x)
        return x

def count_params(collector_fun=tf.trainable_variables):
    size = lambda v: reduce(lambda x, y: x * y, v.get_shape().as_list())
    n = sum(size(v) for v in collector_fun())
    return n

def conv_layer(input, filter, kernel, dilation_rate=(1, 1, 1), stride=1, conv_fun=tf.layers.conv3d, layer_name="conv"):
    with tf.name_scope(layer_name):
        # pad input to required sizes for same output dimensions
        input = pad(input, kernel, dilation_rate, padding="REFLECT")

        network = conv_fun(inputs=input, use_bias=False, filters=filter, kernel_size=kernel, strides=stride,
                           padding='VALID', dilation_rate=dilation_rate)

        return network

def Batch_Normalization(x, training, scope):
    with arg_scope([batch_norm],
                   scope=scope,
                   updates_collections=None,
                   decay=0.9,
                   center=True,
                   scale=True,
                   zero_debias_moving_mean=True):
        return tf.cond(training,
                       lambda: batch_norm(inputs=x, is_training=training, reuse=None),
                       lambda: batch_norm(inputs=x, is_training=training, reuse=True))

def pad(input, kernel, dilation, padding="REFLECT"):
    """https://www.tensorflow.org/api_docs/python/tf/pad"""

    # determine required padding sizes
    def padsize(kernel, dilation):
        p = []
        for k, d in zip(kernel, dilation):
            p.append(int(k / 2) * d)
        return p

    padsizes = padsize(kernel, dilation)

    # [bleft,bright], [tleft,tright], [hleft,hright], [wleft,wright],[dleft,dright]
    paddings = tf.constant([[0, 0]] + [[p, p] for p in padsizes] + [[0, 0]])

    return tf.pad(input, paddings, padding)

def test2():
    from Dataset import Dataset
    import matplotlib.pyplot as plt
    import numpy as np

    dataset = Dataset(datadir="/media/data/marc/tfrecords/fields/L1C/240", verbose=True, temporal_samples=30,
                      section='2016')

    training_dataset, output_shapes, output_datatypes, fm_train = dataset.create_tf_dataset("eval", fold=0, batchsize=2,
                                                                                shuffle=False,
                                                                                prefetch_batches=1,
                                                                                num_batches=-1, threads=8,
                                                                                drop_remainder=False)

    iterator = training_dataset.make_initializable_iterator()

    model = Model()
    model.build_graph(modelpath=FLAGS.modelfolder)

    def get_operation(name):
        return tf.get_default_graph().get_operation_by_name(name).outputs[0]

    iterator_handle_op = get_operation("data_iterator_handle")
    is_train_op = get_operation("is_train")

    mpp_op = get_operation("mask_per_px")
    lpp_op = get_operation("loss_per_px")
    # umlpp_op = get_operation("unmasked_loss_per_px")

    with tf.Session() as sess:
        sess.run([tf.global_variables_initializer(), tf.local_variables_initializer()])

        sess.run([iterator.initializer, tf.tables_initializer()])

        data_handle = sess.run(iterator.string_handle())

        feed_dict = {iterator_handle_op: data_handle, is_train_op: False}

        y, labels, alllabels, mpp, lpp = sess.run([model.y, model.labels, model.alllabels, mpp_op, lpp_op],
                                                  feed_dict=feed_dict)
        # x10, x20, x60, doy, year, labels = sess.run(iterator.get_next())
        a = sess.run(tf.one_hot(alllabels - 1, model.num_classes))

        plt.imshow(y[0, 0])

def test(_):
    FLAGS.pix10 = 24
    FLAGS.modelfolder = "tmp"
    FLAGS.convcell = "lstm"
    model = Model()
    model.build_graph(modelpath=FLAGS.modelfolder)

def main(_):
    # test2()

    model = Model()
    model.build_graph(modelpath=FLAGS.modelfolder)

if __name__ == '__main__':
    tf.app.run()

max_stars_repo_name: jnyborg/MTLCC

max_issues_repo_name: jnyborg/MTLCC

max_forks_repo_name: jnyborg/MTLCC


text:
from carla_utils import carla

import numpy as np

from .state import State
from .waypoint import Waypoint

class CarlaTransform(object):
    @staticmethod
    def cua_state(state: State):
        x, y, theta = state.x, state.y, state.theta
        location = carla.Location(x=x, y=y)
        transform = carla.Transform(location)
        transform.rotation.yaw = np.rad2deg(theta)
        return transform

class CUAState(object):
    @staticmethod
    def carla_transform(transform, **kwargs):
        """

        Args:
            transform: carla.Transform, need to be valid or real
            kwargs: contains except x, y, z, theta

        Returns:
            carla_utils.augment.State
        """

        location, rotation = transform.location, transform.rotation
        x, y, z = location.x, location.y, location.z
        theta = np.deg2rad(rotation.yaw)
        kwargs.update({'x':x, 'y':y, 'z':z, 'theta':theta})
        return State(**kwargs)

max_stars_repo_name: IamWangYunKai/DG-TrajGen


text:
import numpy as np
import pandas as pd
import torch
from torch import nn

predicate = sorted(pd.read_json('data/all_50_schemas', lines=True)['predicate'].values.tolist())
pred2idx = {p: i for i, p in enumerate(predicate)}
idx2pred = {i: p for i, p in enumerate(predicate)}

class Dataset:
    def __init__(self, df):
        self.df = df[['text', 'spo_list']]

    def __len__(self):
        return self.df.__len__()

    def __getitem__(self, item):
        text, spo_list = self.df.iloc[item]
        target = self.data_fn(text, spo_list)

        object_labels_num = len(target['object_labels'])
        # return object_labels_num
        # print(object_labels_num)
        if object_labels_num == 0:
            return None
        else:
            idx = np.random.choice(object_labels_num)
            sample = target['object_labels'][idx]
            return {
                "text": text,
                "subject_label": torch.tensor(target['subject_label'], dtype=torch.float),
                "subject_idx": torch.tensor(sample[0], dtype=torch.long),
                "object_label": torch.tensor(sample[1], dtype=torch.float),
                "spo_list": target['spo_list']
            }

    @staticmethod
    def data_fn(text, spo_list) -> dict:
        """
        生成subject 与 Predict_Object对应的index_span tensor
        :param text: str
        :param spo_list: list
        :return: dict(subject_label:numpy.array, object_labels: list[tuple])
        """
        text = text.lower()
        subject = np.zeros((len(text), 2))
        object = {}
        for spo in spo_list:
            sub_str = spo['subject'].lower()
            ob_str = spo['object'].lower()
            sub_start_idx = text.find(sub_str)
            if sub_start_idx == -1:
                continue
            sub_idx = (sub_start_idx, sub_start_idx + len(sub_str))
            subject[sub_idx[0], 0] = 1
            subject[sub_idx[1] - 1, 1] = 1

            ob_start_idx = text.find(ob_str)
            if ob_start_idx == -1:
                continue
            ob_idx = (ob_start_idx, ob_start_idx + len(ob_str))
            object_matrix = np.zeros((len(text), 50, 2))
            object_matrix[ob_idx[0], pred2idx[spo['predicate']], 0] = 1
            object_matrix[ob_idx[1] - 1, pred2idx[spo['predicate']], 1] = 1
            object[sub_idx] = object.get(sub_idx, 0) + object_matrix

        return {
            'subject_label': subject,
            'object_labels': list(object.items()),
            'spo_list': [(spo['subject'], spo['predicate'], spo['object']) for spo in spo_list]
        }

def label2idx(label, opt='sub'):
    """
    :param label:
    :return:
    """
    assert opt in ['sub', 'ob']

    if opt == 'sub':
        sub = []
        sub_start = torch.nonzero(label[..., 0])
        sub_end = torch.nonzero(label[..., 1])
        for i in sub_start:
            end = [j[0] for j in sub_end if j[0] >= i[0]]
            if len(end) <= 0:
                continue

            sub.append((i[0].item(), end[0].item() + 1))
        return sub

    if opt == 'ob':
        ob = []
        ob_start = torch.nonzero(label[..., 0])
        ob_end = torch.nonzero(label[..., 1])

        for i in ob_start:
            end = [j[0] for j in ob_end if j[-2] >= i[0] and j[1] == i[1]]
            if len(end) <= 0:
                continue

            predicate = idx2pred[i[1].item()]
            ob.append((predicate, (i[0].item(), end[0].item() + 1)))
        return ob

def collate_fn(list_of_data):
    data = {key: [item[key] for item in list_of_data if item is not None] for key in list_of_data[0].keys()}
    data['subject_label'] = nn.utils.rnn.pad_sequence(data['subject_label'], True)
    data['object_label'] = nn.utils.rnn.pad_sequence(data['object_label'], True)

    return data

if __name__ == '__main__':
    from model import BertBaseModel

    df = pd.read_json('./data/train_data.json', lines=True)
    data = Dataset(df)
    list_of_data = [data[0], data[1], data[2]]
    src = collate_fn(list_of_data)
    bert_path = 'C:\\Users\\yhane\\workspace\\RR\\model\\bert_model\\chinese_wwm_ext_pytorch'
    model = BertBaseModel(bert_path, classes_num=50)
    embed = model._embedding(src['text'])
    text_embed = embed['text_embed']


text:
#!/usr/bin/env python3
import numpy as np
from matplotlib import pyplot as plt

def relu(x):
    return np.maximum(0,x)

def sigmoid(x):
    return 1/(1+np.exp(-x))

plt.figure("Activation functions")

x=np.linspace(-5,5,100)
ax=plt.subplot(411)
ax.plot(x[0:50],np.zeros(50),"y-",label=r"$step(x)$")
ax.plot(x[50:100],np.ones(50),"y-")
ax.legend()
ax2=plt.subplot(412)
ax2.plot(x,sigmoid(x),"g-",label=r"$sigmoid(x)=\frac{1}{1+e^{-x}}$")
ax2.legend()
ax3=plt.subplot(413)
ax3.plot(x,relu(x),"r-",label=r"$relu(x)=\max(0,x)$")
ax3.legend()
ax4=plt.subplot(414)
ax4.plot(x,np.tanh(x),"b-",label=r"$\tanh(x)$")
ax4.legend()
plt.show()

max_stars_repo_name: TravorLZH/calculus

max_issues_repo_name: TravorLZH/calculus

max_forks_repo_name: TravorLZH/calculus


text:
import unittest
import yavalath_engine
from players import blue_player
import pprint
import collections
import pickle
import timeit
import numpy
import pathlib
import itertools

class TestDarrylPlayer(unittest.TestCase):
    def test_moves_to_vec(self):
        print(blue_player.moves_to_board_vector([]))
        print(blue_player.moves_to_board_vector(["e1"]))

    def test_vector_player(self):
        blue_player.vector_player([])

class TestConditions(unittest.TestCase):
    def test_win_conditions_board_4(self):
        game_so_far = ['e3', 'b1', 'e4']
        state = blue_player.GameState(game_so_far)
        def index_to_moves(index_iterable):
            return {state.options[index] for index in index_iterable}

        self.assertSetEqual(index_to_moves(state.white_winning_moves), set())
        self.assertSetEqual(index_to_moves(state.black_winning_moves), set())
        self.assertSetEqual(index_to_moves(state.white_losing_moves), {'e2', 'e5'})  # Winning conditions imply 2 losing conditions
        self.assertSetEqual(index_to_moves(state.black_losing_moves),  set())
        self.assertSetEqual(index_to_moves(state.white_single_checks), {'e1', 'e6'})  # This is a new condition, but not a new check.
        self.assertSetEqual(index_to_moves(state.black_single_checks),  set())
        self.assertSetEqual(index_to_moves(state.white_multi_checks), set())
        self.assertSetEqual(index_to_moves(state.black_multi_checks), set())

    def test_win_conditions_board_3(self):
        game_so_far = ['a1', 'b1', 'a2', 'b2', 'a4', 'b4']
        state = blue_player.GameState(game_so_far)
        def index_to_moves(index_iterable):
            return {state.options[index] for index in index_iterable}

        self.assertSetEqual(index_to_moves(state.white_winning_moves), {'a3'})
        self.assertSetEqual(index_to_moves(state.black_winning_moves), {'b3'})
        self.assertSetEqual(index_to_moves(state.white_losing_moves), {'a3'})  # Winning conditions imply 2 losing conditions
        self.assertSetEqual(index_to_moves(state.black_losing_moves), {'b3'})  # Winning conditions imply 2 losing conditions
        self.assertSetEqual(index_to_moves(state.white_single_checks), {'a5'})  # This is a new condition, but not a new check.
        self.assertSetEqual(index_to_moves(state.black_single_checks), {'b5'})
        self.assertSetEqual(index_to_moves(state.white_multi_checks), set())
        self.assertSetEqual(index_to_moves(state.black_multi_checks), set())

    def test_win_conditions_board_2(self):
        game_so_far = ['a1', 'b1', 'a2', 'b2', 'a4']
        state = blue_player.GameState(game_so_far)
        def index_to_moves(index_iterable):
            return {state.options[index] for index in index_iterable}

        self.assertSetEqual(index_to_moves(state.white_winning_moves), {'a3'})
        self.assertSetEqual(index_to_moves(state.black_winning_moves), set())
        self.assertSetEqual(index_to_moves(state.white_losing_moves), {'a3'})  # Winning conditions imply 2 losing conditions
        self.assertSetEqual(index_to_moves(state.black_losing_moves), {'b3'})
        self.assertSetEqual(index_to_moves(state.white_single_checks), {'a5'})  # This is a new condition, but not a new check.
        self.assertSetEqual(index_to_moves(state.black_single_checks), {'b4'})
        self.assertSetEqual(index_to_moves(state.white_multi_checks), set())
        self.assertSetEqual(index_to_moves(state.black_multi_checks), set())

    def test_win_conditions_board_1(self):
        game_so_far = ['a1']
        state = blue_player.GameState(game_so_far)

        def index_to_moves(index_iterable):
            return {state.options[index] for index in index_iterable}

        self.assertSetEqual(index_to_moves(state.white_winning_moves), set())
        self.assertSetEqual(index_to_moves(state.black_winning_moves), set())
        self.assertSetEqual(index_to_moves(state.white_losing_moves), set())
        self.assertSetEqual(index_to_moves(state.black_losing_moves), set())
        self.assertSetEqual(index_to_moves(state.white_single_checks), set())
        self.assertSetEqual(index_to_moves(state.black_single_checks), set())
        self.assertSetEqual(index_to_moves(state.white_multi_checks), set())
        self.assertSetEqual(index_to_moves(state.black_multi_checks), set())

    def test_win_conditions_empty_board(self):
        game_so_far = []
        state = blue_player.GameState(game_so_far)

        def index_to_moves(index_iterable):
            return {state.options[index] for index in index_iterable}

        self.assertSetEqual(index_to_moves(state.white_winning_moves), set())
        self.assertSetEqual(index_to_moves(state.black_winning_moves), set())
        self.assertSetEqual(index_to_moves(state.white_losing_moves), set())
        self.assertSetEqual(index_to_moves(state.black_losing_moves), set())
        self.assertSetEqual(index_to_moves(state.white_single_checks), set())
        self.assertSetEqual(index_to_moves(state.black_single_checks), set())
        self.assertSetEqual(index_to_moves(state.white_multi_checks), set())
        self.assertSetEqual(index_to_moves(state.black_multi_checks), set())

    def test_potential_moves(self):
        def test_game_so_far(game_so_far):
            board = yavalath_engine.HexBoard()
            state = blue_player.GameState(game_so_far)
            self.assertEqual(len(state.options), 61-len(game_so_far), "The number of options is wrong for game: {}.".format(game_so_far))
            options = [space_to_index[space] for space in state.options]
            self.assertEqual(state.white_potential_moves.shape, (62, 62-len(game_so_far))) # 62 rows for the spaces + "1", 61-len(game) columns for potential moves + the "no move"
            for board_space in range(61):  # Always 61, since the board size is fixed
                for potential_move_index in range(len(state.options)):  # Just the number of options
                    expected = 1 if board_space == options[potential_move_index] else 0
                    if board.spaces[board_space] in game_so_far:  # a1
                        expected = 1 if board_space in state.white_move_indices else -1
                    self.assertEqual(state.white_potential_moves[board_space, potential_move_index], expected,
                                     "Mismatch for board_space:{}, potential_move_index:{}".format(board_space, potential_move_index))
            for board_space in range(61):
                potential_move_index = len(state.options)
                expected = 1 if board_space in state.white_move_indices else -1 if board_space in state.black_move_indices else 0
                self.assertEqual(state.white_potential_moves[board_space, potential_move_index], expected,
                                 "Mismatch for 'no move' column, board_space:{}, potential_move_index:{}".format(board_space, potential_move_index))
            for potential_move_index in range(len(state.options)):  # Just the number of options
                board_space = 61
                self.assertEqual(state.white_potential_moves[board_space, potential_move_index], 1,
                                 "Mismatch for 'offset' row, board_space:{}, potential_move_index:{}".format(board_space, potential_move_index))

        def test_all_game_steps(game_so_far):
            for i in range(len(game_so_far)):
                test_game_so_far(game_so_far[0:i])

        # If I think there is a gam ewith the potential moves being incorrectly created, add that game here.
        test_all_game_steps(['a1', 'b2', 'a2', 'b1', 'c3'])
        game_so_far = ['g1', 'd5', 'e7', 'b5', 'e1', 'e3', 'i4', 'g4', 'b6', 'e4', 'f5', 'a5', 'c5', 'e8', 'c3', 'd3', 'f4', 'd2']
        test_all_game_steps(game_so_far)

def main1():
    win_C = blue_player.get_win_conditions()
    loss_C = blue_player.get_loss_conditions()
    pprint.pprint(win_C)
    pprint.pprint(win_C.shape)
    pprint.pprint(loss_C)
    pprint.pprint(loss_C.shape)
    print(blue_player.moves_to_board_vector(["e1", "e2", "e3", "e4"]).transpose())

class TestMoves(unittest.TestCase):

    def test_expected(self):
        player = lambda game: blue_player.vector_player2(game, depth=2)
        blocks = [
            # Pairs of game_so_far, expected_move
            (['a1', 'b1', 'a2', 'b2', 'a4'], 'a3'),
            (['a1', 'b1', 'a2', 'b2', 'a4', 'b4'], 'a3'),
            ]
        for game_so_far, expected_block in blocks:
            move, score = player(game_so_far)
            self.assertEqual(move, expected_block)

def main2():
    import logging
    blue_player.logger.setLevel(logging.DEBUG)
    game_so_far = ['g1', 'd5', 'e7', 'b5', 'e1', 'e3', 'i4', 'g4', 'b6', 'e4', 'f5', 'a5', 'c5', 'e8', 'c3', 'd3', 'f4', 'd2']
    #game_so_far =  ['e8', 'a1', 'a2', 'a3', 'f1', 'a4', 'c1']
    #game_so_far =  ['d3']
    print("Starting white spaces:", sorted(game_so_far[::2]))
    print("Starting black spaces:", sorted(game_so_far[1::2]))

    r = blue_player.vector_player2(game_so_far, depth=2, verbose=True)
    print("Result:", r)

def main3():
    v1 = blue_player.get_linear_condition_matrix((1, 1, 1, 1))
    v2 = blue_player.get_win_conditions()
    print("v1 == v2:", (v1 == v2).all())

    v1 = blue_player.get_linear_condition_matrix((1, 1, 1))
    v2 = blue_player.get_loss_conditions()
    print("v1 == v2:", (v1 == v2).all())

space_to_index = {space: i for i, space in enumerate(yavalath_engine.HexBoard().spaces)}

def get_board_hash(white_move_indices, black_move_indices):
    key = (tuple(sorted(white_move_indices)), tuple(sorted(black_move_indices)))
    return hash(key)

def get_board_hash2(white_move_indices, black_move_indices):
    white_bitset = int()
    for move in white_move_indices:
        white_bitset |= (1 << move)
    black_bitset = int()
    for move in black_move_indices:
        black_bitset |= (1 << move)
    key = (white_bitset, black_bitset)
    return hash(key)

def main4():
    game_so_far = "g1 d5 e7 b5 e1 e3 i4 g4 b6 e4 f5 a5 c5 e8 c3 d3 f4 d2 d4".split()
    print("Starting white spaces:", game_so_far[::2])
    print("Starting black spaces:", game_so_far[1::2])

    white_move_indices = {space_to_index[move] for move in game_so_far[::2]}
    black_move_indices = {space_to_index[move] for move in game_so_far[1::2]}

    t = timeit.timeit(lambda: get_board_hash(white_move_indices, black_move_indices))
    print("get_board_hash: 1M iterations took {}s".format(t))

    t = timeit.timeit(lambda: get_board_hash2(white_move_indices, black_move_indices))
    print("get_board_hash2: 1M iterations took {}s".format(t))

def to_str(p):
    if len(p) == 1:
        return p[0].value
    else:
        white, black = p
        return "{}, {}".format(str(white) if white is None else white.value, str(black) if black is None else black.value)

def main5():
    signature_table, properties_table = pickle.load(open("final.dat", 'rb'))

    for p in sorted([to_str(s) for s in properties_table]):
        print(p)
    index = numpy.nonzero(signature_table)
    print("Number of non-zero:", len(index[0]))
    print("Done")

def main6():
    properties = dict()
    t = timeit.timeit(lambda: blue_player.NextMoveClassifier.find_wins_and_checks_for_token_and_opp_arms_fast((-1, 0, 0), (0, -1, 0), -1))
    print(t)

def combine_results_simple(dirs = [r"data\laptop_files", r"data\desktop_files"]):
    """This one assumes all sub-results use the same properites dictionary, and that the sentinel value is -1"""
    final_complete_tasks_filename = "final_complete_tasks.dat"
    final_tables_filename = "final_tables.dat"

    one_arm_possibilities = list(itertools.product([-1,0,1], [-1,0,1], [-1,0,1]))
    desired_tasks = set()
    for arm1, arm2 in itertools.product(one_arm_possibilities, one_arm_possibilities):
        task = (arm1, arm2)
        desired_tasks.add(task)

    # Load the summary table and complete tasks list, if they exist.
    if pathlib.Path(final_complete_tasks_filename).exists():
        final_complete_tasks = pickle.load(open(final_complete_tasks_filename, 'rb'))
    else:
        final_complete_tasks = set()
    if pathlib.Path(final_tables_filename).exists():
        combined_signature_table, combined_properties_table = pickle.load(open(final_tables_filename, 'rb'))
    else:
        combined_signature_table = None
        combined_properties_table = None

    # Join the results
    for dir in dirs:
        print("Looking at dir:", dir)
        p = pathlib.Path(dir)
        for child in p.iterdir():
            if not child.is_file():
                continue
            print("Looking at child:", child)
            if child.name == "complete_tasks.dat":
                done_tasks = pickle.load(open(child.as_posix(), 'rb'))
                final_complete_tasks = final_complete_tasks.union(set(done_tasks))
                pickle.dump(file=open(final_complete_tasks_filename, 'wb'), obj=final_complete_tasks)
            elif child.name.find("signature_table_worker") == 0:
                print("Loading signature and properties tables from file:", child.name)
                signature_table, properties_table = pickle.load(open(child.as_posix(), 'rb'))
                print(properties_table[0])
                if combined_properties_table is None:
                    combined_properties_table = properties_table
                    assert combined_signature_table is None, "What happened?"
                    combined_signature_table = signature_table
                else:
                    # I messed up... I cannot disambiguate a zero meaning "not yet computed" from a zero meaning the
                    # first entry in the lookup table.  Fortunately, I think all zeros mean "GAME_OVER" since the first
                    # arm will always be (-1,-1,-1) for any task.  I can pass once more over zeros at the end, I guess
                    # Form a mapping from one property table to another
                    print("Combining properties_tables.")
                    assert properties_table == combined_properties_table, "Everyone should be using the same table."

                    print("Computing index where values >= 0...")
                    index = numpy.where(signature_table >= 0)
                    print("Updating combined index...")
                    combined_signature_table[index] = signature_table[index]  # Assign all non-zero values
                    # Should I feel confident that the mappings don't overlap?

            print("Saving final results to final.dat")
            final = (combined_signature_table, combined_properties_table)
            pickle.dump(file=open("final.dat", "wb"), obj=final)

    remaining = desired_tasks - final_complete_tasks
    print("All Done: {}, Remaining {}:\n{}".format(len(final_complete_tasks), len(remaining), remaining))

    print("Counting sentinel values...")
    index = numpy.where(combined_signature_table == -1)
    print("Remaining sentinel values:", len(index[0]))

def main7():
    signature_table, properties_table = pickle.load(open("signature_tables.dat", "rb"))
    signatures = numpy.array([
            303081869,
            306723378,
            44801543,
            173941706,
        ])
    r = numpy.array(properties_table)[signature_table[signatures]]
    pprint.pprint(r)

def main8():
    signatures = [
            303081869,
            306723378,
            44801543,
            173941706,
        ]
    for signature in signatures:
        SIGNATURE_OFFSET = sum([3**i for i in range(18)])  # Add this to all signatures to make them >= 0.
        arms = blue_player.NextMoveClassifier.signature_index_to_arms(int(signature))
        print(signature, arms, SIGNATURE_OFFSET+blue_player.NextMoveClassifier.compute_signature(arms))
        signature, properties = blue_player.NextMoveClassifier.compute_signature_and_properties(arms)
        print(signature+SIGNATURE_OFFSET, properties)

def main9():
    game_so_far = ['g1', 'd5', 'e7', 'b5', 'e1', 'e3', 'i4', 'g4', 'b6', 'e4', 'f5', 'a5', 'c5', 'e8', 'c3', 'd3', 'f4', 'd2']
    # Moves by property:
    # SpaceProperies.WHITE_LOSE: ['c4', 'd6', 'f1', 'f3', 'f6']
    # ERROR: SpaceProperies.BLACK_SINGLE_CHECK: ['b1', 'b3', 'c6', 'd7', 'e6', 'g3'] # b3 is not a check... it is already blocked.
    # SpaceProperies.BLACK_WIN: ['d4']
    # SpaceProperies.BLACK_LOSE: ['c1', 'c2', 'd1', 'e2', 'e5', 'f2', 'f3']
    # ERROR: SpaceProperies.GAME_OVER: ['f7', 'h1']
    # SpaceProperies.WHITE_SINGLE_CHECK: ['d4', 'e5', 'f2']

    #game_so_far =  ['e8', 'a1', 'a2', 'a3', 'f1', 'a4', 'c1']
    # Moves by property:
    # SpaceProperies.BLACK_LOSE: ['a5']
    # game_so_far = ['d3']

    move_stack = ['d2', 'b2', 'd4', 'f5', 'd5', 'd3', 'b5', 'c5', 'f1', 'e2', 'f2', 'a1', 'b1', 'c7', 'c6', 'e8', 'g6', 'f6', 'b6', 'd6', 'a3', 'g1']
    game_so_far = ['g2', 'c2', 'g7', 'i5', 'd1', 'e6', 'c4', 'a2', 'b3', 'e5', 'g4', 'g5', 'c1']
    yavalath_engine.Render(board=yavalath_engine.HexBoard(), moves=game_so_far).render_image("debug.png")
    classifier = blue_player.NextMoveClassifier(game_so_far, verbose=True)
    classifier.compute_moves_by_property()

def regen_status():
    done_tasks = pickle.load(open("data/complete_tasks.dat", 'rb'))
    print("{} Local Done:\n{}".format(len(done_tasks), done_tasks))
    desktop_done = pickle.load(open(r"D:\yavalath\data/complete_tasks.dat", 'rb'))
    print("{} Desktop Done:\n{}".format(len(desktop_done), desktop_done))
    sig_table, prop_table = pickle.load(open("data/signature_table_worker_0.dat", "rb"))
    for index, p in enumerate(prop_table):
        print(index, list(i.value if i is not None else "None" for i in p))
    print("Max(sig_table):{}".format(sig_table.max()))
    h = numpy.histogram(sig_table, bins=[-1,] + list(range(30)))
    print(h)

if __name__ == "__main__":
    #main9()
    # tasks = [
    #     ((1, 1, -1), (-1, 1, -1)),
    #     ((1, 1, -1), (0, -1, 0)),
    #     ((1, 1, -1), (1, -1, -1)),
    #     ((1, 1, 0), (-1, 0, 1)),
    #     ((1, 1, -1), (0, 0, 0)),
    #     ((1, 1, -1), (1, 0, -1))
    # ]
    #tasks=[((1, 1, -1), (0, 0, 0)), ((1, 1, -1), (1, 0, -1))]
    #blue_player.NextMoveClassifier.compute_signature_table()
    combine_results_simple(dirs=[r"data",])
    #blue_player.NextMoveClassifier.compute_signature_table()
    #pprint.pprint(pickle.load(open("data/complete_tasks.dat", "rb")))
    #regen_status()