# Library of useful functions

```julia
# Primes
using Primes

# ToDo: Generalize this so that it can take pregenerated sieves as input.
function primes(limit::Int)::Vector{Int}
	sieve = Int[2]

	for odd ∈ 3:2:limit    # Consider only odd numbers
		isprime = true
		for prime ∈ sieve
			prime * prime > odd && break    # No need to check beyond this
			odd % prime == 0 && (isprime = false)
		end
		isprime && push!(sieve, odd)
	end

	return sieve
end

isprime(n::Int) = (primes(n)[end] == n)

function nthprime(n::Int)::Int
	upperbound = Int(floor(n * (log(n) + log(log(n)))))
	return primes(upperbound)[n]
end

# Fibonacci
# ToDo: Generalize this to return Int or Int128 if in range.
# Thought: The Binet's formula computation can be optimized by storing ϕ^n and using it to compute ϕ^(n+1).
function fibonacci_Binet(index::Int)::BigInt
	# Uses Binet's formula
	sqrt5 = √5
	ϕ = big(1 + sqrt5) / 2
	ψ = big(1 - sqrt5) / 2
	return round(BigInt, (ϕ^index - ψ^index) / sqrt5)
end

# Factorization
function primefactorize(n::Int)
	sieve = primes(n)    # Have to figure out a way to not generate so many primes

	factors = Int[]
	powers = Int[]

	for prime ∈ sieve
		prime > n && break
		pow = 0
		while n % prime == 0
			n ÷= prime
			pow += 1
		end
		if pow > 0
			push!(factors, prime)
			push!(powers, pow)
		end
	end

	return collect(zip(factors, powers))
end

function numdivisors(n::Int)::Int
	n == 1 && return 1
	factorization = primefactorize(n)
	count = 1
	return prod(map(x -> x[2] + 1, factorization))
end

function sumdivisors(n::Int)::Int
	n == 1 && (return 1)
	# https://www.xarg.org/2016/06/calculate-the-sum-of-divisors/
	factored = primefactorize(n)
	return prod((factor[1]^(factor[2] + 1) - 1) ÷ (factor[1] - 1) for factor in factored)
end

# sumdivisors(n::Int)::Int  =  sum(i for i ∈ 1:n if n % i == 0)    # Slow

# The following are implemented in Julia.Base
# # The ith position gives the coefficient of the (i-1)th power of the base.
# function digits(n::Int, base::Int)::Vector{Int}
# 	d = Int[]
# 	while n > 0
# 		push!(d, n % base)
# 		n ÷= base
# 	end
# 	return d
# end

# # Combinatorics
# function binomial(n::Int, k::Int)::Int
# 	product = 1
# 	for i ∈ 0:(k - 1)
# 		product *= (n - i) / (k - i)
# 	end
# 	return Int(product)
# end

# count_digits(n::Int, base::Int) = length(digits(n, base))
```

# Problem 001: Multiples of 3 and 5

If we list all the natural numbers below $ 10 $ that are multiples of $ 3 $ or $ 5 $, we get $ 3 $, $ 5 $, $ 6 $ and $ 9 $. The sum of these multiples is $ 23 $.

Find the sum of all the multiples of $ 3 $ or $ 5 $ below $ 1000 $.

```julia
# function multiples(limit::Int)::Int
# 	sum::Int = 0
# 	for i ∈ 3:limit
# 		if i%3 == 0 || i%5 == 0
# 			sum += i
# 		end
# 	end
# 	return sum
# end

multiples(limit::Int)::Int = sum(i for i ∈ 3:limit if i%3 == 0 || i%5 == 0)
@time println(multiples(999))
```

    233168
      0.036204 seconds (28.68 k allocations: 1.638 MiB)

# Problem 002: Even Fibonacci numbers

Each new term in the Fibonacci sequence is generated by adding the previous two terms. By starting with $ 1 $ and $ 2 $, the first 10 terms will be
$ 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, … $.

By considering the terms in the Fibonacci sequence whose values do not exceed four million, find the sum of the even-valued terms.

### Comments

It would seem that a closed form formula would be much faster to evaluate than iterate through all the possibilities. Yet, the iterative solution is much faster!

*TODO*: A time-complexity analysis.

```julia
# Slower
function fibonaccieven_Binet(limit::Int)::Int
	Σ = 0
	i = 0
	while true
		i += 3
		f_i = fibonacci_Binet(i)
		f_i ≥ limit  &&  break
		Σ += f_i
	end
	return Σ
end

function fibonaccieven_iter(limit::Int)::Int
	Σ::Int = 10
	a::Int = 2
	b::Int = 8
	while true
		new = 4b + a
		new ≥ limit  &&  break
		(a, b) = (b, new)
		Σ += b
	end
	return Σ
end

@time println(fibonaccieven_Binet(4_000_000))
@time println(fibonaccieven_iter(4_000_000))
```

# Problem 003: Largest prime factor

The prime factors of $ 13195 $ are $ 5, 7, 13, 29 $.

What is the largest prime factor of the number $ 600851475143 $?

```julia
# The idea is to keep dividing by the small divisors so that we are left with the largest prime number.
function maxprimefactor(n::Int)::Int
	# Special case: 2^k is a factor
	while n ≠ 2 && n % 2 == 0
		n ÷= 2
	end

	# Odd prime factors p^k
	i::Int = 3
	while i * i ≤ n
		if isprime(i) && n % i == 0
			n ÷= i
		else
			i += 2
		end
	end

	return n
end

@time println(maxprimefactor(600851475143))
```

# Problem 004: Largest palindrome product

A palindromic number reads the same both ways. The largest palindrome made from the product of two 2-digit numbers is $ 9009 = 91 × 99 $.

Find the largest palindrome made from the product of two 3-digit numbers.

Note that $ 999 ⋅ 998 = 998 ⋅ 999 $, so we only need to check the upper diagonal matrix.

```julia
function reverse(n::Int)::Int
	decimal = digits(n)
	rev::Int = 0
	for d ∈ decimal
		rev = 10 * rev + d
	end
	return rev
end

ispalindrome(n::Int)::Bool  =  (n == reverse(n))

function maxpalindromeproduct(max_decimal_digits::Int)::Int
	max::Int = 10^max_decimal_digits - 1
	palindromes = Int[]

	for i ∈ Iterators.reverse(1:max), j ∈ Iterators.reverse(i:max)
		n::Int = i * j

		# The i*max cannot go below the any of the palindromes found.
		length(palindromes) > 0 && i * max ≤ palindromes[1] && break

		ispalindrome(n) && push!(palindromes, n)
	end
	return maximum(palindromes)
end

@time println(maxpalindromeproduct(3))
```

    906609
      0.000794 seconds (6.14 k allocations: 764.109 KiB)

# Problem 005: Smallest multiple

$ 2520 $ is the smallest number that can be divided by each of the numbers from $ 1 $ to $ 10 $ without any remainder.

What is the smallest positive number that is evenly divisible by all of the numbers from $ 1 $ to $ 20 $?

```julia
@time println(2^4 * 3^2 * 5 * 7 * 11 * 13 * 17 * 19)
```

    232792560
      0.000089 seconds (34 allocations: 768 bytes)

# Problem 006: Sum square difference

The sum of the squares of the first ten natural numbers is
$ 1^2 + 2^2 + ⋯ + 10^2 = 385 $.

The square of the sum of the first ten natural numbers is
$ (1 + 2 + ⋯ + 10)^2 = 55^2 = 3025 $.

Hence the difference between the sum of the squares of the first ten natural numbers and the square of the sum is $ 3025 - 385 = 2640 $.

Find the difference between the sum of the squares of the first one hundred natural numbers and the square of the sum.

$$ Δ = \left( \frac{n (n+1)}{2} \right)^2 - \frac{n (n + 1) (2n + 1)}{6} = \frac{1}{12} (n (n + 1) (3 n^2 - n - 2)) $$

```julia
@time println(100 * (100 + 1) * (3 * 100^2 - 100 - 2) ÷ 12)
```

    25164150
      0.000103 seconds (35 allocations: 1.078 KiB)

# Problem 007: 10001st prime

By listing the first six prime numbers: $ 2, 3, 5, 7, 11, 13 $, we can see that the 6th prime is $ 13 $.

What is the 10001st prime number?

The trick is to use upper bound for the nth prime as given [here](https://en.m.wikipedia.org/wiki/Prime_number_theorem#Approximations_for_the_nth_prime_number).

```julia
@time println(nthprime(10001))
```

# Problem 008: Largest product in a series

The four adjacent digits in the 1000-digit number that have the greatest product are $ 9 × 9 × 8 × 9 = 5832 $.

```
73167176531330624919225119674426574742355349194934
96983520312774506326239578318016984801869478851843
85861560789112949495459501737958331952853208805511
12540698747158523863050715693290963295227443043557
66896648950445244523161731856403098711121722383113
62229893423380308135336276614282806444486645238749
30358907296290491560440772390713810515859307960866
70172427121883998797908792274921901699720888093776
65727333001053367881220235421809751254540594752243
52584907711670556013604839586446706324415722155397
53697817977846174064955149290862569321978468622482
83972241375657056057490261407972968652414535100474
82166370484403199890008895243450658541227588666881
16427171479924442928230863465674813919123162824586
17866458359124566529476545682848912883142607690042
24219022671055626321111109370544217506941658960408
07198403850962455444362981230987879927244284909188
84580156166097919133875499200524063689912560717606
05886116467109405077541002256983155200055935729725
71636269561882670428252483600823257530420752963450
```

Find the thirteen adjacent digits in the 1000-digit number that have the greatest product. What is the value of this product?

```julia
n = 7316717653133062491922511967442657474235534919493496983520312774506326239578318016984801869478851843858615607891129494954595017379583319528532088055111254069874715852386305071569329096329522744304355766896648950445244523161731856403098711121722383113622298934233803081353362766142828064444866452387493035890729629049156044077239071381051585930796086670172427121883998797908792274921901699720888093776657273330010533678812202354218097512545405947522435258490771167055601360483958644670632441572215539753697817977846174064955149290862569321978468622482839722413756570560574902614079729686524145351004748216637048440319989000889524345065854122758866688116427171479924442928230863465674813919123162824586178664583591245665294765456828489128831426076900422421902267105562632111110937054421750694165896040807198403850962455444362981230987879927244284909188845801561660979191338754992005240636899125607176060588611646710940507754100225698315520005593572972571636269561882670428252483600823257530420752963450

function largest_product(n::BigInt, adjacentdigits::Int)::Int
	decimal = digits(n)
	productmax::Int = 1

	for i ∈ 1:(length(decimal) - adjacentdigits + 1)
		product::Int = 1
		for k ∈ 0:(adjacentdigits - 1)
			product *= decimal[i + k]
		end
		product > productmax && (productmax = product)
	end

	return productmax
end

@time println(largest_product(n, 13))
```

    23514624000
      0.008736 seconds (9.02 k allocations: 364.484 KiB)

# Problem 009: Special Pythagorean triplet

A Pythagorean triplet is a set of three natural numbers, $ a < b < c $, for which, $ a^2 + b^2 = c^2 $.
For example, $ 3^2 + 4^2 = 9 + 16 = 25 = 5^2 $.

There exists exactly one Pythagorean triplet for which $ a + b + c = 1000 $.
Find the product abc.

### Solution

Let $ p $ be the given perimeter of the triangle. We parameterize the sides as $ a = m^2 - n^2, b = 2mn, c = m^2 + n^2 $, where $ m > n > 0 $. Using the fact that $ a + b + c = p $, we get $ 2 m (m + n) = p $.

Now we need good bounds for $ m, n $ in order to reduce the search space. Note that for $ n $ to be maximum, we must have $ m = n + 1 $, so we get the estimate $ n ≤ \frac{\sqrt{p}}{2} $. On the other hand, for $ m $ to maximum, we require $ n = 1 $, so we get the estimate $ m ≤ \sqrt{\frac{p}{2}} $.

Finally, the required product after simplification is $ p n (m - n) (m^2 + n^2) $.

References:
1. The Overview PDF.
2. Pier's forum post on Thu, 19 Aug 2004, 09:04.

```julia
# Using a parameterization of Pythagorean triplets
function special_Pythagorean_triplet(perimeter::Int)::Int
	for n ∈ 1:(isqrt(perimeter) ÷ 2), m ∈ n:isqrt(perimeter ÷ 2)
		2m * (m + n) == perimeter  &&  return perimeter * n * (m - n) * (m^2 + n^2)
	end
	return 0
end

# # Naive
# function special_Pythagorean_triplet(perimeter::Int)::Int
# 	# Assume a < b < c
# 	for a ∈ 1:((perimeter - 3) ÷ 3)
# 		for b ∈ a:((perimeter - a) ÷ 2)
# 			c = perimeter - a - b
# 			a*a + b*b == c*c  &&  return a * b * c
# 		end
# 	end
# 	return -1
# end

@time println(special_Pythagorean_triplet(1000))

```

    31875000
      0.000104 seconds (34 allocations: 768 bytes)

# Problem 010: Summation of primes

The sum of the primes below $ 10 $ is $ 2 + 3 + 5 + 7 = 17 $.

Find the sum of all the primes below two million.

```julia
@time println(sum(primes(2000000)))
```

# Problem 011: Largest product in a grid

In the 20×20 grid below, four numbers along a diagonal line have been marked in bold.

08 02 22 97 38 15 00 40 00 75 04 05 07 78 52 12 50 77 91 08
49 49 99 40 17 81 18 57 60 87 17 40 98 43 69 48 04 56 62 00
81 49 31 73 55 79 14 29 93 71 40 67 53 88 30 03 49 13 36 65
52 70 95 23 04 60 11 42 69 24 68 56 01 32 56 71 37 02 36 91
22 31 16 71 51 67 63 89 41 92 36 54 22 40 40 28 66 33 13 80
24 47 32 60 99 03 45 02 44 75 33 53 78 36 84 20 35 17 12 50
32 98 81 28 64 23 67 10 26 38 40 67 59 54 70 66 18 38 64 70
67 26 20 68 02 62 12 20 95 63 94 39 63 08 40 91 66 49 94 21
24 55 58 05 66 73 99 26 97 17 78 78 96 83 14 88 34 89 63 72
21 36 23 09 75 00 76 44 20 45 35 14 00 61 33 97 34 31 33 95
78 17 53 28 22 75 31 67 15 94 03 80 04 62 16 14 09 53 56 92
16 39 05 42 96 35 31 47 55 58 88 24 00 17 54 24 36 29 85 57
86 56 00 48 35 71 89 07 05 44 44 37 44 60 21 58 51 54 17 58
19 80 81 68 05 94 47 69 28 73 92 13 86 52 17 77 04 89 55 40
04 52 08 83 97 35 99 16 07 97 57 32 16 26 26 79 33 27 98 66
88 36 68 87 57 62 20 72 03 46 33 67 46 55 12 32 63 93 53 69
04 42 16 73 38 25 39 11 24 94 72 18 08 46 29 32 40 62 76 36
20 69 36 41 72 30 23 88 34 62 99 69 82 67 59 85 74 04 36 16
20 73 35 29 78 31 90 01 74 31 49 71 48 86 81 16 23 57 05 54
01 70 54 71 83 51 54 69 16 92 33 48 61 43 52 01 89 19 67 48

The product of these numbers is $ 26 × 63 × 78 × 14 = 1788696 $.

What is the greatest product of four adjacent numbers in the same direction (up, down, left, right, or diagonally) in the 20×20 grid?

The code below works, but does not show in the Jupyter Lab viewer. I am sure it has something to do with the leading zeros in the numbers. I do not know why they would matter, though. One can always go to the JSON view and read the code in `cell[22]`.

```julia
numbers = Int[
	08 02 22 97 38 15 00 40 00 75 04 05 07 78 52 12 50 77 91 08
	49 49 99 40 17 81 18 57 60 87 17 40 98 43 69 48 04 56 62 00
	81 49 31 73 55 79 14 29 93 71 40 67 53 88 30 03 49 13 36 65
	52 70 95 23 04 60 11 42 69 24 68 56 01 32 56 71 37 02 36 91
	22 31 16 71 51 67 63 89 41 92 36 54 22 40 40 28 66 33 13 80
	24 47 32 60 99 03 45 02 44 75 33 53 78 36 84 20 35 17 12 50
	32 98 81 28 64 23 67 10 26 38 40 67 59 54 70 66 18 38 64 70
	67 26 20 68 02 62 12 20 95 63 94 39 63 08 40 91 66 49 94 21
	24 55 58 05 66 73 99 26 97 17 78 78 96 83 14 88 34 89 63 72
	21 36 23 09 75 00 76 44 20 45 35 14 00 61 33 97 34 31 33 95
	78 17 53 28 22 75 31 67 15 94 03 80 04 62 16 14 09 53 56 92
	16 39 05 42 96 35 31 47 55 58 88 24 00 17 54 24 36 29 85 57
	86 56 00 48 35 71 89 07 05 44 44 37 44 60 21 58 51 54 17 58
	19 80 81 68 05 94 47 69 28 73 92 13 86 52 17 77 04 89 55 40
	04 52 08 83 97 35 99 16 07 97 57 32 16 26 26 79 33 27 98 66
	88 36 68 87 57 62 20 72 03 46 33 67 46 55 12 32 63 93 53 69
	04 42 16 73 38 25 39 11 24 94 72 18 08 46 29 32 40 62 76 36
	20 69 36 41 72 30 23 88 34 62 99 69 82 67 59 85 74 04 36 16
	20 73 35 29 78 31 90 01 74 31 49 71 48 86 81 16 23 57 05 54
	01 70 54 71 83 51 54 69 16 92 33 48 61 43 52 01 89 19 67 48
]

function largestproduct(numbers::Array{Int, 2}, moves::Int)::Int
	# ToDo: Update everything to allow for custom indexing
	# https://docs.julialang.org/en/v1/devdocs/offset-arrays/
	# https://julialang.org/blog/2016/02/iteration/
	# https://docs.julialang.org/en/v1/manual/arrays/

	dim = size(numbers)
	productmax::Int = 1

	# Along dimension 1
	for i ∈ 1:dim[1], j ∈ 1:(dim[2] - moves)
		product::Int = 1
		for k ∈ 0:(moves-1)
			product *= numbers[i, j + k]
		end
		if product > productmax
			productmax = product
		end
	end

	# Along dimension 2
	for j ∈ 1:dim[2], i ∈ 1:(dim[1] - moves)
		product::Int = 1
		for k ∈ 0:(moves-1)
			product *= numbers[i + k, j]
		end
		if product > productmax
			productmax = product
		end
	end

	# Diagonal
	for i ∈ 1:dim[1], j ∈ 1:dim[2]
		product::Int = 1
		for k ∈ 0:(moves-1)
			if (i + k > dim[1]) || (j + k > dim[2])
				break
			end
			product *= numbers[i + k, j + k]
		end
		if product > productmax
			productmax = product
		end
	end

	# Antidiagonal
	for i ∈ 1:dim[1], j ∈ 1:dim[2]
		product::Int = 1
		for k ∈ 0:(moves-1)
			if (i + k > dim[1]) || (j - k < 1)
				break
			end
			product *= numbers[i + k, j - k]
		end
		if product > productmax
			productmax = product
		end
	end

	return productmax
end

@time println(largestproduct(numbers, 4))
```

    70600674
      0.000112 seconds (36 allocations: 800 bytes)

# Problem 012: Highly divisible triangular number

The sequence of triangle numbers is generated by adding the natural numbers. So the 7th triangle number would be $ 1 + 2 + 3 + 4 + 5 + 6 + 7 = 28 $.

The first ten terms would be
$ 1, 3, 6, 10, 15, 21, 28, 36, 45, 55, … $.

Let us list the factors of the first seven triangle numbers:

```
 1: 1
 3: 1,3
 6: 1,2,3,6
10: 1,2,5,10
15: 1,3,5,15
21: 1,3,7,21
28: 1,2,4,7,14,28
```
We can see that $ 28 $ is the first triangle number to have over five divisors.

What is the value of the first triangle number to have over five hundred divisors?

```julia
# ToDo

# function first_triangular_number(factors::Int)::Int
# 	s::Int = 0
# 	for i ∈ Iterators.countfrom(1)
# 		s += i
# 		count_factors(s) > factors && return s
# 	end
# end

# first_triangular_number(500)
# @time println(numdivisors(prod(primes(23))))
# primes(23)
```

# Problem 013: Large sum

Work out the first ten digits of the sum of the following one-hundred 50-digit numbers.

```julia
numbers = BigInt[
	37107287533902102798797998220837590246510135740250,
	46376937677490009712648124896970078050417018260538,
	74324986199524741059474233309513058123726617309629,
	91942213363574161572522430563301811072406154908250,
	23067588207539346171171980310421047513778063246676,
	89261670696623633820136378418383684178734361726757,
	28112879812849979408065481931592621691275889832738,
	44274228917432520321923589422876796487670272189318,
	47451445736001306439091167216856844588711603153276,
	70386486105843025439939619828917593665686757934951,
	62176457141856560629502157223196586755079324193331,
	64906352462741904929101432445813822663347944758178,
	92575867718337217661963751590579239728245598838407,
	58203565325359399008402633568948830189458628227828,
	80181199384826282014278194139940567587151170094390,
	35398664372827112653829987240784473053190104293586,
	86515506006295864861532075273371959191420517255829,
	71693888707715466499115593487603532921714970056938,
	54370070576826684624621495650076471787294438377604,
	53282654108756828443191190634694037855217779295145,
	36123272525000296071075082563815656710885258350721,
	45876576172410976447339110607218265236877223636045,
	17423706905851860660448207621209813287860733969412,
	81142660418086830619328460811191061556940512689692,
	51934325451728388641918047049293215058642563049483,
	62467221648435076201727918039944693004732956340691,
	15732444386908125794514089057706229429197107928209,
	55037687525678773091862540744969844508330393682126,
	18336384825330154686196124348767681297534375946515,
	80386287592878490201521685554828717201219257766954,
	78182833757993103614740356856449095527097864797581,
	16726320100436897842553539920931837441497806860984,
	48403098129077791799088218795327364475675590848030,
	87086987551392711854517078544161852424320693150332,
	59959406895756536782107074926966537676326235447210,
	69793950679652694742597709739166693763042633987085,
	41052684708299085211399427365734116182760315001271,
	65378607361501080857009149939512557028198746004375,
	35829035317434717326932123578154982629742552737307,
	94953759765105305946966067683156574377167401875275,
	88902802571733229619176668713819931811048770190271,
	25267680276078003013678680992525463401061632866526,
	36270218540497705585629946580636237993140746255962,
	24074486908231174977792365466257246923322810917141,
	91430288197103288597806669760892938638285025333403,
	34413065578016127815921815005561868836468420090470,
	23053081172816430487623791969842487255036638784583,
	11487696932154902810424020138335124462181441773470,
	63783299490636259666498587618221225225512486764533,
	67720186971698544312419572409913959008952310058822,
	95548255300263520781532296796249481641953868218774,
	76085327132285723110424803456124867697064507995236,
	37774242535411291684276865538926205024910326572967,
	23701913275725675285653248258265463092207058596522,
	29798860272258331913126375147341994889534765745501,
	18495701454879288984856827726077713721403798879715,
	38298203783031473527721580348144513491373226651381,
	34829543829199918180278916522431027392251122869539,
	40957953066405232632538044100059654939159879593635,
	29746152185502371307642255121183693803580388584903,
	41698116222072977186158236678424689157993532961922,
	62467957194401269043877107275048102390895523597457,
	23189706772547915061505504953922979530901129967519,
	86188088225875314529584099251203829009407770775672,
	11306739708304724483816533873502340845647058077308,
	82959174767140363198008187129011875491310547126581,
	97623331044818386269515456334926366572897563400500,
	42846280183517070527831839425882145521227251250327,
	55121603546981200581762165212827652751691296897789,
	32238195734329339946437501907836945765883352399886,
	75506164965184775180738168837861091527357929701337,
	62177842752192623401942399639168044983993173312731,
	32924185707147349566916674687634660915035914677504,
	99518671430235219628894890102423325116913619626622,
	73267460800591547471830798392868535206946944540724,
	76841822524674417161514036427982273348055556214818,
	97142617910342598647204516893989422179826088076852,
	87783646182799346313767754307809363333018982642090,
	10848802521674670883215120185883543223812876952786,
	71329612474782464538636993009049310363619763878039,
	62184073572399794223406235393808339651327408011116,
	66627891981488087797941876876144230030984490851411,
	60661826293682836764744779239180335110989069790714,
	85786944089552990653640447425576083659976645795096,
	66024396409905389607120198219976047599490197230297,
	64913982680032973156037120041377903785566085089252,
	16730939319872750275468906903707539413042652315011,
	94809377245048795150954100921645863754710598436791,
	78639167021187492431995700641917969777599028300699,
	15368713711936614952811305876380278410754449733078,
	40789923115535562561142322423255033685442488917353,
	44889911501440648020369068063960672322193204149535,
	41503128880339536053299340368006977710650566631954,
	81234880673210146739058568557934581403627822703280,
	82616570773948327592232845941706525094512325230608,
	22918802058777319719839450180888072429661980811197,
	77158542502016545090413245809786882778948721859617,
	72107838435069186155435662884062257473692284509516,
	20849603980134001723930671666823555245252804609722,
	53503534226472524250874054075591789781264330331690,
]

function large_sum(numbers::Vector{BigInt})::Int
	s::BigInt = sum(numbers)
	numdigits = length(digits(s))
	strip = numdigits - 10
	return s ÷ big(10)^strip
end

@time println(large_sum(numbers))
```

    5537376230
      0.000192 seconds (494 allocations: 9.945 KiB)

# Problem 014: Longest Collatz sequence

The following iterative sequence is defined for the set of positive integers:

$ n → n/2 $ ($ n $ is even)

$ n → 3n + 1 $ ($ n $ is odd)

Using the rule above and starting with $ 13 $, we generate the following sequence:

$ 13 → 40 → 20 → 10 → 5 → 16 → 8 → 4 → 2 → 1 $.

It can be seen that this sequence (starting at $ 13 $ and finishing at $ 1 $) contains $ 10 $ terms. Although it has not been proved yet (Collatz Problem), it is thought that all starting numbers finish at $ 1 $.

Which starting number, under one million, produces the longest chain?

**NOTE**: Once the chain starts the terms are allowed to go above one million.

```julia
Collatz_transform(n::Int)::Int  =  iseven(n) ? n ÷ 2 : 3n + 1

Collatz_length(start::Int)::Int  =  start == 1 ? 1 : 1 + Collatz_length(Collatz_transform(start))

@time println(argmax(map(Collatz_length, 1:1_000_000)))

# This code may be optimized by memoization as a whole. I am sure a lot of paths are repeated.
```

    837799
      0.543555 seconds (38 allocations: 7.630 MiB)

# Problem 015: Lattice paths

Starting in the top left corner of a 2×2 grid, and only being able to move to the right and down, there are exactly 6 routes to the bottom right corner.

How many such routes are there through a 20×20 grid?

```julia
function lattice_paths(size::Int)::Int
	return binomial(2*size, size)
end

@time println(lattice_paths(20))
```

    137846528820
      0.000139 seconds (34 allocations: 768 bytes)

# Problem 016: Power digit sum

$ 2^{15} = 32768 $ and the sum of its digits is $ 3 + 2 + 7 + 6 + 8 = 26 $.

What is the sum of the digits of the number $ 2^{1000} $?

```julia
@time println(sum(digits(big(2)^1000)))
```

    1366
      0.000407 seconds (2.74 k allocations: 67.938 KiB)

# Problem 017: Number letter counts

If the numbers $ 1 $ to $ 5 $ are written out in words: one, two, three, four, five, then there are $ 3 + 3 + 5 + 4 + 4 = 19 $ letters used in total.

If all the numbers from $ 1 $ to $ 1000 $ (one thousand) inclusive were written out in words, how many letters would be used?

*NOTE*: Do not count spaces or hyphens. For example, 342 (three hundred and forty-two) contains 23 letters and 115 (one hundred and fifteen) contains 20 letters. The use of "and" when writing out numbers is in compliance with British usage.

```julia
"""This uses short scales.
https://en.wikipedia.org/wiki/Long_and_short_scales"""
function decimaltotext(n::Int)::String
	# We do not need momoization beyond 999 because things become regular.
	store = Vector{String}(undef, 999)
	store[1:19] = ["one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen", "sixteen", "seventeen", "eighteen", "nineteen"]

	function decimaltotext_aux(n::Int, memo::Vector{String})::String
		tens = ["ten", "twenty", "thirty", "forty", "fifty", "sixty", "seventy", "eighty", "ninety"]

		if isassigned(memo, n)
			return memo[n]
		end

		# Put spaces between words to make this more legible.
		if 20 ≤ n ≤ 99
			if n % 10 == 0
				return tens[n ÷ 10]
			else
				return string(tens[n ÷ 10], decimaltotext_aux(n % 10, memo))
			end
		elseif 100 ≤ n ≤ 999
			if n % 100 == 0
				return string(decimaltotext_aux(n ÷ 100, memo), "hundred")
			else
				return string(decimaltotext_aux(n ÷ 100, memo), "hundredand", decimaltotext_aux(n % 100, memo))
			end
		elseif 1_000 ≤ n ≤ 999_999
			if n % 1_000 == 0
				return string(decimaltotext_aux(n ÷ 1_000, memo), "thousand")
			else
				return string(decimaltotext_aux(n ÷ 1_000, memo), "thousandand", decimaltotext_aux(n % 1_000, memo))
			end
		elseif 1_000_000 ≤ n ≤ 999_999_999
			if n % 1_000_000 == 0
				return string(decimaltotext_aux(n ÷ 1_000_000, memo), "million")
			else
				return string(decimaltotext_aux(n ÷ 1_000_000, memo), "millionand", decimaltotext_aux(n % 1_000_000, memo))
			end
		elseif 1_000_000_000 ≤ n ≤ 999_999_999_999
			if n % 1_000_000_000 == 0
				return string(decimaltotext_aux(n ÷ 1_000_000_000, memo), "billion")
			else
				return string(decimaltotext_aux(n ÷ 1_000_000_000, memo), "billionand", decimaltotext_aux(n % 1_000_000_000, memo))
			end
		elseif 1_000_000_000_000 ≤ n
			if n % 1_000_000_000_000 == 0
				return string(decimaltotext_aux(n ÷ 1_000_000_000_000, memo), "trillion")
			else
				return string(decimaltotext_aux(n ÷ 1_000_000_000_000, memo), "trillionand", decimaltotext_aux(n % 1_000_000_000_000, memo))
			end
		end
	end

	return decimaltotext_aux(n, store)
end

@time println(sum(map(length ∘ decimaltotext, 1:1000)))
```

    21124
      0.070191 seconds (139.13 k allocations: 15.383 MiB)

# Problem 018: Maximum path sum I

By starting at the top of the triangle below and moving to adjacent numbers on the row below, the maximum total from top to bottom is $ 23 $.
```
3
7 4
2 4 6
8 5 9 3
```
That is, $ 3 + 7 + 4 + 9 = 23 $.

Find the maximum total from top to bottom of the triangle below:
```
75
95 64
17 47 82
18 35 87 10
20 04 82 47 65
19 01 23 75 03 34
88 02 77 73 07 63 67
99 65 04 28 06 16 70 92
41 41 26 56 83 40 80 70 33
41 48 72 33 47 32 37 16 94 29
53 71 44 65 25 43 91 52 97 51 14
70 11 33 28 77 73 17 78 39 68 17 57
91 71 52 38 17 14 91 43 58 50 27 29 48
63 66 04 68 89 53 67 30 73 16 69 87 40 31
04 62 98 27 23 09 70 98 73 93 38 53 60 04 23
```
For a scaled up version, see **Problem 67**.

```julia
triangle =
"75
95 64
17 47 82
18 35 87 10
20 04 82 47 65
19 01 23 75 03 34
88 02 77 73 07 63 67
99 65 04 28 06 16 70 92
41 41 26 56 83 40 80 70 33
41 48 72 33 47 32 37 16 94 29
53 71 44 65 25 43 91 52 97 51 14
70 11 33 28 77 73 17 78 39 68 17 57
91 71 52 38 17 14 91 43 58 50 27 29 48
63 66 04 68 89 53 67 30 73 16 69 87 40 31
04 62 98 27 23 09 70 98 73 93 38 53 60 04 23"

function numberarray(input::String)::Array{Int, 2}
	lines = map(split, split(input, "\n"))
	lines = filter(x -> length(x) > 0, lines)    #	Remove empty lines
	# Put all this in an T-array
	n = length(lines)
	Δ = zeros(Int, (n, n))
	for i ∈ 1:n, j ∈ 1:i
		Δ[i, j] = parse(Int, lines[i][j])
	end
	return Δ
end

function max_path_sum(triangle::Array{Int, 2})::Int
	store = zeros(Int, size(triangle))
	store[end, :] = triangle[end, :]    # Base case

	function max_path_sum_aux(triangle::Array{Int, 2}, memo::Array{Int, 2}, row::Int, col::Int)::Int
		memo[row, col] > 0 && (return memo[row, col])
		memo[row, col] = triangle[row, col] + max(max_path_sum_aux(triangle, memo, row + 1, col), max_path_sum_aux(triangle, memo, row + 1, col + 1))
		return memo[row, col]
	end
	return max_path_sum_aux(triangle, store, 1, 1)
end

@time println(max_path_sum(numberarray(triangle)))
```

    1074
      0.037189 seconds (6.55 k allocations: 335.569 KiB)

# Problem 19: Counting Sundays

You are given the following information, but you may prefer to do some research for yourself.

*  1 Jan 1900 was a Monday.
*  Thirty days has September,
   April, June and November.
   All the rest have thirty-one,
   Saving February alone,
   Which has twenty-eight, rain or shine.
   And on leap years, twenty-nine.
*  A leap year occurs on any year evenly divisible by 4, but not on a century unless it is divisible by 400.

How many Sundays fell on the first of the month during the twentieth century (1 Jan 1901 to 31 Dec 2000)?

```julia
function numSundays(startyear::Int, stopyear::Int)::Int
	isleapyear(year::Int)::Bool = (year % 400 == 0 || (year % 4 == 0 && year % 100 ≠ 0))

	monthdays = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
	totaldays = 1
	sundays = 0

	startyear < 1900  &&  error("Start date must be on or after 1900.")

	for year ∈ 1900:(startyear - 1)
		totaldays += isleapyear(year) ? 366 : 365
	end

	for year ∈ startyear:stopyear
		for month ∈ 1:12
			if month == 2 && isleapyear(year)
				totaldays += 29    # Leap year
			else
				totaldays += monthdays[month]
			end
			totaldays % 7 == 0  &&  (sundays += 1)
		end
	end

	# The final day is stopyear-01-01, so we have to check for an extra Sunday.
	totaldays % 7 == 0  &&  (sundays -= 1)
	return sundays
end

@time println(numSundays(1901, 2000))
```

    171
      0.000106 seconds (34 allocations: 928 bytes)

# Problem 020: Factorial digit sum

$ n! $ means $ n × (n − 1) × ... × 3 × 2 × 1 $.

For example, $ 10! = 10 × 9 × ... × 3 × 2 × 1 = 3628800 $,
and the sum of the digits in the number $ 10! $ is $ 3 + 6 + 2 + 8 + 8 + 0 + 0 = 27 $.

Find the sum of the digits in the number $ 100! $.

```julia
@time println(sum(digits(factorial(big(100)))))
```

    648
      0.000340 seconds (1.45 k allocations: 31.906 KiB)

# Problem 021: Amicable numbers

Let $ d(n) $ be defined as the sum of proper divisors of $ n $ (numbers less than $ n $ which divide evenly into $ n $).
If $ d(a) = b $ and $ d(b) = a $, where $ a ≠ b $, then $ a $ and $ b $ are an amicable pair and each of $ a $ and $ b $ are called amicable numbers.

For example, the proper divisors of $ 220 $ are $ 1, 2, 4, 5, 10, 11, 20, 22, 44, 55, 110 $; therefore $ d(220) = 284 $. The proper divisors of $ 284 $ are $ 1, 2, 4, 71, 142 $; so $ d(284) = 220 $.

Evaluate the sum of all the amicable numbers under $ 10000 $.

```julia
function sumamicable(limit::Int)::Int
	amicable = Int[]
	for i ∈ 2:limit
		j = sumdivisors(i) - i
		k = sumdivisors(j) - j
		if i ≠ j && i == k    # Check for amicability
			i ∉ amicable && append!(amicable, sort([i, j]))
		end
	end
	return sum(amicable)
end

@time println(sumamicable(10000))
```

# Problem 022: Names scores

Using `names.txt` (right click and 'Save Link/Target As...'), a 46K text file containing over five-thousand first names, begin by sorting it into alphabetical order. Then working out the alphabetical value for each name, multiply this value by its alphabetical position in the list to obtain a name score.

For example, when the list is sorted into alphabetical order, COLIN, which is worth $ 3 + 15 + 12 + 9 + 14 = 53 $, is the 938th name in the list. So, COLIN would obtain a score of $ 938 × 53 = 49714 $.

What is the total of all the name scores in the file?

```julia
alphabeticvalue(name::String)::Int  =  sum(char - 'A' + 1 for char in name)

function getwords(path::AbstractString)::Vector{String}
	words = open(path) do file
	    read(file, String)
	end
	return map(string, split(words, r"\W", keepempty=false))    # SubString to String
end

function totalscore(words::Vector{String})::Int
	sorted = sort(words)
	total::Int = 0
	for i ∈ 1:length(sorted)
		total += alphabeticvalue(sorted[i]) * i
	end
	return total
end

@time println(totalscore(getwords("p022_names.txt")))
```

    871198282
      0.014028 seconds (5.23 k allocations: 694.453 KiB)

# Problem 023: Non-abundant sums

A perfect number is a number for which the sum of its proper divisors is exactly equal to the number. For example, the sum of the proper divisors of $ 28 $ would be $ 1 + 2 + 4 + 7 + 14 = 28 $, which means that $ 28 $ is a perfect number.

A number $ n $ is called deficient if the sum of its proper divisors is less than $ n $ and it is called abundant if this sum exceeds $ n $.

As $ 12 $ is the smallest abundant number, $ 1 + 2 + 3 + 4 + 6 = 16 $, the smallest number that can be written as the sum of two abundant numbers is $ 24 $. By mathematical analysis, it can be shown that all integers greater than $ 28123 $ can be written as the sum of two abundant numbers. However, this upper limit cannot be reduced any further by analysis even though it is known that the greatest number that cannot be expressed as the sum of two abundant numbers is less than this limit.

Find the sum of all the positive integers which cannot be written as the sum of two abundant numbers.

```julia
isabundant(n::Int)::Bool  =  (sumdivisors(n) - n) > n

function abundants(upperbound)::Vector{Bool}
	lowerbound::Int = 12
	isabundantarray::Vector{Bool} = falses(upperbound)
	for i ∈ lowerbound:(upperbound - lowerbound)
		isabundant(i) && (isabundantarray[i] = true)
	end
	return isabundantarray
end

function nonabundantdecomposible_sum(upperbound::Int = 28123)::Int
	total::Int = 0
	lowerbound::Int = 12
	isabundantarray::Vector{Bool} = abundants(upperbound)

	for n ∈ 1:upperbound
		isdecomposible::Bool = false
		for i ∈ lowerbound:(n - lowerbound)
			if isabundantarray[i] && isabundantarray[n - i]
				isdecomposible = true
				break
			end
		end
		if !isdecomposible
			total += n
		end
	end

	return total
end

@time println(nonabundantdecomposible_sum())
```

# Problem 024: Lexicographic permutations

A permutation is an ordered arrangement of objects. For example, 3124 is one possible permutation of the digits 1, 2, 3 and 4. If all of the permutations are listed numerically or alphabetically, we call it lexicographic order. The lexicographic permutations of 0, 1 and 2 are:
```
012   021   102   120   201   210
```
What is the millionth lexicographic permutation of the digits 0, 1, 2, 3, 4, 5, 6, 7, 8 and 9?

```julia
function permute(sortedcharacters::Vector{Char}, order::Int)::Vector{Char}
	n = length(sortedcharacters)
	thisorder::Int = 0
	upperbound::Int = factorial(n - 1)
	poschar::Int = 1

	order == upperbound && return sortedcharacters    # Base case

	# Find the first character.
	# E.g. If the permutation starts with 2, the order will be between 2⋅9! and 3⋅9!
	while thisorder + upperbound < order
		thisorder += upperbound
		poschar += 1
	end

	unusedcharacters = [sortedcharacters[1:(poschar - 1)]; sortedcharacters[(poschar + 1):end]]
	return [sortedcharacters[poschar]; permute(unusedcharacters, order - thisorder)]
end

@time println(String(permute(collect('0':'9'), 1_000_000)))
```

    2783915460
      0.100794 seconds (69.86 k allocations: 3.579 MiB)

# Problem 025: 1000-digit Fibonacci number

The Fibonacci sequence is defined by the recurrence relation $ F_n = F_{n−1} + F_{n−2} $, where $ F_1 = 1 $ and $ F_2 = 1 $.

Hence the first 12 terms will be:
\begin{align}
	F_{1}  & =  1  \\
	F_{2}  & =  1  \\
	F_{3}  & =  2  \\
	F_{4}  & =  3  \\
	F_{5}  & =  5  \\
	F_{6}  & =  8  \\
	F_{7}  & =  13  \\
	F_{8}  & =  21  \\
	F_{9}  & =  34  \\
	F_{10}  & =  55  \\
	F_{11}  & =  89  \\
	F_{12}  & =  144  \\
\end{align}

The 12th term, $ F_{12} $, is the first term to contain three digits.

What is the index of the first term in the Fibonacci sequence to contain 1000 digits?

```julia
## Slower
function fibonacciindex_Binet(numdigits::Int)::Int
	index = 1
	while 1 + log10(fibonacci_Binet(index)) < numdigits
		index += 1
	end
	return index
end

function fibonacciindex_iter(numdigits::Int)::Int
	# sum::Int = 0
	a::BigInt = 1
	b::BigInt = 1
	index = 2
	while 1 + log10(b) < numdigits
		(a, b) = (b, a + b)
		index += 1
	end
	return index
end

@time println(fibonacciindex_Binet(1000))
@time println(fibonacciindex_iter(1000))
```

# Problem 026: Reciprocal cycles

A unit fraction contains $ 1 $ in the numerator. The decimal representation of the unit fractions with denominators $ 2 $ to $ 10 $ are given:
\begin{align}
\frac12  & =  0.5  \\
	\frac13  & =  0.\overline{3}  \\
	\frac14  & =  0.25  \\
	\frac15  & =  0.2  \\
	\frac16  & =  0.1\overline{6}  \\
	\frac17  & =  0.\overline{142857}  \\
	\frac18  & =  0.125  \\
	\frac19  & =  0.\overline{1}  \\
	\frac{1}{10}  & =  0.1  \\
\end{align}
Where $ 0.1\overline{6} $ means $ 0.166666... $, and has a 1-digit recurring cycle. It can be seen that $ \frac17 $ has a 6-digit recurring cycle.

Find the value of $ d < 1000 $ for which $ \frac1d $ contains the longest recurring cycle in its decimal fraction part.

### Solution

#### Method 1
The simplest approach is to use long division to find out when the remainder is repeats (or is 0).

#### Method 2
The idea comes from the following sources
*  https://en.wikipedia.org/wiki/Full_reptend_prime
*  https://en.wikipedia.org/wiki/Cyclic_number

I need to show that the final answer is the largest full reptend prime. Some guidance may be received from the following sources.
*  https://math.stackexchange.com/questions/56989/cyclic-numbers-are-characterized-by-the-reciprocals-of-full-reptend-primes
*  https://math.stackexchange.com/questions/443/why-is-the-decimal-representation-of-frac17-cyclical

This has to be thought about and formalized.

```julia
# Method 1. Use long division and check for repetition of the remainder.
function recurrencelength(d::Int, base::Int = 10)::Int
	num = base
	remainders = Int[1]
	while true
		num = remainders[end] * base
		remainder = num % d
		remainder == 0  &&  return 0    # no recurrence

		pos = findall(x -> x == remainder, remainders)
		length(pos) != 0  &&  (return length(remainders) - pos[1] + 1)
		push!(remainders, remainder)
	end
end

function maxrecurrencelength(limit::Int, base::Int = 10)::Int
	sieve = primes(limit)
	return sieve[argmax(map(recurrencelength, sieve))]
end

@time println(maxrecurrencelength(1000))
```

```julia
# Method 2: Find the largest full reptend prime.

# Use string instead of Integers to account for 0s.
# Also, permutations are more natural for strings than numbers.
function rotate(original::String)::Vector{String}
	len = length(original)
	rotations = Vector{String}(undef, len)
	rotations[1] = original
	for i ∈ 1:(len - 1)
		rotations[i + 1] = join([rotations[i][2:end], rotations[i][1]])
	end
	return rotations
end

function fermatquotient(prime::T, base::T = 10)::Union{T, BigInt} where T <: Integer
	return prime ≤ 19 ? (base^(prime - 1) - 1) ÷ prime : (big(base)^(prime - 1) - 1) ÷ prime
end

function isfullreptendprime(prime::T)::Bool where T <: Integer
	fq = fermatquotient(prime)
	rotations = rotate(string(fq, pad=(prime - 1)))    # Add leading 0s.
	return isempty(setdiff([string(fq * i, pad=(prime - 1)) for i ∈ 1:(prime - 1)], rotations))
end

function maxrecurringcycle(limit::Int)::Int
	sieve = primes(limit)
	for prime ∈ reverse(sieve)
		isfullreptendprime(prime)  &&  return prime
	end
end

@time println(maxrecurringcycle(1000))
```

```julia

```

# Problem 028: Number spiral diagonals

Starting with the number $ 1 $ and moving to the right in a clockwise direction a 5 by 5 spiral is formed as follows:
```
21 22 23 24 25
20  7  8  9 10
19  6  1  2 11
18  5  4  3 12
17 16 15 14 13
```
It can be verified that the sum of the numbers on the diagonals is $ 101 $.

What is the sum of the numbers on the diagonals in a 1001 by 1001 spiral formed in the same way?

The matrix can be written as
$$ \begin{matrix}
3^2 + 2 ⋅ 3 ⋅ 3 &  &  &  &  &  & 3^2 + 2 ⋅ 3 ⋅ 4 \\
 & 3^2 + 2 ⋅ 2 ⋅ 3 &  &  &  & 3^2 + 2 ⋅ 2 ⋅ 4 & \\
 &  & 1^2 + 2 ⋅ 1 ⋅ 3 &  & 1^2 + 2 ⋅ 1 ⋅ 4 &  & \\
 &  &  & 1 &  &  & \\
 &  & 1^2 + 2 ⋅ 1 ⋅ 2 &  & 1^2 + 2 ⋅ 1 ⋅ 1 &  & \\
 & 3^2 + 2 ⋅ 2 ⋅ 2 &  &  &  & 3^2 + 2 ⋅ 2 ⋅ 1 & \\
3^2 + 2 ⋅ 3 ⋅ 2 &  &  &  &  &  & 3^2 + 2 ⋅ 3 ⋅ 1
\end{matrix} $$

Let the dimension of the matrix be $ n \times n $, where $ n $ is necessarily odd. Let $ m = \frac{n - 1}{2} $. Then, going from inside to outside, the required sum is
$$ 1 + \sum_{i = 1}^m \left[ 4(-1 + 2i)^2 + 2 i (1 + 2 + 3 + 4) \right] = 1 + 4m + 2m(m+1) + \frac83 m(m+1)(2m+1) . $$

```julia
function number_spiral_diagonals(n::Int)    # n must be odd
	m::Int = (n - 1) ÷ 2
	return 1 + 4 * m + 2 * m * (m+1) + 8 * m * (m+1) * (2m+1) ÷ 3
end

@time println(number_spiral_diagonals(1001))
```

    669171001
      0.000103 seconds (34 allocations: 768 bytes)

# Problem 033: Digit cancelling fractions

The fraction $ \frac{49}{98} $ is a curious fraction, as an inexperienced mathematician in attempting to simplify it may incorrectly believe that $ \frac{49}{98} = \frac{4}{8} $, which is correct, is obtained by cancelling the 9s.

We shall consider fractions like, $ \frac{30}{50} = \frac{3}{5} $, to be trivial examples.

There are exactly four non-trivial examples of this type of fraction, less than one in value, and containing two digits in the numerator and denominator.

If the product of these four fractions is given in its lowest common terms, find the value of the denominator.

```julia
function iscancelling(p::Int, q::Int)::Bool
	digitsp = digits(p)
	digitsq = digits(q)

	# Special cases
	length(Set(digitsp)) + length(Set(digitsq)) == length(Set([digitsp; digitsq]))  &&  return false    # Trivially false: no common elements
	isempty(setdiff(digitsp, digitsq))  &&  return false    # Trivially false, e.g. 12 and 21.
	p % 10 == q % 10 == 0  &&  return false    # Trivially true, so ignore.

	posq = 1
	while posq ≤ length(digitsq)
		posp = findfirst([digitp == digitsq[posq] for digitp in digitsp])
		if !isnothing(posp)    # Remove the relevant elements
			digitsp = [digitsp[1:(posp - 1)]; digitsp[posp + 1:end]]
			digitsq = [digitsq[1:(posq - 1)]; digitsq[posq + 1:end]]
		end
		posq += 1
	end

	return parse(Int, join(string(d) for d in reverse(digitsp))) // parse(Int, join(string(d) for d in reverse(digitsq))) == p // q
end

function cancellingdenominator(numdigits::Int)::Int
	product = 1 // 1
	for p ∈ 10^(numdigits - 1):(10^numdigits - 1), q ∈ (p + 1):(10^numdigits - 1)
		iscancelling(p, q)  &&  (product *= p//q)
	end
	return denominator(product)
end

@time println(cancellingdenominator(2))
```

# Problem 034: Digit factorials

$ 145 $ is a curious number, as $ 1! + 4! + 5! = 1 + 24 + 120 = 145 $.

Find the sum of all numbers which are equal to the sum of the factorial of their digits.

Note: as $ 1! = 1 $ and $ 2! = 2 $ are not sums they are not included.

```julia
function sumdigitfactorial()::Int
	# Note that 9! + 9! = 2 ⋅ 9! > 99. At which point does this inequality reverse?
	# This can be further optimized. See
	# https://www.xarg.org/puzzle/project-euler/problem-34/
	function maxdigits()
		ninefactorial = factorial(9)
		i = 1
		num = 9
		while num < i * ninefactorial
			i += 1
			num = 10 * num + 9
		end
		return i
	end

	total = 0
	limit = 10^(maxdigits() + 1)
	for i ∈ 10:limit
		if sum(factorial(d) for d in digits(i)) == i
			total += i
		end
	end
	return total
end

@time println(sumdigitfactorial())
```

    40730
     23.519041 seconds (100.00 M allocations: 13.396 GiB, 19.11% gc time)

# Problem 035: Circular primes

The number $ 197 $, is called a circular prime because all rotations of the digits: 197, 971, and 719, are themselves prime.

There are thirteen such primes below $ 100 $: $ 2, 3, 5, 7, 11, 13, 17, 31, 37, 71, 73, 79, 97 $.

How many circular primes are there below one million?

```julia
# ToDo

# This code is also suboptimal. It checks the primality of all of the rotations.
# For example, it will check if 197 is prime while checking for the prime circularity of each of 197, 971, and 719.

function rotate(n::Int)::Vector{Int}
	numdigits = length(digits(n))
	rotations = Vector{Int}(undef, numdigits)
	rotations[1] = n
	for i ∈ 1:(numdigits - 1)
		rotated = (rotations[i] % 10) * 10^(numdigits-1) + (rotations[i] ÷ 10)
		rotations[i + 1] = rotated
	end
	return rotations
end

function count_circular_primes(limit::Int)::Int
	count::Int = 0
	count1::Int = 0
	sieve = primes(limit)
	for p ∈ sieve
		rotations = rotate(p)
# 		all(r ∈ sieve for r ∈ rotations)) && (count += 1)    # suboptimal: sieve is sorted.
		all(!isempty(searchsorted(sieve, r)) for r ∈ rotations) && (count += 1)
	end
	return count
end
@time println(count_circular_primes(1_000_000))
```

# Problem 036: Double-base palindromes

The decimal number, $ 585 = 1001001001_2 $ (binary), is palindromic in both bases.

Find the sum of all numbers, less than one million, which are palindromic in base 10 and base 2.

(Please note that the palindromic number, in either base, may not include leading zeros.)

```julia
function ispalindrome(n::Int, base::Int = 10)::Bool
	str = string(n, base = base)
	return reverse(str) == str
end

sum2Apalindromes(limit::Int)::Int = sum(i for i ∈ 1:limit if ispalindrome(i, 2) && ispalindrome(i, 10))

@time println(sum2Apalindromes(1_000_000))
```

# Problem 037: Truncatable primes

The number $ 3797 $ has an interesting property. Being prime itself, it is possible to continuously remove digits from left to right, and remain prime at each stage: $ 3797, 797, 97, 7 $. Similarly we can work from right to left: $ 3797, 379, 37, 3 $.

Find the sum of the only eleven primes that are both truncatable from left to right and right to left.

_NOTE_: $ 2, 3, 5, 7 $ are not considered to be truncatable primes.

```julia
# Gives the answer 748317 after running for a lifetime.
function istruncatableprime(n::Int)::Bool
	decimaldigits = length(digits(n))

	# Step 1: Check by removing lowest significant digits
	for i ∈ 0:(decimaldigits - 1)
		!isprime(n ÷ 10^i)  &&  (return false)
	end

	# Step 2: Check by removing highest significant digits
	for i ∈ decimaldigits:-1:1
		!isprime(n % 10^i)  &&  (return false)
	end

	println(n)
	return true
end

function sumtruncatableprimes()::Int
	truncatableprimes = Int[]
	for i ∈ primes(1_000_000)[5:end]
		istruncatableprime(i)  &&  push!(truncatableprimes, i)
		length(truncatableprimes) == 11  &&  break
	end
	return sum(truncatableprimes)
end

@time println(sumtruncatableprimes())
```

# Problem 041: Pandigital prime

We shall say that an $ n $-digit number is pandigital if it makes use of all the digits $ 1 $ to $ n $ exactly once. For example, $ 2143 $ is a $ 4 $-digit pandigital and is also prime.

What is the largest $ n $-digit pandigital prime that exists?

Clearly, for a $ n $-digit pandigital prime, it must be less than $ 987654321$. But the code does not run in finite time. **ToDo**.

```julia
# ToDo

function ispandigital(n::Int)::Bool
	decimal = digits(n)

	for i ∈ 1:length(decimal)
		length(filter(x -> x == i, decimal)) ≠ 1 && return false
	end

	for i ∈ union([0], (length(decimal)+1):9)
		length(filter(x -> x == i, decimal)) ≠ 0 && return false
	end

	return true
end

function largest_pandigital_prime()::Int
	sieve = primes(987654321)
	for prime ∈ Iterators.reverse(sieve)
		ispandigital(prime) && return prime
	end
end

# @time println(largest_pandigital_prime())
```

    largest_pandigital_prime (generic function with 1 method)

# Problem 050: Consecutive prime sum

The prime $ 41 $, can be written as the sum of six consecutive primes, $ 41 = 2 + 3 + 5 + 7 + 11 + 13 $.
This is the longest sum of consecutive primes that adds to a prime below one-hundred.

The longest sum of consecutive primes below one-thousand that adds to a prime, contains $ 21 $ terms, and is equal to $ 953 $.

Which prime, below one-million, can be written as the sum of the most consecutive primes?

### Comment

Here the optimization criteria is not the sum, but the number of primes contributing to the sum.

```julia
function consecutiveprimesum(limit::Int)::Int
	sieve = primes(limit)
	nummax = 0
	Σ0 = 0
	for i ∈ 1:length(sieve)
		j = length(sieve)
		Σ = sum(sieve[i:end])
		while Σ >= limit || Σ ∉ sieve    # Decrease until Σ < limit or is not prime
			Σ -= sieve[j]
			j -= 1
		end
		num = j - i + 1    # Number of consecutive primes
		if num > nummax
			nummax = num
			Σ0 = Σ
		end
	end
	return Σ0
end

@time println(consecutiveprimesum(1_000_000))
```

# Problem 067: Maximum path sum II

By starting at the top of the triangle below and moving to adjacent numbers on the row below, the maximum total from top to bottom is 23.
```
3
7 4
2 4 6
8 5 9 3
```
That is, $ 3 + 7 + 4 + 9 = 23 $.

Find the maximum total from top to bottom in `p067_triangle.txt` (right click and 'Save Link/Target As...'), a 15K text file containing a triangle with one-hundred rows.

Comment: For the code, see **Problem 18**.

```julia
triangle = open("p067_triangle.txt") do file
    read(file, String)
end
@time println(max_path_sum(numberarray(triangle)))
```

    7273
      0.003268 seconds (22.85 k allocations: 951.984 KiB)

```julia
sum(primes(10))
```

```julia

```


```python
import loader
from sympy import *
init_printing()
from root.solver import *
```

#### Find the general solution of  $y^{(4)} - 4y''' + 4y'' = 0$

```python
yc, p = nth_order_const_coeff(1, -4, 4, 0, 0)
p.display()
```

$\displaystyle \text{Characteristic equation: }$

$\displaystyle r^{4} - 4 r^{3} + 4 r^{2} = 0$

$\displaystyle \text{Roots: }\left\{ \begin{array}{ll}r_{1,2} = 0\\r_{3,4} = 2\\\end{array} \right.$

$\displaystyle \text{General Solution: }$

$\displaystyle y = C_{1} + C_{2} t + C_{3} e^{2 t} + C_{4} t e^{2 t}$

#### Find the general solution of  $y^{(6)} + y = 0$

```python
yc, p = nth_order_const_coeff(1, 0, 0, 0, 0, 0, 1)
p.display()
```

$\displaystyle \text{Characteristic equation: }$

$\displaystyle r^{6} + 1 = 0$

$\displaystyle \text{Roots: }\left\{ \begin{array}{ll}r_{1} = i\\r_{2} = - i\\r_{3} = - \frac{\sqrt{3}}{2} - \frac{i}{2}\\r_{4} = - \frac{\sqrt{3}}{2} + \frac{i}{2}\\r_{5} = \frac{\sqrt{3}}{2} - \frac{i}{2}\\r_{6} = \frac{\sqrt{3}}{2} + \frac{i}{2}\\\end{array} \right.$

$\displaystyle \text{General Solution: }$

$\displaystyle y = C_{1} \sin{\left(t \right)} + C_{2} \cos{\left(t \right)} + C_{3} e^{- \frac{\sqrt{3} t}{2}} \sin{\left(\frac{t}{2} \right)} + C_{4} e^{- \frac{\sqrt{3} t}{2}} \cos{\left(\frac{t}{2} \right)} + C_{5} e^{\frac{\sqrt{3} t}{2}} \sin{\left(\frac{t}{2} \right)} + C_{6} e^{\frac{\sqrt{3} t}{2}} \cos{\left(\frac{t}{2} \right)}$


# Supervised Learning: Decision Trees

One very direct way of performing supervised learning is expressing output as a combination of the predictors (features). An **adaptive basis-function model** (ABM) is one example of this.

$$f(x) = w_0 + \sum_{j=1}^k w_j \phi_j(\mathbf{x})$$

here, $\phi_j$ is a *basis function*, which is typically parametric:

$$\phi_j(\mathbf{x}) = \phi_j(\mathbf{x}|\alpha_j)$$

The parameter set for this model is thus $\theta = \{\mathbf{w} = w_0,\ldots,w_k; \mathbf{\alpha} = \alpha_1, \ldots, \alpha_k\}$. This model is *not* linear in the parameters.

**Decision trees** use an ABM to *recursively partition* the space of predictor variables into a piecewise-constant response surface. We can consider each component $j=1,\ldots,k$ to be a region in the response surface, and $w_j$ the expected response in that region.

$$f(x) = \sum_{j=1}^k w_j I(\mathbf{x} \in R_j)$$

Each paramter $\alpha_j$ encodes both (1) a variable used for splitting and (2) the corresponding threshold value. Specifically, the basis functions define the regions, and the weights encode the response value in each region.

This particular formulation implies a regression-type model, but we can generalize this to classification by storing the *distribution over classes* in each leaf, instead of the mean response.

To get a sense of how decision trees work, consider a diabetes dataset from which we wish to predict disease progression from a range of predictors. In the plot below, the response variable (`target`, an index of disease progression) is color-coded as a function of two variables, metabolic rate (`bmi`) and a blood serum measurement (`ltg`).

```python
%matplotlib inline
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
sns.set()
from sklearn.datasets import load_diabetes

# Predictors: "age" "sex" "bmi" "map" "tc"  "ldl" "hdl" "tch" "ltg" "glu"
diabetes = load_diabetes()
y = diabetes['target']
bmi, ltg = diabetes['data'][:,[2,8]].T

plt.scatter(ltg, bmi,  c=y, cmap="Reds")
plt.colorbar()
plt.xlabel('ltg'); plt.ylabel('bmi');
```

One approach to building a predictive model is to subdivide the variable space into regions, by sequentially subdividing each variable. For example, if we split `ltg` at a threshold value of -0.01, it does a reasonable job of isolating the large values in one of the resulting subspaces.

```python
ltg_split = -0.01
plt.scatter(ltg, bmi,  c=y, cmap="Reds")
plt.vlines(ltg_split, *plt.gca().get_ylim(), linestyles='dashed')
plt.colorbar()
plt.xlabel('ltg'); plt.ylabel('bmi');
```

However, that region still contains a fair number of low (light) values, so we can similarly bisect the region using a `bmi` value of -0.03 as a threshold value:

```python
bmi_split = -0.03
plt.scatter(ltg, bmi,  c=y, cmap="Reds")
plt.vlines(ltg_split, *plt.gca().get_ylim(), linestyles='dashed')
plt.hlines(bmi_split, ltg_split, plt.gca().get_xlim()[1], linestyles='dashed')
plt.colorbar()
plt.xlabel('ltg'); plt.ylabel('bmi');
```

We can use this partition to create a piecewise-constant function, which returns the average value of the observations in each region defined by the threshold values. We could then use this rudimentary function as a predictive model.

```python
np.mean(y[(bmi>bmi_split) & (ltg>ltg_split)])
```

```python
np.mean(y[(bmi<=bmi_split) & (ltg>ltg_split)])
```

```python
np.mean(y[ltg<ltg_split])
```

The choices for splitting the variables here were relatively arbitrary. Better choices can be made using a cost function $C$, such as residual sums of squares (RSS).

$$C = \sum_j \sum_{i \in R_j} (y_i - \hat{y}_{R_j})^2$$

where $\hat{y}_{R_j}$ is the mean response for the training observations in the jth region.

### Exercise

Use residual sums of squares to select competitive threshold values for the predictive model defined above

```python
# Write your answer here
```

The recursive partitioning demonstrated above results in a **decision tree**. The regions defined by the trees are called *terminal nodes*. Locations at which a predictor is split, such as `bmi`=-0.03, are called  *internal nodes*. As with this simple example, splits are not generally symmetric, in the sense that splits do not occur similarly on all branches.

Now consider a subset of three variables from the Titanic dataset, which we would like to use to predict survival from the disaster. The following describes one such decision tree:

We first check if gender of the passenger is male. If "no", we follow the right branch and end up in a leaf where the probability of survival is $p(y=1,x_1=F)=0.73$, so we predict survival ($y=1$) at this node (36% of observations fall under this leaf). If the passenger is male, we then check the age of the passenger. If he is older than 9.5 years, then the probability of survival $p(y=1,x_1=M,x_2>9.5)=0.17$, so we predict death ($y=0$). If, on the other hand, the passenger is younger than 9.5 years, we then check if the number of siblings and spouses on board was higher than 2.5; if "yes", then the probability of survival  $p(y=1, x_1=M, x_2>9.5, x_3>2.5)=0.05$, so we predict death, otherwise we predict survival with $p(y=1, x_1=M, x_2>9.5 , x_3 \lt 2.5)=0.89$. Hence, these probabilities are just the empirical fraction of positive examples that satisfy each conjunction of feature values, which defines a path from the root to a leaf.

There is no way to feasibly evaluate all possible partitions. Instead, the strategy is to use a top-down, **greedy** approach that is optimal (according to a particular cost function) for the current split only. By "greedy", we mean that at each step it chooses the most advantageous binary partition, not taking into account the impact of the choice on the quality of subsequent partitions.

$$(j^*, t^*) = \text{argmin}_{j,t} C(\{\mathbf{x}_i,y_i: x_{ij} \le t\}) + C(\{\mathbf{x}_i,y_i: x_{ij} \gt t\})$$

where $C$ is a cost function, $j$ and $t$ are a variable index and cutpoint, respectively. We will restrict consideration to binary partitions.

## Classification Trees

In addition to regression trees, we can also use decision trees on categorical outcomes, and these are called classification trees. The primary difference in implementation is that residual sums of squares is no longer an appropriate splitting criterion.

### Entropy

An alternative splitting criterion for decision tree learning algorithms is *information gain*. It measures how well a particular attribute distinguishes among different target classifications. Information gain is measured in terms of the expected reduction in the entropy or impurity of the data. The entropy of a set of probabilities is:

$$H(p) = -\sum_i p_i log_2(p_i)$$

If we have a set of binary responses from some variable, all of which are positive/true/1, then knowing the values of the variable does not hold any predictive value for us, since all the outcomes are positive. Hence, the entropy is zero:

```python
import numpy as np

entropy = lambda p: -np.sum(p * np.log2(p)) if not 0 in p else 0
```

```python
entropy([.4,.6])
```

However, if the variable splits responses into equal numbers of positive and negative values, then entropy is maximized, and we wish to know about the feature:

```python
entropy([0.5, 0.5])
```

```python
pvals = np.linspace(0, 1)
plt.plot(pvals, [entropy([p,1-p]) for p in pvals])
```

The entropy calculation tells us how much additional information we would obtain with knowledge of the variable.

So, if we have a set of candidate covariates from which to choose as a node in a decision tree, we should choose the one that gives us the most information about the response variable (*i.e.* the one with the highest entropy).

### Misclassification Rate

Alternatively, we can use the misclassification rate:

$$C(j,t) = \frac{1}{n_{jt}} \sum_{y_i: x_{ij} \gt t} I(y_i \ne \hat{y})$$

where $\hat{y}$ is the most probable class label and $n_{ij}$ is the number of observations in the data subset obtained from splitting via $j,t$.

### Gini index

The Gini index is simply the expected error rate:

$$C(j,t) = \sum_{k=1}^K \hat{\pi}_{jt}[k] (1 - \hat{\pi}_{jt}[k]) = 1 - \sum_{k=1}^K \hat{\pi}_{jt}[k]^2$$

where $\hat{\pi}_{jt}[k]$ is the probability of an observation being correctly classified as class $k$ for the data subset obtained from splitting via $j,t$ (hence, $(1 - \hat{\pi}_{jt}[k])$ is the misclassification probability).

```python
gini = lambda p: 1. - (np.array(p)**2).sum()
```

```python
pvals = np.linspace(0, 1)
plt.plot(pvals, [entropy([p,1-p])/2. for p in pvals], label='Entropy')
plt.plot(pvals, [gini([p,1-p]) for p in pvals], label='Gini')
plt.legend()
```

## ID3

A given cost function can be used to construct a decision tree via one of several algorithms. The Iterative Dichotomiser 3 (ID3) is on such algorithm, which uses entropy, and a related concept, *information gain*, to choose features and partitions at each classification step in the tree.

Information gain is the difference between the current entropy of a system and the entropy measured after a feature is chosen. If $S$ is a set of examples and $X$ is a possible feature on which to partition the examples, then:

$$G(S,X) = \text{Entropy}(S) - \sum_{x \in X} \frac{\#(S_x)}{\#(S)} \text{Entropy}(S_x)$$

where $\#$ is the count function and $x$ is a particular value of $X$.

Let's say $S$ is a set of survival events, $S = \{s_1=survived, s_2=died, s_3=died, s_4=died\}$ and a particular variable $X$ can have values $\{x_1, x_2, x_3\}$. To perform a sample calculation of information gain, we will say that:

* $X(s_1) = x_2$
* $X(s_2) = x_2$
* $X(s_3) = x_3$
* $X(s_4) = x_1$

The current entropy of this state is:

$$\begin{align}
\text{Entropy}(S) &= -p^{(+)} \log_2(p^{(+)}) - p^{(-)} \log_2(p^{(-)}) \\
&= -0.25 \log_2(0.25) - 0.75 \log_2(0.75) \\
&= 0.5 + 0.311 = 0.811
\end{align}$$

Now, we need to compute the information after selecting variable $X$, which is the sum of three terms:

$$\begin{align}
\frac{\#(S_{x1})}{\#(S)} \text{Entropy}(S) &= 0.25 (-0 \log_2(0) - 1 \log_2(1)) = 0\\
\frac{\#(S_{x2})}{\#(S)} \text{Entropy}(S) &= 0.5 (-0.5 \log_2(0.5) - 0.5 \log_2(0.5) = 0.5\\
\frac{\#(S_{x3})}{\#(S)} \text{Entropy}(S) &= 0.25 (-0 \log_2(0) - 1 \log_2 1) = 0\\
\end{align}$$

Therefore, the information gain is:

$$G(S,X) = 0.811 - (0 + 0.5 + 0) = 0.311$$

```python
import numpy as np

def info_gain(X, y, feature):
    # Calculates the information gain based on entropy

    gain = 0
    n = len(X)

    # List the values that feature can take
    values = list(set(X[feature]))

    feature_counts = np.zeros(len(values))
    E = np.zeros(len(values))
    ivalue = 0

    # Find where those values appear in X[feature] and the corresponding class
    for value in values:

        new_y = [y[i] for i,d in enumerate(X[feature].values) if d==value]
        feature_counts[ivalue] += len(new_y)

        # Get the values in newClasses
        class_values = list(set(new_y))

        class_counts = np.zeros(len(class_values))
        iclass = 0
        for v in class_values:
            for c in new_y:
                if c == v:
                    class_counts[iclass] += 1
            iclass += 1

        nc = float(np.sum(class_counts))
        new_entropy = entropy([class_counts[c] / nc for c in range(len(class_values))])
        E[ivalue] += new_entropy

        # Computes both the Gini gain and the entropy
        gain += float(feature_counts[ivalue])/n * E[ivalue]
        ivalue += 1

    return gain
```

Consider a few variables from the titanic database:

```python
titanic = pd.read_excel("../data/titanic.xls", "titanic")
titanic.head(1)
```

Here, we have selected pasenger class (`pclass`), sex, port of embarcation (`embarked`), and a derived variable called `adult`. We can calculate the information gain for each of these.

```python
y = titanic['survived']
X = titanic[['pclass','sex','embarked']]
X['adult'] = titanic.age<17

info_gain(X, y, 'pclass')
```

```python
info_gain(X, y, 'sex')
```

```python
info_gain(X, y, 'embarked')
```

```python
info_gain(X, y, 'adult')
```

Hence, the ID3 algorithm computes the information gain for each variable, selecting the one with the highest value (in this case, `adult`). In this way, it searches the "tree space" according to a greedy strategy.

A tree can be constructed by recursively selecting the feature from the current dataset with the largest information gain, then removing it from the datset. Recursion stops when there are either no variables remaining, or there is only one class left in the subset (*e.g.* all `True` or all `False`).

The ID3 algorithm is as follows:

> * if all response data have the same class:
>
>     - return leaf with data label
>
> * else if no features:
>
>     - return leaf with most common label
>
> * else:
>
>     - choose variable $X'$ that maximizes information gain to be a tree node
>     - add branch from node for each value of $X'$
>     - for each branch of node:
>
>         * calculate $S_{x}$ by removing $X'$ from $S$
>         * set $S=S_{x}$ and call algorithm again

The greedy approach of maximizing information gain at each step tends to bias solutions towards smaller trees.

## Decision Trees in `scikit-learn`

Classification trees, either binary or multi-class, are implemented in `scikit-learn` in the `DecisionTreeClassifier` class. Where trees are binary, it expects the response variable to be coded as `[-1,1]` for negative and positive outcomes.

Let's build a decision tree on a wine dataset.

```python
wine = pd.read_table("../data/wine.dat", sep='\s+')

attributes = ['Alcohol',
            'Malic acid',
            'Ash',
            'Alcalinity of ash',
            'Magnesium',
            'Total phenols',
            'Flavanoids',
            'Nonflavanoid phenols',
            'Proanthocyanins',
            'Color intensity',
            'Hue',
            'OD280/OD315 of diluted wines',
            'Proline']

grape = wine.pop('region')
y = grape
wine.columns = attributes
X = wine
```

```python
from sklearn import tree
from sklearn import model_selection

X_train, X_test, y_train, y_test = model_selection.train_test_split(
        X, y, test_size=0.4, random_state=0)

clf = tree.DecisionTreeClassifier(criterion='entropy',
                                  max_features="auto",
                                  min_samples_leaf=10)
clf.fit(X_train, y_train)
```

If you have [GraphViz](http://www.graphviz.org) installed, you can draw the resulting tree:

```python
with open("wine.dot", 'w') as f:
    f = tree.export_graphviz(clf, out_file=f)
```

```python
! dot -Tpng wine.dot -o wine.png
```

```python
for i,x in enumerate(X.columns):
    print(i,x)
```

```python
from IPython.core.display import Image
Image("wine.png")
```

```python
preds = clf.predict(X_test)
pd.crosstab(y_test, preds, rownames=['actual'],
            colnames=['prediction'])
```

### Pruning

Despite the *inductive bias* associated with trees that tend to make them small, the ID3 algorithm continues choosing nodes and branches until either it runs out of variables, or all outputs are of the same class. This can clearly lead to overfit trees.

To prevent overfitting, we can stop growing the tree if the information gain (or reduction in error, etc.) is not sufficient to justify the extra complexity of adding another node. However, this simple rule is not optimal, because an uninformative subtree can lead to informative ones later on.

The standard approach is therefore to grow a full tree, and then to *prune* it. The easiest approach is to remove branches that give the least increase in the error (information gain). To determine how far back to prune, we can evaluate the cross-validated error on each candidate pruning, and then pick the tree whose CV error is within 1 standard error of the minimum.

Analogous to the lasso or ridge regression, we can penalize the number of terminal nodes in a tree:

$$\sum_{m=1}^{|T|} \sum_{x_i \in R_m} (y_i - \hat{y}_{R_j})^2 + \alpha |T|$$

where $|T|$ is the number of terminal nodes in tree T.

### Pruned Decision Tree Algorithm

1. Use recursive binary splitting to grow a large tree, such that each terminal node has fewer than some minimum number of observations.
2. Apply pruning to obtain a sequence of best subtrees, as a function of $\alpha$.
3. Use k-fold cross-validation to choose $\alpha$. Average results and pick $\alpha$ to minimize the average error.
4. Return subtree from (2) that corresponds to chosen $\alpha$.

## Random Forests

Decision trees have several advantages:

* ease of interpretation
* handles continuous and discrete features
* invariant to monotone transformation of features
* variable selection automated
* robust
* scalable

However, relative to other statistical learning methods, trees do not predict very accurately, due to the greedy nature of the tree construction algorithm. Also, trees tend to be **unstable**, as small changes to the inputs can have large effects on the structure of the tree; poor decisions near the root of the tree will propogate to the rest of the tree. Hence, trees are **high variance** (*i.e.* noisy) estimators.

One way to reduce the variance of an estimate is to average together many estimates. In the case of decision trees, we can train $T$ different trees on random subsets of the data (with replacement) then average according to:

$$\hat{f}(\mathbf{x}) = \frac{1}{T} \sum_{i=1}^T f_t(\mathbf{x})$$

where $f_t$ is the $t^{th}$ tree. This approach is called "bootstrap aggregating", or **bagging**.

Note that, since we are averaging over trees, there is *no need to prune*. With bagging, we reduce variance by averaging, rather than by pruning.

```python
from sklearn.ensemble import BaggingClassifier

bc = BaggingClassifier(n_jobs=4, oob_score=True)
bc
```

```python
bc.fit(X_train, y_train)

preds = bc.predict(X_test)
pd.crosstab(y_test, preds, rownames=['actual'],
            colnames=['prediction'])
```

Test error of a bagged model is measured by estimating **out-of-bag error**.

On average, each bagged tree uses about 2/3 of observations, leaving the remaining third as "out-of bag". The response for the ith observation for each of the trees in which that observation was excluded (on average, B/3) is averaged. This essentially the same as performing leave-one-out (LOO) cross-validation.

```python
bc.oob_score_
```

This approach is an **ensemble learning** method, because it takes a set of *weak* learners, and combines them to construct a *strong* learner that is more robust, with lower generalization error.

An average of B trees, each with variance $\sigma^2$, has variance $\sigma^2/B$. If the variables are simply identically distributed, with positive pairwise correlation $\rho$, then the variance of the average of the B trees is:

$$\rho \sigma^2 + \frac{1-\rho}{B}\sigma^2$$

As the number of trees becomes large, the second term goes to zero. Further reductions in variance are limited by the size of the correlation among the trees $\rho$.

**Random forests** improves upon bagging by creating a set of decision trees that are less correlated than bootstrapped trees. This is done by selecting from only a subset $m$ out of $M$ possible predictors at each split. Typically, we choose approximately the square root of the available number.

This procedure is used to create a set of trees, most of which will be poor at predicting a given observation. However, classification is based on a *majority vote* of the constituent trees.

```python
from sklearn.ensemble import RandomForestClassifier

rf = RandomForestClassifier(n_jobs=4)
rf.fit(X_train, y_train)

preds = rf.predict(X_test)
pd.crosstab(y_test, preds, rownames=['actual'],
            colnames=['prediction'])
```

With random forests, it is possible to quantify the relative importance of feature inputs for classification. In scikit-learn, the Gini index (recall, a measure of error reduction) is calculated for each internal node that splits on a particular feature of a given tree, which is multiplied by the number of samples that were routed to the node (this approximates the probability of reaching that node). For each variable, this quantity is averaged over the trees in the forest to yield a measure of importance.

```python
importances = rf.feature_importances_
indices = np.argsort(importances)[::-1]

# Print the feature ranking
print("Feature ranking:")

for f in range(X.shape[1]):
    print("%d. %s (%f)" % (f + 1, X.columns[indices[f]], importances[indices[f]]))
```

```python
plt.figure()
plt.title("Feature importances")
plt.bar(range(X.shape[1]), importances[indices],
       color="r", align="center")
plt.xticks(range(X.shape[1]), X.columns[indices], rotation=90)
plt.xlim([-1, X.shape[1]]);
```

`RandomForestClassifier` uses the Gini impurity index by default; one may instead use the entropy information gain as a criterion.

```python
rf = RandomForestClassifier(n_jobs=4, criterion='entropy')
rf.fit(X_train, y_train)
importances = rf.feature_importances_
indices = np.argsort(importances)[::-1]

# Print the feature ranking
print("Feature ranking:")

for f in range(X.shape[1]):
    print("%d. %s (%f)" % (f + 1, X.columns[indices[f]], importances[indices[f]]))
```

## Decision Tree Regression

While it may not be apparent how to use trees for regression analysis, it requires only a straightforward modification to the algorithm. A popular tree-based regression algorithm is the **classification and regression tree** (CART).

The file `TNNASHVI.txt` in your data directory contains daily temperature readings for Nashville, courtesy of the [Average Daily Temperature Archive](http://academic.udayton.edu/kissock/http/Weather/). This data, as one would expect, oscillates annually. We can use a decision tree regression model to fit the data.

```python
daily_temps = pd.read_table("../data/TNNASHVI.txt", sep='\s+',
                            names=['month','day','year','temp'], na_values=-99)

```

```python
daily_temps.temp[daily_temps.year>2010].plot(style='b.', figsize=(10,6))
```

In this context, none of the cost functions considered so far would be appropriate. Instead, it would be more suitable to use something like mean squared error (MSE) to guide the growth of the tree. With this, we can proceed to choose (1) a variable on which to split the dataset and (2) (in the case of continuous features) a value of the variable at which to place a node.

Recall that the output of a tree is just a constant value for each leaf; here, we simply return the average of all the response values in the region. Thus, we choose a cut point that minimizes the MSE at each step.

```python
# Transmogrify data
y = daily_temps.temp[daily_temps.year>2010]
X = np.atleast_2d(np.arange(len(y))).T
```

```python
from sklearn.tree import DecisionTreeRegressor
clf = DecisionTreeRegressor(max_depth=7, min_samples_leaf=2)
clf.fit(X, y)

X_fit = np.linspace(0, len(X), 1000).reshape((-1, 1))
y_fit_1 = clf.predict(X_fit)

plt.plot(X.ravel(), y, '.k', alpha=0.3)
plt.plot(X_fit.ravel(), y_fit_1, color='red')

```

A single decision tree allows us to estimate the signal in a non-parametric way,
but clearly has some issues.  In some regions, the model shows high bias and
under-fits the data
(seen in the long flat lines which don't follow the contours of the data),
while in other regions the model shows high variance and over-fits the data
(reflected in the narrow spikes which are influenced by noise in single points).

One way to address this is to use an ensemble method, like random forests, so that the
effects of their over-fitting go away on average.

Here we will use a random forest of 200 trees to reduce the tendency of each
tree to over-fitting the data.

```python
from sklearn.ensemble import RandomForestRegressor
clf = RandomForestRegressor(n_estimators=200, max_depth=9,
                            min_samples_leaf=10)
clf.fit(X, y)

y_fit_200 = clf.predict(X_fit)

plt.plot(X.ravel(), y, '.k', alpha=0.3)
plt.plot(X_fit.ravel(), y_fit_200, color='red')

```

### Prediction intervals

The predictions from random forests are not accompanied by estimates of uncertainty, as is the case with Bayesian regression models. However, it is possible to obtain probability intervals using a random forests approach. Since we are using an ensemble of trees, it is possible to track *all* predicted values for all leaf nodes in a random forest, rather than just the mean or modal value. This results in conditional distributions $P(y|X=x)$ for every x, from which percentiles can be calculated for desired endpoints in a prediction interval. This approach is called **quantile regression forests**.

To implement quantile regression forests in scikit-learn, we need to allow each tree to grow so that each leaf node contains exactly one value. Then, each tree returns a single response variable, from which a conditional distribution can be approximated. Of course, fully expanding trees will result in overfitting, but these can also be cross-validated.

scikit-learn does not automatically calculate prediction intervals, but the estimators from each constitutent tree in the `RandomForestRegressor` is avaialable, from which individual tree predictions can be made.

```python
def prediction_intervals(mod, X, alpha=0.05):
    preds = [pred.predict(X) for pred in mod.estimators_]
    lower = np.percentile(preds, 100*(alpha/2), axis=0)
    upper = np.percentile(preds, 100*(1-alpha/2), axis=0)
    return lower, upper, np.array(preds)
```

```python
X_train, X_test, y_train, y_test = model_selection.train_test_split(
        X, y, test_size=0.4, random_state=0)
```

```python
clf = RandomForestRegressor(n_estimators=1000, min_samples_leaf=1)
clf.fit(X_train, y_train)

y_fit_200 = clf.predict(X_fit)
```

```python
lower, upper, preds = prediction_intervals(clf, X_test, alpha=0.1)
```

```python
x_sorted = np.sort(X_test.ravel())
order = np.argsort(X_test.ravel())
plt.errorbar(x_sorted, y_test.values[order],
             yerr=[(y_test.values-lower)[order], (upper-y_test.values)[order]])
plt.plot(X_test, y_test, '.r', alpha=0.3)
```

### Exercise

Select the optimal random forest regression model for the nashville daily temperature data via cross-validation in `scikit-learn`. Use the number of estimators and the maximim leaf nodes as tuning parameters.

```python
# Write your answer here
```

## References

Meinshausen, N. [Quantile Regression Forests](http://www.jmlr.org/papers/volume7/meinshausen06a/meinshausen06a.pdf) Journal of Machine Learning Research 7 (2006) 983–999

T. Hastie, R. Tibshirani and J. Friedman. (2009) [Elements of Statistical Learning: Data Mining, Inference, and Prediction](http://statweb.stanford.edu/~tibs/ElemStatLearn/), second edition. Springer.