#coding=utf-8

import requests
import sys
import threading
from time import sleep
import csv

csvfile=open('valid_ip.csv','w')
writer=csv.writer(csvfile)

class test_thread(threading.Thread):
    def __init__(self,ip,port):
        threading.Thread.__init__(self)
        self.ip=ip
        self.port=port
        #self.protocol=protocol
    def run(self):
        s = requests.session()
        s.keep_alive = False
        requests.adapters.DEFAULT_RETRIES = 5
        proxies = {"http": "http://"+self.ip + ":" + str(self.port)}
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36',
            'Connection': 'close'
        }
        try:
            maskedIP = str(requests.get("http://icanhazip.com/", headers=headers, timeout=8, proxies=proxies).text).replace("\n","")
        except Exception as e:
            return
        if maskedIP == self.ip:
            writer.writerow([self.ip, self.port])
            print(self.ip+" "+self.port)

def test_proxy(filename):
    reader=csv.reader(open(filename))
    for row in reader:
        ip, port=row[0],row[1]
        while threading.activeCount() >30:
            sleep(2)
        test_thread(ip,port).start()

def main():
    filename=sys.argv[1]
    print("The result will be saved in valid_ip.csv")
    test_proxy(filename)

if __name__ == '__main__':
    main()


import datetime as dt
from random import randint

from django.test import SimpleTestCase
from django.urls import reverse
from django.utils import timezone
from freezegun import freeze_time

from zoo_checks.ingest import TRACKS_REQ_COLS
from zoo_checks.models import AnimalCount, Enclosure
from zoo_checks.views import (
    enclosure_counts_to_dict,
    get_accessible_enclosures,
    get_selected_role,
    redirect_if_not_permitted,
)

DST_DATETIME = dt.datetime(2021, 3, 14, 12, 0, 0)

def test_home(client, user_base):
    client.force_login(user_base)
    url = reverse("home")
    resp = client.get(url)

    assert resp.status_code == 200
    assert resp.context["selected_role"] is None
    assert user_base.first_name in resp.content.decode()
    assert (
        "No enclosures. Contact a manager to be added to a role"
        in resp.content.decode()
    )

def test_home_counts(client, create_many_counts, user_base):

    num_enc = 7
    num_anim = 4
    num_species = 5

    create_many_counts(
        num_enc=num_enc,
        num_anim=num_anim,
        num_species=num_species,
    )

    client.force_login(user_base)
    url = reverse("home")
    resp = client.get(url)

    assert resp.status_code == 200
    assert list(resp.context["roles"]) == list(user_base.roles.all())
    assert resp.context["selected_role"] is None
    assert "Individuals" in resp.content.decode()

def test_count(
    client, user_base, enclosure_base, animal_A, animal_count_A_BAR, group_B
):
    client.force_login(user_base)

    # GET

    resp = client.get(f"/count/{enclosure_base.slug}/")
    assert resp.context["enclosure"] == enclosure_base
    assert resp.context["conditions"] == AnimalCount.CONDITIONS

    # untested context:
    # "dateday": dateday,
    # "species_formset": species_formset,
    # "groups_formset": groups_formset,
    # "animals_formset": animals_formset,
    # "formset_order": formset_order,
    # "dateform": dateform,
    # "conditions": AnimalCount.CONDITIONS,

    # test a different date

    # create some counts

    # todo:
    # POST

def test_count_todays_date(
    client, user_base, enclosure_base, animal_A, animal_count_A_BAR, group_B
):
    """ whenever you ge the count, it should always have todays date in it """

    client.force_login(user_base)

    resp = client.get(f"/count/{enclosure_base.slug}/")
    assert "Today's Count" in resp.content.decode()

@freeze_time(DST_DATETIME)  # daylight savings
def test_count_todays_date_dst(
    client, user_base, enclosure_base, animal_A, animal_count_A_BAR, group_B
):
    """mock the current date to be on daylight savings (@12pm)
    we should still be seeing "todays date" -- this requires us being at 12pm currently
    because today_time utility uses localtime to generate a date
    """

    client.force_login(user_base)

    resp = client.get(f"/count/{enclosure_base.slug}/")
    assert "Today's Count" in resp.content.decode()
    assert resp.context["todays_date"] == DST_DATETIME.strftime("%Y-%m-%d")

def test_tally_date_handler(client, enclosure_base, user_base):
    client.force_login(user_base)

    yesterday = dt.date.today() - dt.timedelta(days=1)
    tomorrow = dt.date.today() + dt.timedelta(days=1)

    resp = client.post(f"/tally_date_handler/{enclosure_base.slug}")
    assert resp.status_code == 302
    SimpleTestCase().assertRedirects(resp, f"/count/{enclosure_base.slug}/")

    resp = client.get(f"/tally_date_handler/{enclosure_base.slug}")
    assert resp.status_code == 302
    SimpleTestCase().assertRedirects(resp, f"/count/{enclosure_base.slug}/")

    resp = client.post(
        f"/tally_date_handler/{enclosure_base.slug}",
        {"tally_date": yesterday.strftime("%m/%d/%Y")},
    )
    assert resp.status_code == 302
    SimpleTestCase().assertRedirects(
        resp,
        (
            "/count/"
            f"{enclosure_base.slug}/{yesterday.year}/{yesterday.month}/{yesterday.day}/"
        ),
    )

    resp = client.post(
        f"/tally_date_handler/{enclosure_base.slug}",
        {"tally_date": "not_a_date"},
    )
    assert resp.status_code == 302
    # normally this is in resp.context but context is None since we're doing a redirect
    messages = list(resp.wsgi_request._messages)
    assert messages[0].message == "Error in date entered"
    SimpleTestCase().assertRedirects(resp, f"/count/{enclosure_base.slug}/")

    resp = client.post(
        f"/tally_date_handler/{enclosure_base.slug}",
        {"tally_date": tomorrow.strftime("%m/%d/%Y")},
    )
    assert resp.status_code == 302
    messages = list(resp.wsgi_request._messages)
    assert messages[0].message == "Error in date entered"
    SimpleTestCase().assertRedirects(resp, f"/count/{enclosure_base.slug}/")

def test_edit_species_count(
    client,
    user_base,
    species_base,
    enclosure_base,
    enclosure_factory,
    species_base_count,
):
    client.force_login(user_base)

    yesterday_time = timezone.localtime() - dt.timedelta(days=1)
    yesterday = yesterday_time.date()

    enc_not_permit = enclosure_factory("not_permit", None)

    # not permitted
    resp = client.get(
        (
            "/edit_species_count/"
            f"{species_base.slug}/{enc_not_permit.name}"
            f"/{yesterday.year}/{yesterday.month}/{yesterday.day}/"
        )
    )
    assert resp.status_code == 302
    SimpleTestCase().assertRedirects(resp, "/")

    count = species_base_count(yesterday_time)

    # GET
    resp = client.get(
        (
            "/edit_species_count/"
            f"{species_base.slug}/{enclosure_base.name}/"
            f"{yesterday.year}/{yesterday.month}/{yesterday.day}/"
        )
    )
    assert resp.status_code == 200
    assert resp.context["species"] == species_base
    assert resp.context["enclosure"] == enclosure_base
    assert resp.context["dateday"].year == yesterday.year
    assert resp.context["dateday"].month == yesterday.month
    assert resp.context["dateday"].day == yesterday.day
    assert resp.context["count"] == count
    # todo: form context

    # POST
    post_data = {
        "count": randint(1, 400),
        "species": species_base.id,
        "enclosure": enclosure_base.id,
    }
    resp = client.post(
        (
            "/edit_species_count/"
            f"{species_base.slug}/{enclosure_base.name}/"
            f"{yesterday.year}/{yesterday.month}/{yesterday.day}/"
        ),
        data=post_data,
        follow=True,
    )
    assert resp.status_code == 200
    assert resp.redirect_chain == [(f"/count/{enclosure_base.slug}/", 302)]
    assert species_base.counts.latest("datetimecounted").count == post_data["count"]

def test_edit_group_count(
    client,
    user_base,
    group_B,
    group_factory,
    enclosure_base,
    enclosure_factory,
    group_B_count_datetime_factory,
):
    client.force_login(user_base)

    yesterday_time = timezone.localtime() - dt.timedelta(days=1)
    yesterday = yesterday_time.date()

    enc_not_permit = enclosure_factory("not_permit", None)
    group_not_permit = group_factory("abc123", 5, 4, 3, 12, enclosure=enc_not_permit)

    # not permitted
    resp = client.get(
        (
            "/edit_group_count/"
            f"{group_not_permit.accession_number}/"
            f"{yesterday.year}/{yesterday.month}/{yesterday.day}/"
        )
    )
    assert resp.status_code == 302
    SimpleTestCase().assertRedirects(resp, "/")

    count = group_B_count_datetime_factory(yesterday_time)

    # GET
    resp = client.get(
        (
            "/edit_group_count/"
            f"{group_B.accession_number}/"
            f"{yesterday.year}/{yesterday.month}/{yesterday.day}/"
        )
    )
    assert resp.status_code == 200
    assert resp.context["count"] == count
    assert resp.context["group"] == group_B
    assert resp.context["enclosure"] == enclosure_base
    assert resp.context["dateday"].year == yesterday.year
    assert resp.context["dateday"].month == yesterday.month
    assert resp.context["dateday"].day == yesterday.day
    # todo: form context

    # POST
    count_bar = randint(1, 400)
    count_seen = count_bar + randint(5, 25)
    count_total = count_bar + count_seen
    post_data = {
        "count_seen": count_seen,
        "enclosure": enclosure_base.id,
        "count_bar": count_bar,
        "comment": "this is a randomly generated count",
        "group": group_B.id,
        "count_total": count_total,
        "needs_attn": False,
    }
    resp = client.post(
        (
            "/edit_group_count/"
            f"{group_B.accession_number}/"
            f"{yesterday.year}/{yesterday.month}/{yesterday.day}/"
        ),
        data=post_data,
        follow=True,
    )
    assert resp.status_code == 200
    assert resp.redirect_chain == [(f"/count/{enclosure_base.slug}/", 302)]

    latest_count = group_B.counts.latest("datetimecounted")
    assert latest_count.count_seen == post_data["count_seen"]
    assert latest_count.count_bar == post_data["count_bar"]

def test_edit_animal_count(
    client,
    user_base,
    enclosure_factory,
    animal_factory,
    animal_A,
    enclosure_base,
    animal_count_A_BAR_datetime_factory,
):
    client.force_login(user_base)

    yesterday_time = timezone.localtime() - dt.timedelta(days=1)
    yesterday = yesterday_time.date()

    enc_not_permit = enclosure_factory("not_permit", None)

    anim_not_permit = animal_factory(
        name="kermitthefrog",
        identifier="green",
        sex="m",
        accession_number="abc123",
        enclosure=enc_not_permit,
    )

    # not permitted
    resp = client.get(
        (
            "/edit_animal_count/"
            f"{anim_not_permit.accession_number}/"
            f"{yesterday.year}/{yesterday.month}/{yesterday.day}/"
        )
    )
    assert resp.status_code == 302
    SimpleTestCase().assertRedirects(resp, "/")

    count = animal_count_A_BAR_datetime_factory(yesterday_time)

    # GET
    resp = client.get(
        (
            "/edit_animal_count/"
            f"{animal_A.accession_number}/"
            f"{yesterday.year}/{yesterday.month}/{yesterday.day}/"
        )
    )
    assert resp.status_code == 200
    assert resp.context["count"] == count
    assert resp.context["animal"] == animal_A
    assert resp.context["enclosure"] == enclosure_base
    assert resp.context["dateday"].year == yesterday.year
    assert resp.context["dateday"].month == yesterday.month
    assert resp.context["dateday"].day == yesterday.day
    # todo: form context

    # POST
    post_data = {
        "condition": "NA",
        "comment": "this is a test count",
        "animal": animal_A.id,
        "enclosure": enclosure_base.id,
    }
    resp = client.post(
        (
            "/edit_animal_count/"
            f"{animal_A.accession_number}/"
            f"{yesterday.year}/{yesterday.month}/{yesterday.day}/"
        ),
        data=post_data,
        follow=True,
    )
    assert resp.status_code == 200
    assert resp.redirect_chain == [(f"/count/{enclosure_base.slug}/", 302)]

    latest_count = animal_A.conditions.latest("datetimecounted")
    assert latest_count.condition == "NA"
    assert latest_count.comment == "this is a test count"
    assert latest_count.enclosure == enclosure_base

def test_animal_counts(
    client,
    animal_A,
    animal_count_factory,
    user_base,
    user_factory,
    enclosure_base,
):

    num_counts = 100

    rando_user = user_factory("rando")
    client.force_login(rando_user)
    url = reverse("animal_counts", args=[animal_A.accession_number])
    resp = client.get(url)
    assert resp.status_code == 302
    messages = list(resp.wsgi_request._messages)
    assert (
        messages[0].message
        == f"You do not have permissions to access enclosure {animal_A.enclosure}"
    )

    counts = []
    for d in range(num_counts):
        counts.append(
            animal_count_factory(
                "BA",
                timezone.localtime() - dt.timedelta(days=d),
                comment=f"test count on day {d}",
            )
        )

    # test some counts w/ context
    client.force_login(user_base)
    url = reverse("animal_counts", args=[animal_A.accession_number])
    resp = client.get(url)
    assert resp.status_code == 200
    assert resp.context["animal"] == animal_A
    assert resp.context["enclosure"] == enclosure_base
    assert list(resp.context["animal_counts"]) == counts[:10]
    assert resp.context["page_range"][0] == 1
    assert resp.context["page_range"][-1] == 1 + 5
    # chart_data
    # chart_labels

    # test pagination
    url = "{}{}".format(
        reverse("animal_counts", args=[animal_A.accession_number]), f"?page={2}"
    )
    resp = client.get(url)
    assert resp.status_code == 200
    assert list(resp.context["animal_counts"]) == counts[10:20]
    assert resp.context["page_range"][0] == 1
    assert resp.context["page_range"][-1] == 2 + 5

def test_group_counts(
    client,
    group_B,
    group_B_count_datetime_factory,
    user_base,
    user_factory,
    enclosure_base,
):
    num_counts = 120

    # test perms
    rando_user = user_factory("rando")
    client.force_login(rando_user)
    url = reverse("group_counts", args=[group_B.accession_number])
    resp = client.get(url)
    assert resp.status_code == 302  # redirect to home
    messages = list(resp.wsgi_request._messages)
    assert (
        messages[0].message
        == f"You do not have permissions to access enclosure {group_B.enclosure}"
    )

    counts = []
    for d in range(num_counts):
        counts.append(
            group_B_count_datetime_factory(timezone.localtime() - dt.timedelta(days=d))
        )

    # test some counts w/ context
    client.force_login(user_base)
    url = reverse("group_counts", args=[group_B.accession_number])
    resp = client.get(url)
    assert resp.status_code == 200
    assert resp.context["group"] == group_B
    assert resp.context["enclosure"] == enclosure_base
    assert list(resp.context["counts"]) == counts[:10]
    assert resp.context["page_range"][0] == 1
    assert resp.context["page_range"][-1] == 1 + 5

    # test pagination
    url = "{}{}".format(
        reverse("group_counts", args=[group_B.accession_number]), f"?page={2}"
    )
    resp = client.get(url)
    assert resp.status_code == 200
    assert list(resp.context["counts"]) == counts[10:20]
    assert resp.context["page_range"][0] == 1
    assert resp.context["page_range"][-1] == 2 + 5

def test_species_counts(
    client, species_count_factory, species_base, user_base, user_factory, enclosure_base
):
    num_counts = 30

    # test perms
    rando_user = user_factory("rando")
    client.force_login(rando_user)
    url = reverse("species_counts", args=[species_base.slug, enclosure_base.slug])
    # should redirect to home
    resp = client.get(url)
    assert resp.status_code == 302
    messages = list(resp.wsgi_request._messages)
    assert (
        messages[0].message
        == f"You do not have permissions to access enclosure {enclosure_base}"
    )

    # create some counts
    counts = []
    for d in range(num_counts):
        counts.append(
            species_count_factory(
                50, datetimecounted=timezone.localtime() - dt.timedelta(days=d)
            )
        )

    # test some counts w/ context
    client.force_login(user_base)
    url = reverse("species_counts", args=[species_base.slug, enclosure_base.slug])
    resp = client.get(url)
    assert resp.status_code == 200
    assert resp.context["obj"] == species_base
    assert resp.context["enclosure"] == enclosure_base
    assert list(resp.context["counts"]) == counts[:10]
    assert resp.context["page_range"][0] == 1
    assert resp.context["page_range"][-1] == 3
    # "chart_data_line_total"
    # "chart_labels_line"
    # "chart_data_pie"
    # "chart_labels_pie"

def test_ingest_form(client, user_base, user_super):
    client.force_login(user_base)

    url = reverse("ingest_form")
    resp = client.get(url)
    assert resp.status_code == 302
    SimpleTestCase().assertRedirects(resp, "/accounts/login/", target_status_code=302)

    client.force_login(user_super)
    resp = client.get(url)
    assert resp.status_code == 200
    assert resp.context["req_cols"] == TRACKS_REQ_COLS

    # todo: test session data manipulation (changesets/upload_file)

    # TODO: POST

def test_confirm_upload():
    # /confirm_upload

    # changesets and upload_file in session

    # test GET

    # test POST (writes changes to db)
    pass

def test_export(client, user_base, enclosure_base, user_factory, caplog):

    # GET

    # user w/ no enclosures empty list of enclosures
    rando_user = user_factory("rando")
    client.force_login(rando_user)
    resp = client.get("/export/")
    assert resp.status_code == 200
    assert list(resp.context["form"].fields["selected_enclosures"].queryset) == list(
        Enclosure.objects.none()
    )

    # user_base has enclosure_base in list of options
    client.force_login(user_base)
    resp = client.get("/export/")
    assert resp.status_code == 200
    assert enclosure_base.name in resp.content.decode()
    assert (
        resp.context["form"].fields["selected_enclosures"].queryset[0] == enclosure_base
    )

    # POST
    # still with user_base logged in
    yesterday = dt.date.today() - dt.timedelta(days=1)

    caplog.clear()
    # w/ no counts
    resp = client.post(
        "/export/",
        {
            "start_date": yesterday.strftime("%m/%d/%Y"),
            "end_date": dt.date.today().strftime("%m/%d/%Y"),
            "selected_enclosures": enclosure_base.id,
        },
    )
    assert resp.status_code == 200
    record = caplog.records[0]

    assert record.message == "no data to export for enclosures"
    assert record.enclosures[0]["name"] == enclosure_base.name
    assert record.start_date == yesterday.strftime("%m/%d/%Y")
    assert record.end_date == dt.date.today().strftime("%m/%d/%Y")

    # create some counts
    # assert 201 status code
    # load in excel data, convert to dataframe and check the counts

def test_get_accessible_enclosures(
    user_base, enclosure_base, enclosure_factory, user_super
):
    enclosures = get_accessible_enclosures(user_base)
    assert list(enclosures) == [enclosure_base]

    # create a bunch of enclosures, but don't assign to role
    enc_set = {enclosure_base}
    for i in range(10):
        enc_set.add(enclosure_factory(str(i), role=None))

    # super user should get all the enclosures
    enclosures_super = get_accessible_enclosures(user_super)
    assert set(enclosures_super) == enc_set

    # regular user should still have the same single enclosure
    enclosures = get_accessible_enclosures(user_base)
    assert list(enclosures) == [enclosure_base]

def test_redirect_if_not_permitted(
    rf_get_factory, enclosure_factory, enclosure_base, user_super
):
    """uses our request factory to directly test `redirect_if_not_permitted`
    without actually making requests to any endpoints
    """

    request = rf_get_factory("/count/")
    forbidden_enc = enclosure_factory("forbidden_enc", role=None)

    # test if user doesn't have role that includes enclosure (True)
    assert redirect_if_not_permitted(request, forbidden_enc)

    # test if user has role that includes enclosure (False)
    assert not redirect_if_not_permitted(request, enclosure_base)

    # test if user is superuser (False)
    request.user = user_super
    assert not redirect_if_not_permitted(request, enclosure_base)
    assert not redirect_if_not_permitted(request, forbidden_enc)

def test_selected_role(rf_get_factory, role_base):

    # test view all query param returns None and clears session
    request = rf_get_factory("/home/?view_all=true")
    request.session["selected_role"] = "role_base"
    selected_role = get_selected_role(request)
    assert selected_role is None
    assert request.session.get("selected_role") is None

    # test role in session & no query params returns session role
    request = rf_get_factory("/home/")
    request.session["selected_role"] = "role_base"
    selected_role = get_selected_role(request)
    assert selected_role == role_base
    assert request.session.get("selected_role") == "role_base"

    # test role query param returns role and sets session
    request = rf_get_factory("/home/?role=role_base")
    selected_role = get_selected_role(request)
    assert selected_role == role_base
    assert request.session.get("selected_role") == "role_base"

    # test role query param and diff session query param returns query param role and
    # sets session
    request = rf_get_factory("/home/?role=role_base")
    request.session["selected_role"] = "diff_role"
    selected_role = get_selected_role(request)
    assert selected_role == role_base
    assert request.session.get("selected_role") == "role_base"

    # test absence of all of these return None
    request = rf_get_factory("/home/")
    selected_role = get_selected_role(request)
    assert selected_role is None

def test_enclosure_counts_to_dict(create_many_counts, django_assert_num_queries):
    """
    Tests the dictionary creation from list of counts
    Tests the structure of the dict
    """
    num_enc = 7
    num_anim = 4
    num_species = num_groups = 5

    _, _, _, enc_list = create_many_counts(
        num_enc=num_enc,
        num_anim=num_anim,
        num_species=num_species,
    )

    # create a query similar to how we build it in the view
    encl_q = Enclosure.objects.filter(name__in=enc_list).prefetch_related(
        "animals", "groups"
    )
    with django_assert_num_queries(5):
        # 3 for enclosures, groups, animals (w/ prefetch related)
        # 2 for animal_counts and group_counts

        animal_counts, group_counts = Enclosure.all_counts(encl_q)
        all_counts_dict = enclosure_counts_to_dict(encl_q, animal_counts, group_counts)

    # assert dict keys present in original query
    assert list(encl_q) == list(all_counts_dict.keys())

    # assert dict structure
    for enc in enc_list:
        enc_dict = all_counts_dict[enc]

        keys = {
            "animal_count_total",
            "animal_conditions",
            "group_counts",
            "group_count_total",
            "total_animals",
            "total_groups",
        }
        assert set(enc_dict.keys()) == keys

        assert enc_dict["animal_count_total"] == num_anim
        assert enc_dict["total_animals"] == num_anim

        # (1+3) Seen/BAR from input to group_count_factory
        assert enc_dict["group_count_total"] == num_groups * (1 + 3)

        # pop total = 30
        assert enc_dict["total_groups"] == num_groups * 30

        assert list(enc_dict["animal_conditions"].keys()) == [
            c[1] for c in AnimalCount.CONDITIONS
        ]
        assert all(
            [
                all([isinstance(c, AnimalCount) for c in c_l])
                for c_l in enc_dict["animal_conditions"].values()
            ]
        )

        assert list(enc_dict["group_counts"].keys()) == ["Seen", "BAR", "Needs Attn"]
        # These values come from the factory inputs to create group_counts
        assert enc_dict["group_counts"]["Seen"] == num_groups
        assert enc_dict["group_counts"]["BAR"] == num_groups * 3
        assert enc_dict["group_counts"]["Needs Attn"] == 0


from enum import Enum
import numpy as np
import os

class Action(Enum):
    UP = (0, -1)
    DOWN = (0, 1)
    LEFT = (-1, 0)
    RIGHT = (1, 0)

class Color:
    BACKGROUND = (0, 0, 0)
    BORDER = (100, 100, 100)
    END_POSITION_GOOD = (0, 255, 0)
    END_POSITION_BAD = (255, 0, 0)
    END_POSITION_TEXT = (0, 0, 0)
    EMPTY_FIELD = (255, 255, 255)
    WALL = (0, 0, 0)
    AGENT = (0, 0, 255)  # not used currently
    BEST_ARROW_OUTLINE = BORDER
    HOVER_COLOR = (0, 0, 255)
    INFO_TEXT = (255, 255, 255)
    TRANSPARENT = (0, 0, 0, 0)

    @staticmethod
    def for_q_value(q_value: float, max_q_value: float):

        # No arrow will be drawn if None is returned
        if q_value == 0:
            return None

        # tanh mapping squeezes the values between 0 and 1.
        # A multiplication factor is calculated to stretch the Q-values with respect to
        # the highest absolute Q-value. This improves visualization
        color_value = int(abs(np.tanh(q_value * (np.arctanh(0.9999) / max_q_value))) * 255)

        assert 0 <= color_value <= 255

        if q_value > 0:
            # green channel
            color = (0, 255, 0, color_value)
        else:
            # red channel
            color = (255, 0, 0, color_value)
        return color

class FieldType:
    EMPTY = 0
    END_POS = 1
    BLOCKED = 2

class SaveSettings:
    ENABLED = True
    INTERVAL = 100
    SAVE_PATH = "checkpoints"

class RenderSettings:
    TIME_BETWEEN_FRAMES = 300  # ms
    ENABLED = True
    INTERVAL = 50  # epochs
    GAME_PADDING = 30  # pixels
    PIXEL_PER_FIELD = 100  # pixels
    FONT_NAME = "assets/fonts/arial.ttf"
    FONT_SIZE = 30  # pt
    INFO_FONT_SIZE = 20
    ARROW_WIDTH = 0.2  # percentage of field pixels
    AGENT_SIZE = int(0.8 * PIXEL_PER_FIELD)  # pixels
    AGENT_IMAGE_PATH_GOOD = os.path.join("assets", "robot.png")
    AGENT_IMAGE_PATH_BAD = os.path.join("assets", "robot_broken.png")
    ARROW_OUTLINE_WIDTH = int(PIXEL_PER_FIELD / 20)  # pixels
    UPDATE_FREQ_TITLE = 10  # epochs
    INFO_HEIGHT = 100


import pandas as pd
import numpy as np
import os
import glob

from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import f_classif, chi2
from sklearn.model_selection import RepeatedStratifiedKFold
from sklearn.neural_network import MLPClassifier
from sklearn.metrics import confusion_matrix

from copy import deepcopy

script_dir = os.path.dirname(__file__)

def get_data(files):
    data_matrix = np.loadtxt(files[0], dtype='i', delimiter='\t')
    data_matrix = data_matrix.T

    last_col = [0] * len(data_matrix)
    data_matrix = np.column_stack((data_matrix, last_col))

    for x in range(len(files) - 1):
        temp_matrix = np.loadtxt(files[x + 1], dtype='i', delimiter='\t')
        temp_matrix = temp_matrix.T
        last_col = [x+1] * len(temp_matrix)
        temp_matrix = np.column_stack((temp_matrix, last_col))
        data_matrix = np.concatenate((data_matrix, temp_matrix), axis=0)

    X = data_matrix[:, :-1]
    Y = data_matrix[:, -1]
    return X, Y

def classification(X, Y, activation_value, momentum_value, top_rank, fvalue_selector, layer_size, best_score, list_score):

    # Utworzenie walidacji krzyżowej, podział na 2 grupy, 5 powtórzeń
    rkf = RepeatedStratifiedKFold(n_splits=2, n_repeats=5)

    # Utworzenie modelu o wcześniej zdefiniowanych parametrach
    mlp = MLPClassifier(hidden_layer_sizes=layer_size,
                        activation=activation_value, max_iter=1000, momentum=momentum_value)

    first_time = True
    # Pętla dla 5 krotnej walidacji krzyżowej
    for train, test in rkf.split(X, Y):
        x_train, x_test = X[train], X[test]
        y_train, y_test = Y[train], Y[test]

        # Dopasowanie x_train do y_train
        mlp.fit(x_train, y_train)

        # Przypisanie wyniku na podstawie nowych zbiorów
        score = mlp.score(x_test, y_test)

        # Przewidywanie za pomocą wcześniej utworzonego mlp przy użyciu zbioru x_test
        predict = mlp.predict(x_test)

        # Macierz pomyłek
        confusion_matrix_value = confusion_matrix(y_test, predict)

        print(score)

        # Utworzenie nowego rekordu z danymi odnośnie wykonanego badania
        if first_time:
            temp_score = score
            temp_confusion_matrix = confusion_matrix_value
            first_time = False
        else:
            temp_score = temp_score + score
            temp_confusion_matrix = temp_confusion_matrix + confusion_matrix_value

    current_score = [temp_score/10, activation_value, momentum_value,
                     layer_size, len(top_rank), (temp_confusion_matrix/10).astype(int)]

    print(f"----\navg: {current_score[0]}\n-----")
    # Ustawienie aktualnie najlepszego wyniku
    if best_score[0] < current_score[0]:
        best_score = deepcopy(current_score)

    list_score.append(current_score)
    return best_score

def main():
    # 1. Wyznaczenie rankingu cech
    files = glob.glob('sets/*.txt')
    X, Y = get_data(files)

    fvalue_selector = SelectKBest(f_classif)
    fvalue_selector.fit(X, Y)

    rank = fvalue_selector.scores_
    top_rank = []
    indexes = rank.argsort()[-30:][::-1]

    for index in indexes:
        top_rank.append(rank[index])

    # 2. Implementacja środowiska eksperymentowania

    # Format zapisu wyniku - [wynik, typ funkcji aktywacji, wartość momentum, rozmiar warstwy, macierz pomyłek]
    best_score = [0, '', 0, 0, 0, np.ndarray]
    # Tablica przechowująca wyniki wszystkich badań w formacie takim samym jak zmienna "best_score"
    list_score = []

    # Zdefiniowane trzech iczb neuronów w warstwie ukrytej
    layers = [100, 200, 300]
    # layers = [100]

    # Typy funkcji aktywacji
    activation_values = ['relu', 'logistic']
    # activation_values = ['relu']

    # Wartości momentum (domyślna wartość wynosi 0)
    momentum_values = [0, 0.9]
    # momentum_values = [0]

    # Całkowita liczba wszystkich badań
    total_examinations = len(momentum_values) * \
        len(activation_values) * len(layers) * 15

    # Indeks aktualnego badania
    current_examination = 1

    # Badania przeprowadzane przy użyciu wszystkich możliwych wcześniej zdefiniowanych kombinacji paramterów
    for layer in layers:
        for activation_value in activation_values:
            for momentum_value in momentum_values:
                for feature_number in range(15, 20):
                    to_train = indexes[:feature_number]

                    print("---------------------------------")
                    print(
                        f"layers {layer}, activation value: {activation_value}, momentum_value {momentum_value}, feature_number: {feature_number}")
                    print(f'{current_examination}/{total_examinations} - start')

                    best_score = classification(X=X[:, to_train], Y=Y, activation_value=activation_value,
                                                momentum_value=momentum_value, top_rank=top_rank[
                                                    :feature_number - 1],
                                                fvalue_selector=fvalue_selector,
                                                layer_size=layer, best_score=best_score, list_score=list_score)

                    print(f'{current_examination}/{total_examinations} - end')
                    current_examination += 1

    dflist = pd.DataFrame(list_score)
    dflist.to_csv('wyniki.txt', encoding='utf-8', index=False)

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
import django.db.models.deletion
import wagtail.wagtailcore.fields

def create_homepage(apps, schema_editor):
    # Get models
    ContentType = apps.get_model('contenttypes.ContentType')
    Page = apps.get_model('wagtailcore.Page')
    Site = apps.get_model('wagtailcore.Site')
    HomePage = apps.get_model('core.HomePage')

    # Delete the default homepage
    Page.objects.get(id=2).delete()

    # Create content type for homepage model
    homepage_content_type, created = ContentType.objects.get_or_create(
        model='homepage', app_label='core', defaults={'name': 'Homepage'})

    # Create a new homepage
    homepage = HomePage.objects.create(
        title="Homepage",
        slug='home',
        content_type=homepage_content_type,
        path='00010001',
        depth=2,
        numchild=0,
        url_path='/home/',
    )

    # Create a site with the new homepage set as the root
    Site.objects.create(
        hostname='localhost', root_page=homepage, is_default_site=True)

# Functions from the following migrations need manual copying.
# Move them and any dependencies into this file, then update the
# RunPython operations to refer to the local versions:
# core.migrations.0001_squashed_0005_auto_20150207_2353

class Migration(migrations.Migration):

    dependencies = [
        ('wagtailimages', '0005_make_filter_spec_unique'),
        ('wagtailcore', '__latest__'),
        ('wagtailforms', '0001_initial'),
        ('wagtailcore', '0010_change_page_owner_to_null_on_delete'),
        ('wagtailredirects', '0001_initial'),
        ('wagtailsearch', '0001_initial'),
    ]

    operations = [
        migrations.CreateModel(
            name='HomePage',
            fields=[
                ('page_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='wagtailcore.Page')),
            ],
            options={
                'abstract': False,
            },
            bases=('wagtailcore.page',),
        ),
        migrations.RunPython(
            code=create_homepage,
            reverse_code=None,
            atomic=True,
        ),
        migrations.AddField(
            model_name='homepage',
            name='body',
            field=wagtail.wagtailcore.fields.RichTextField(null=True, blank=True),
            preserve_default=True,
        ),
        migrations.AddField(
            model_name='homepage',
            name='cover_image',
            field=models.ForeignKey(related_name='+', on_delete=django.db.models.deletion.SET_NULL, blank=True, to='wagtailimages.Image', null=True),
            preserve_default=True,
        ),
        migrations.CreateModel(
            name='EBook',
            fields=[
                ('homepage_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='core.HomePage')),
                ('author', models.CharField(max_length=64, null=True, verbose_name='author / director', blank=True)),
                ('country', models.CharField(max_length=64, null=True, verbose_name='country of origin', blank=True)),
                ('resource_link', models.URLField(default='', help_text='Find a book or movie file available on the intranet and copy the link here, e.g. http://localserver/path/to/book.pdf', verbose_name='resource link')),
                ('thumbnail', models.ForeignKey(related_name='+', on_delete=django.db.models.deletion.SET_NULL, blank=True, to='wagtailimages.Image', null=True)),
                ('year', models.PositiveSmallIntegerField(null=True, verbose_name='year of publication', blank=True)),
            ],
            options={
                'abstract': False,
            },
            bases=('core.homepage',),
        ),
        migrations.CreateModel(
            name='Movie',
            fields=[
                ('homepage_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='core.HomePage')),
                ('year', models.PositiveSmallIntegerField(null=True, verbose_name='year of publication', blank=True)),
                ('country', models.CharField(max_length=64, null=True, verbose_name='country of origin', blank=True)),
                ('author', models.CharField(max_length=64, null=True, verbose_name='author / director', blank=True)),
                ('resource_link', models.URLField(default='http://dr.dk', help_text='Find a book or movie file available on the intranet and copy the link here, e.g. http://localserver/path/to/book.pdf', verbose_name='resource link')),
                ('thumbnail', models.ForeignKey(related_name='+', on_delete=django.db.models.deletion.SET_NULL, blank=True, to='wagtailimages.Image', null=True)),
            ],
            options={
                'abstract': False,
            },
            bases=('core.homepage',),
        ),
        migrations.AlterField(
            model_name='homepage',
            name='body',
            field=wagtail.wagtailcore.fields.RichTextField(null=True, verbose_name='Body / main text on page', blank=True),
            preserve_default=True,
        ),
        migrations.CreateModel(
            name='Collection',
            fields=[
                ('homepage_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='core.HomePage')),
                ('icon', models.ImageField(help_text='Cheerful icon to display on buttons next to the title', upload_to=b'', null=True, verbose_name='icon', blank=True)),
                ('license', models.CharField(max_length=64, null=True, verbose_name='license', blank=True)),
            ],
            options={
                'abstract': False,
            },
            bases=('core.homepage',),
        ),
        migrations.CreateModel(
            name='ExternalCollection',
            fields=[
                ('collection_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='core.Collection')),
                ('retrieved', models.DateField(help_text='When this collection was copied from the internet', null=True, verbose_name='Retrieval date', blank=True)),
            ],
            options={
                'abstract': False,
            },
            bases=('core.collection',),
        ),
        migrations.AddField(
            model_name='collection',
            name='button_caption',
            field=models.CharField(max_length=64, null=True, verbose_name='button caption', blank=True),
            preserve_default=True,
        ),
        migrations.AlterField(
            model_name='collection',
            name='icon',
            field=models.ForeignKey(related_name='+', on_delete=django.db.models.deletion.SET_NULL, blank=True, to='wagtailimages.Image', help_text='Cheerful icon to display on buttons next to the title', null=True, verbose_name='icon'),
            preserve_default=True,
        ),
    ]


from collections import namedtuple

Point = namedtuple("Point", "x y height")

class HeightMap:
    def __init__(self, height_map):
        self.height_map = height_map
        self.y_count = len(self.height_map)
        self.x_count = len(self.height_map[0])

    def is_in_bounds(self, x, y):
        return x >= 0 and x < self.x_count and y >= 0 and y < self.y_count

    def get_height(self, x, y):
        if not self.is_in_bounds(x, y):
            return None
        return self.height_map[y][x]

    def calc_neighbor_points(self, x, y):
        neighbors = []

        new_points = [(x, y - 1), (x, y + 1), (x + 1, y), (x - 1, y)]
        for new_point in new_points:
            new_x, new_y = new_point
            height = self.get_height(new_x, new_y)
            if height != None:
                neighbors.append(Point(new_x, new_y, height))
        return neighbors

    def calc_neighbor_heights(self, x, y):
        return [point.height for point in self.calc_neighbor_points(x, y)]

    def is_lowpoint(self, point):
        return self.is_lowpoint_xy(point.x, point.y)

    def is_lowpoint_xy(self, x, y):
        height = self.get_height(x, y)
        if height is None:
            return None

        larger_neighbors = filter(
            lambda v: v <= height, self.calc_neighbor_heights(x, y)
        )
        return len(list(larger_neighbors)) == 0

    def calc_lowpoints(self):
        return list(filter(lambda point: self.is_lowpoint(point), self))

    def calc_total_risk(self):
        return sum([point.height + 1 for point in self.calc_lowpoints()])

    def calc_basin(self, lowpoint):
        basin = set()
        to_expand = [lowpoint]
        while len(to_expand) > 0:
            point = to_expand.pop()
            basin.add(point)
            neighbors = self.calc_neighbor_points(point.x, point.y)
            neighbors = filter(lambda point: point.height < 9, neighbors)
            to_expand.extend(set(neighbors).difference(basin))
        return basin

    def calc_basins(self):
        return [self.calc_basin(lowpoint) for lowpoint in self.calc_lowpoints()]

    def __iter__(self):
        for x in range(self.x_count):
            for y in range(self.y_count):
                yield Point(x, y, self.get_height(x, y))

    def __str__(self):
        rows_for_printing = []
        for row in self.height_map:
            rows_for_printing.append("".join([str(h) for h in row]))
        return "\n".join(rows_for_printing)

def read_height_map(filename="test_advent09.txt"):
    """return test data if no file is given"""
    height_map = []
    with open(filename) as file:
        for line in file:
            height_map.append([int(h) for h in line.strip()])
    return HeightMap(height_map)

if __name__ == "__main__":
    height_map = read_height_map("advent09.txt")
    total_risk = height_map.calc_total_risk()
    print(f"part1: total risk level is {total_risk}")

    m = read_height_map("advent09.txt")
    basin_sizes = sorted([len(basin) for basin in m.calc_basins()], reverse=True)
    multi = basin_sizes[0] * basin_sizes[1] * basin_sizes[2]
    print(
        f"part2: the largest basins are of size {basin_sizes[0:3]}, multiplied that's {multi}"
    )


import pathlib
import numpy as np
import torch
import torch.utils.data as data
import torchvision.transforms as transforms
import PIL.Image as Image

class MRAProjected(data.Dataset):
    """
    Class defined to handle 3D MRA data reprojected as
    2D axial and saggital slices

    MIP images should already be created and in folder
    split into train and val with txt file

    raw_dir
        - /case_dir
            - /pre/TOF.nii.gz

    img_dir
        -/raw
            - /case_dir_axial*.png
            - /case_dir_sag*.png
        -/seg
            - /case_dir_axial*_label.png
            - /case_dir_sag*_label.png
    """

    def __init__(self, cfg, mode="train"):
        super().__init__()
        self.cfg = cfg
        self.orig_dir = pathlib.Path(cfg.ORIGINAL_PATH)
        self.img_dir = pathlib.Path(cfg.MIP_PATH)
        # Dictionary storing paths of all axial and sagittal images to use for dataset
        self.img_list = {"axial": [], "sag": []}
        self.transform = transforms.Compose([
                            transforms.Resize((cfg.MODEL.IMAGE_SIZE, cfg.MODEL.IMAGE_SIZE)),
                            transforms.ToTensor(),
                            ])
        self.mode = mode

        # Set the split file
        if self.mode == "train":
            self.split_file = pathlib.Path(cfg.TRAIN_SPLIT_FILE)
        elif self.mode == "val":
            self.split_file = pathlib.Path(cfg.VAL_SPLIT_FILE)
        elif self.mode == "":
            self.split_file = None
        else:
            raise Exception("Mode can only be train or val or empty")

        # Read paths of images into img_list from split file
        if self.split_file:
            with open(str(self.split_file), "r") as sf:
                case_nums = sf.readlines()
            self.get_img_list(case_nums)
            self.raw_case_dirs = [self.orig_dir.joinpath(c.strip() + "/") for c in case_nums]

        # If no split file, just use all image files
        else:
            self.get_img_list()
            self.raw_case_dirs = [d for d in sorted(self.orig_dir.glob("100*/"))]

    def __len__(self):
        return len(self.img_list["axial"])

    def __getitem__(self, index):
        axial_path = self.img_list["axial"][index]
        sag_path = self.img_list["sag"][index]

        axial_img = self.transform(Image.open(str(axial_path)))

        sag_img = self.transform(Image.open(str(sag_path)))

        axial_label_path = str(axial_path)[:-4].replace("raw", "seg") + "_label.png"
        sag_label_path = str(sag_path)[:-4].replace("raw", "seg") + "_label.png"

        axial_label = self.transform(Image.open(str(axial_label_path)))
        axial_label[axial_label > 0.005] = 0.
        axial_label[axial_label > 0.] = 1.
        if len(torch.unique(axial_label)) > 1:
            axial_aneurysm = True
        else:
            axial_aneurysm = False

        sag_label = self.transform(Image.open(str(sag_label_path)))
        sag_label[sag_label > 0.005] = 0.
        sag_label[sag_label > 0.] = 1.
        if len(torch.unique(sag_label)) > 1:
            sag_aneurysm = True
        else:
            sag_aneurysm = False

        img_name = str(axial_path).split("/")[-1]
        case_num = img_name[0:5]

        sample = {"top_image": axial_img, "bottom_image": sag_img, "top_label": axial_label, "bottom_label": sag_label,
                  "case_num": case_num, "aneurysm_top": axial_aneurysm, "aneurysm_bottom": sag_aneurysm}

        return sample

    def get_img_list(self, case_num=None):
        """
        Get paths of all images given a case number.
        If no case number, return all image paths in img_dirs folder
        :param case_num:
        """
        if not case_num:
            axial = list(sorted(self.img_dir.glob("raw/*_axial*.png")))
            sag = list(sorted(self.img_dir.glob("raw/*_sag*.png")))

            assert len(axial) == len(sag), "Number of axial and saggital images should be the same"

            self.img_list["axial"].append(axial)
            self.img_list["sag"].append(sag)

        elif isinstance(case_num, str):
            ax_name = case_num + "_axial*.png"
            sag_name = case_num + "_sag*.png"
            axial = list(sorted(self.img_dir.glob("raw/" + ax_name)))
            sag = list(sorted(self.img_dir.glob("raw/" + sag_name)))

            assert len(axial) == len(sag), "Number of axial and saggital images are not the same for case {0}".format(case_num)

            ax_list = self.img_list.get("axial", [])
            sag_list = self.img_list.get("sag", [])

            ax_list.append([i for i in axial])
            sag_list.append([i for i in sag])

            self.img_list["axial"] = ax_list
            self.img_list["sag"] = sag_list

        elif isinstance(case_num, list):
            for num in case_num:
                num = num.strip()
                self.get_img_list(num)

            self.img_list["axial"] = [item for sublist in self.img_list["axial"] for item in sublist]
            self.img_list["sag"] = [item for sublist in self.img_list["sag"] for item in sublist]

        else:
            raise Exception("Unrecognized format for case_num input")


from encoderMod import encoderMod
from decoderMod import decoderMod
from dataprocess import *
from keras.layers import Input, Dense
from keras.models import Model
from keras.optimizers import Adadelta, RMSprop,SGD,Adam
from keras.callbacks import CSVLogger
from keras import regularizers

def autoencoder():
	encoder = encoderMod()
	encoder.summary()
	decoder = decoderMod()
	decoder.summary()
	input = Input(shape=(128*128,))
	#print(encoder(input)[3])
	outputs = decoder(encoder(input))
	autoencoder = Model(input, outputs)
	return autoencoder

def custom_loss(input, output):
	alpha = 0.5
	reconstruction_loss = mse(input, output)
#	reconstruction_loss *= 128*128
	reconstruction_loss *= alpha
	relation_loss = mse(K.dot(K.transpose(input), input),K.dot(K.transpose(output), output))
#	relation_loss *= 128*128
	relation_loss *= (1-alpha)
	auto_loss = reconstruction_loss + relation_loss
	return reconstruction_loss

def compiler(autoencoder):
	autoencoder.compile(optimizer=Adam(lr=0.01, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False), loss='mse', metrics=['accuracy'])
	return autoencoder

def Fit(autoencoder, x_trian, x_test, epochs, batch_size):
	history = autoencoder.fit(x_train, x_train,
                epochs=epochs,
                batch_size=batch_size,
                shuffle=True,
                validation_data=(x_test, x_test))
	return history

def saveWeight(model):
	model.save_weights("deepconvo1reg.h5")

if __name__ == "__main__":
	autoencoder = autoencoder()
	autoencoder = compiler(autoencoder)
	data = read_csv("/home/tonystark/Downloads/gh/abc2classwoenrolid.csv")
	labels, result_train = process_data(data, 10000)
	x_train = reshape(result_train)
	x_train, x_test, y_train, y_test = split_train_test(x_train, labels)
	x_train, x_test = datapreprocess(x_train, x_test)
	history = Fit(autoencoder, x_train, x_test, 50, 8)
	saveWeight(autoencoder)

	import matplotlib
	matplotlib.use('agg')
	from matplotlib import pyplot as plt
	plt.switch_backend('agg')
	# summarize history for accuracy
	plt.plot(history.history['acc'])
	plt.plot(history.history['val_acc'])
	plt.title('model accuracy')
	plt.ylabel('accuracy')
	plt.xlabel('epoch')
	plt.legend(['train', 'test'], loc='upper left')
	plt.savefig('ddreg')
	# summarize history for loss
	plt.plot(history.history['loss'])
	plt.plot(history.history['val_loss'])
	plt.title('model loss')
	plt.ylabel('loss')
	plt.xlabel('epoch')
	plt.legend(['train', 'test'], loc='upper left')
	plt.savefig('eereg')

	loss_history = history.history["loss"]
	import numpy
	numpy_loss_history = numpy.array(loss_history)
	numpy.savetxt("loss_historyreg.txt", numpy_loss_history, delimiter=",")
	loss_accuracy = history.history["acc"]
	numpy_loss_accuracy = numpy.array(loss_accuracy)
	numpy.savetxt("loss_accuracyreg.txt", numpy_loss_accuracy, delimiter=",")
	loss_val_history = history.history["val_loss"]
	import numpy
	numpy_val_loss_history = numpy.array(loss_val_history)
	numpy.savetxt("val_loss_historyreg.txt", numpy_val_loss_history, delimiter=",")
	val_loss_accuracy = history.history["val_acc"]
	numpy_val_loss_accuracy = numpy.array(val_loss_accuracy)
	numpy.savetxt("val_loss_accuracyreg.txt", numpy_val_loss_accuracy, delimiter=",")


import logging
logger = logging.getLogger(__name__)
import json
import time
from threading import Thread
import random

class RightMoveAreaSaleHistory(Thread):
    """
    This class is used to extract previous sale results for a given postcode. This data
    can be used to match the individual property sale info, providing the address.
    """

    def __init__(self, web_client, db, running, sleep_time):
        super().__init__()
        # Web Client Session
        self.web_client = web_client
        # Datebase Connection
        self.db = db
        # Define the exit event placeholder
        self.running = running
        # Define a sleep time placeholder
        self.sleep_time = sleep_time

    def get_area_sale_history_page(self, postcode_url, page_number=None):
        """
        Method gets a page of the previous house sales for a given postcodes, indiviual properties
        contain the full address.
        """
        if page_number:
            postcode_url = postcode_url + f"?page={page_number}"

        logger.info("getting area sale history page")
        # Get the property page.
        page = self.web_client.get_page_for_url(postcode_url)
        # Check for a value of None or a non 200 status code.
        if not page or page.status_code != 200:
            # Continue for the next url
            return
        logger.info("got area sale history page")
        return page

    def get_area_sale_history_properties(self, json):
        """
        Method returns a list of properties and their previous sale dates and prices.
        Example: "properties":[{"address":"14, Hillwood Grove, Hutton, Brentwood, Essex CM13 2PD","propertyType":"Detached","bedrooms":5,"images":{"imageUrl":"https://media.rightmove.co.uk/dir/147k/146846/48181536/146846_6187143_IMG_00_0000_max_135x100.jpg","count":23},"hasFloorPlan":true,"transactions":[{"displayPrice":"£1,800,000","dateSold":"12 Mar 2020","tenure":"Freehold","newBuild":false}],"location":{"lat":51.62568,"lng":0.33711},"detailUrl":"https://www.rightmove.co.uk/house-prices/detailMatching.html?prop=48181536&sale=11345743&country=england"}]
        """
        try:
            properties = json["properties"]
        except Exception as e:
            logger.exception(str(e))
            return
        else:
            # Return the results.
            logger.info("extracted properties successfully")
            return properties

    def get_area_sale_history_result_count(self, json):
        """
        Method extracts the result count value from the json input parameter.
        """
        try:
            result_count = int(json["resultCount"])
        except Exception as e:
            logger.exception(str(e))
            return
        else:
            logger.info(f"got {result_count} sales results")
            # Return the results.
            return result_count

    def get_area_sale_history_json(self, page):
        """
        Method extracts the sales history json from the given page.
        """
        # Break out the json data from the page and load into a python dict.
        try:
            # Get the page text value from the requests object
            page_text = page.text
            js = json.loads(page_text.split("</script><script>window.__PRELOADED_STATE__ = ")[1].split("</script>")[0])
        except Exception as e:
            logger.exception(str(e))
            return
        else:
            logger.info("extracted sale history json successfully")
            # Return the results.
            return js["results"]

    def get_sale_history_for_postcode(self, postcode_url):
        """
        This method handles the process of extracting all area sales history for a given postcode.
        A page will yield a maximum of 25 results, therefore it maybe necessary to request additional pages,
        depending on the number of results.
        """
        # Create an area sale history placeholder.
        area_sale_history = list()

        # Page number
        page_number = 1

        # Result Count
        result_count = None

        # WHile Loop
        while True:

            # Get the Page of results
            page = self.get_area_sale_history_page(postcode_url, page_number=page_number)

            # Check we have a valid object.
            if not page:
                # break out of the while loop.
                break

            # Extract the json from the page.
            json = self.get_area_sale_history_json(page)

            # If the result count is None then we want to break out the results from the page.
            if not result_count:
                # Update the result_count variable.
                result_count = self.get_area_sale_history_result_count(json)

            # Get the sales history from the first page.
            sales_history = self.get_area_sale_history_properties(json)

            # Add the sales history to the area_sale_history list.
            area_sale_history.extend(sales_history)

            # Check if the number of results in the area_sale_history list is equal or greater
            # than the result_count variable to indicate we have retrived all results.
            if len(area_sale_history) >= result_count:
                # Break out of the while loop.
                break

            # To get more results we increase the page number variable.
            page_number += 1

            # Sleep for a random period.
            time.sleep(random.randint(1, 5))

        # Return the area_sale_history list
        return area_sale_history

        # # Get the first page for a given postcode.
        # first_page = self.get_area_sale_history_page(postcode_url)
        # if not first_page:
        #     return area_sale_history

        # # Extract the Json from the page.
        # json = self.get_area_sale_history_json(first_page)
        # # Get the Number of results found.
        # result_count = self.get_area_sale_history_result_count(json)
        # logger.info("extracted result count successfully")
        # Get the sales history from the first page.
        # sales_history = self.get_area_sale_history_properties(json)
        # Add the sales history to the area_sale_history list.
        # area_sale_history.extend(sales_history)

        # # Check if the length of the area_sale_history list is equal to the result count.
        # while len(area_sale_history) < result_count:
        #     page_number = 2
        #     next_page = self.get_area_sale_history_page(postcode_url, page_number=page_number)
        #     if not next_page:
        #         continue

        #     # Extract the Json from the page.
        #     json = self.get_area_sale_history_json(next_page)
        #     # Get the sales history from the first page.
        #     sales_history = self.get_area_sale_history_properties(json)
        #     # Add the sales history to the area_sale_history list.
        #     area_sale_history.extend(sales_history)
        #     # Increment the page counter
        #     page_number += 1

        # # Return the area_sale_history list
        # return area_sale_history

    def get_address_for_property(self, property_sale_history, area_sales_history):
        # Set a place holder for address.
        address = None
        # Iterate through the sales history dates and prices for the property.
        for i in range(len(property_sale_history)):
            if address:
                break
            try:
                # Break out a sale_date and sale_price object.
                psh = property_sale_history[i]
                # Break out the sale date.
                psh_date = psh["sale_date"]
                # Break out the sale price.
                psh_price = str(psh["sale_price"]).replace("£", "").replace(",", "") # "£530,000" becomes "530000"
                logger.info(f"property sale date {psh_date}, sale price {psh_price}")
                # Iterate through the area sale history object.
                for j in range(len(area_sales_history)):
                    # Break out the transactions.
                    transactions = area_sales_history[j]["transactions"]
                    # Iterate through the transactions.
                    for t in transactions:
                        ash_price = str(t["displayPrice"]).replace("£", "").replace(",", "") # "£530,000" becomes "530000"
                        ash_date = str(t["dateSold"]).split()[-1] # "8 Nov 2019" becomes 2019
                        # logger.info(f"area sale date {ash_date}, sale price {ash_price}")
                        # Compare the date and price values.
                        if (psh_date == ash_date and psh_price == ash_price):
                            # Extract the Address value and update
                            address = area_sales_history[j]["address"]
                            logger.info(f"Located address - {address}")
                            # Break out of the loop.
                            break
                    # Check if we have a value other than None for address.
                    if address:
                        break

            except Exception as e:
                logger.exception(str(e))
                continue
        # Return the Value of address.
        return address

    def process_property_area_sale_histories(self, properties):
        # Check if we have some valid properties in the list or return.
        if not properties:
            return

        # Create a list to hold the property Id and real address value.
        property_real_address_data = list()

        # Iterate through the properties.
        for property in properties:

            try:

                # Get the sale history for the given property id from the database.
                property_sale_history = self.db.get_sale_history_for_property_id(property["id"])

                # Check for a None value.
                if not property_sale_history:
                    continue

                # Get the area sale history for the given sold nearby url which shows all logged sales for
                # the same postcode.
                area_sales_history = self.get_sale_history_for_postcode(property["rm_sold_nearby_url"])

                # Try and get an address for the property
                address = self.get_address_for_property(property_sale_history, area_sales_history)

                # Check for a None value.
                if not address:
                    # We need to update the database to set "rm_real_address_attempted" to true so that we dont attempt this
                    # property search again.
                    self.db.update_real_address_attempted([(True, property["id"])])
                    continue

                # Create a tuple of the real address value and the rm_real_address_attempted (True) and property id, and add to the
                # property_real_address_data list
                property_real_address_data.append((address, True, property["id"]))

            except Exception as e:
                logger.exception(str(e))

        # Check that we have some entries to update.
        if property_real_address_data:

            # Update the datebase with the real address for the given property id.
            properties_updated = self.db.update_real_address_value(property_real_address_data)

            # Print the number of rows updated.
            logger.info(f"Properties real address updated {str(properties_updated)}")

    def run(self):

        # Forever Loop
        while not self.running.is_set():
            try:
                # Get properties from the database. This returns a list of property
                # id, rm_id, rm_postcode, rm_sold_nearby_url values for properties which have some sale history.
                properties = self.db.get_properties_with_no_real_address_and_sale_history_attempted()

                # Process Properties and area sale histories
                self.process_property_area_sale_histories(properties)

            except Exception as e:
                logger.exception(str(e))

            finally:
                logger.info("Sleeping")
                time.sleep(self.sleep_time)

        logger.info("Exited while loop gracefully")