"""Test .NET autoapi domain"""

from unittest import mock
from unittest.mock import patch

from autoapi.mappers import dotnet

class MockConfig:
    def __getattr__(self, key):
        attrs = {
            "autoapi_dirs": ["/tmp/autoapi/tmp"],
            "autoapi_root": "/tmp/autoapi/root",
        }
        return attrs.get(key, None)

class MockApplication:
    config = MockConfig()

    def warn(self, *args, **kwargs):
        pass

class TestDotNetSphinxMapper:
    def test_create_class(self):
        """Test .NET class instance creation helper"""
        dom = dotnet.DotNetSphinxMapper(MockApplication())

        def _create_class(data):
            return list(dom.create_class(data))[0]

        cls = _create_class({"id": "Foo.Bar", "type": "Namespace"})
        assert isinstance(cls, dotnet.DotNetNamespace)
        cls = _create_class({"id": "Foo.Bar", "type": "Class"})
        assert isinstance(cls, dotnet.DotNetClass)
        cls = _create_class({"id": "Foo.Bar", "type": "Property"})
        assert isinstance(cls, dotnet.DotNetProperty)
        cls = _create_class({"id": "Foo.Bar", "type": "Method"})
        assert isinstance(cls, dotnet.DotNetMethod)
        cls = _create_class({"id": "Foo.Bar", "type": "Enum"})
        assert isinstance(cls, dotnet.DotNetEnum)
        cls = _create_class({"id": "Foo.Bar", "type": "Constructor"})
        assert isinstance(cls, dotnet.DotNetConstructor)
        cls = _create_class({"id": "Foo.Bar", "type": "Struct"})
        assert isinstance(cls, dotnet.DotNetStruct)
        cls = _create_class({"id": "Foo.Bar", "type": "Interface"})
        assert isinstance(cls, dotnet.DotNetInterface)
        cls = _create_class({"id": "Foo.Bar", "type": "Delegate"})
        assert isinstance(cls, dotnet.DotNetDelegate)
        cls = _create_class({"id": "Foo.Bar", "type": "Field"})
        assert isinstance(cls, dotnet.DotNetField)
        cls = _create_class({"id": "Foo.Bar", "type": "Event"})
        assert isinstance(cls, dotnet.DotNetEvent)

    def test_create_class_with_children(self):
        dom = dotnet.DotNetSphinxMapper(MockApplication())

        def _create_class(data):
            return list(dom.create_class(data))[0]

        cls = _create_class(
            {
                "id": "Foo.Bar",
                "type": "Class",
                "items": [{"id": "Foo.Bar.Baz", "type": "Method"}],
            }
        )
        assert isinstance(cls, dotnet.DotNetClass)
        assert cls.item_map == {}

    @patch("subprocess.check_output", lambda foo: foo)
    def test_get_objects(self):
        """Test basic get objects"""
        objs = []

        def _mock_find(self, patterns, **kwargs):
            return {"items": ["foo", "bar"]}

        def _mock_read(self, path):
            return {
                "items": [
                    {"id": "Foo.Bar", "name": "Foo", "type": "property"},
                    {"id": "Foo.Bar2", "name": "Bar", "type": "property"},
                ],
                "id": "Foo.Bar",
                "type": "Class",
                "summary": path,
            }

        with patch("autoapi.mappers.dotnet.DotNetSphinxMapper.find_files", _mock_find):
            with patch(
                "autoapi.mappers.dotnet.DotNetSphinxMapper.read_file", _mock_read
            ):
                dom = dotnet.DotNetSphinxMapper(MockApplication())
                dom.load("", "", "")
                dom.map()
                objs = dom.objects
                assert len(objs) == 2
                assert objs["Foo.Bar"].id == "Foo.Bar"
                assert objs["Foo.Bar"].name == "Foo.Bar"
                assert objs["Foo.Bar2"].id == "Foo.Bar2"
                assert objs["Foo.Bar2"].name == "Foo.Bar2"

class TestDotNetPythonMapper:
    def test_xml_parse(self):
        """XML doc comment parsing"""
        ret = dotnet.DotNetPythonMapper.transform_doc_comments(
            'This is an example comment <see cref="FOO" />'
        )
        assert ret == "This is an example comment :any:`FOO`"

        ret = dotnet.DotNetPythonMapper.transform_doc_comments(
            'This is an example comment <see cref="!:FOO" />'
        )
        assert ret == "This is an example comment FOO"

        ret = dotnet.DotNetPythonMapper.transform_doc_comments(
            'This is an example comment <see cref="N:FOO">inner foo</see>'
        )
        assert ret == "This is an example comment :dn:ns:`FOO`"

        ret = dotnet.DotNetPythonMapper.transform_doc_comments(
            'Test <see cref="P:FOO" /> and <see cref="E:BAR">Blah</see>'
        )
        assert ret == "Test :dn:prop:`FOO` and :dn:event:`BAR`"

        ret = dotnet.DotNetPythonMapper.transform_doc_comments(
            'This is an example comment <paramref name="FOO" />'
        )
        assert ret == "This is an example comment ``FOO``"

        ret = dotnet.DotNetPythonMapper.transform_doc_comments(
            'This is an example comment <typeparamref name="FOO" />'
        )
        assert ret == "This is an example comment ``FOO``"

        ret = dotnet.DotNetPythonMapper.transform_doc_comments(
            'With surrounding characters s<see cref="FOO" />s'
        )
        assert ret == r"With surrounding characters s :any:`FOO`\s"

        ret = dotnet.DotNetPythonMapper.transform_doc_comments(
            'With surrounding characters s<paramref name="FOO" />s'
        )
        assert ret == r"With surrounding characters s ``FOO``\s"

    def test_xml_transform_escape(self):
        """XML transform escaping"""
        ret = dotnet.DotNetPythonMapper.transform_doc_comments(
            'Foo <see cref="Foo`1" /> Bar'
        )
        assert ret == "Foo :any:`Foo\\`1` Bar"

        ret = dotnet.DotNetPythonMapper.transform_doc_comments(
            'No space before<see cref="M:Foo`1" />or after'
        )
        assert ret == "No space before :dn:meth:`Foo\\`1`\\or after"

    def test_parsing_obj(self):
        """Parse out object, test for transforms, etc"""
        obj = {
            "uid": "Foo`1",
            "name": "Foo<TUser>",
            "summary": 'Test parsing <see cref="Bar" />',
            "syntax": {
                "parameters": [
                    {
                        "id": "a",
                        "type": "{TUser}",
                        "description": 'Test <see cref="TUser" />',
                    }
                ],
                "return": {
                    "type": "Bar",
                    "description": (
                        'Test references <see cref="Bar" /> '
                        'and paramrefs <paramref name="a" />'
                    ),
                },
            },
        }
        mapped = dotnet.DotNetPythonMapper(obj, app=mock.MagicMock(), jinja_env=None)
        expected = {"name": "a", "type": "{TUser}", "desc": "Test :any:`TUser`"}
        assert mapped.parameters[0] == expected
        assert (
            mapped.returns["description"]
            == "Test references :any:`Bar` and paramrefs ``a``"
        )


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import os
import io
from rest_framework import generics
from rest_framework.response import Response
from rest_framework.permissions import IsAuthenticated
from intranet.apps.search.views import get_search_results
from .models import User, Class, Grade
from .serializers import UserSerializer, ClassSerializer, StudentSerializer, CounselorTeacherSerializer
from .renderers import JPEGRenderer
from intranet import settings

class ProfileDetail(generics.RetrieveAPIView):
    """API endpoint that retrieves an Ion profile

    /api/profile: retrieve your profile
    /api/profile/<pk>: retrieve the profile of the user with id <pk>
    """
    serializer_class = UserSerializer
    permission_classes = (IsAuthenticated,)

    def retrieve(self, request, *args, **kwargs):
        if 'pk' in kwargs:
            user = User.objects.get(pk=kwargs['pk'])
        else:
            user = request.user

        serializer = self.get_serializer(user)
        data = serializer.data
        return Response(data)

class ProfilePictureDetail(generics.RetrieveAPIView):
    """API endpoint that retrieves an Ion profile picture

    /api/profile/<pk>/picture: retrieve default profile picture
    /api/profile/<pk>/picture/<photo_year>: retrieve profile picture for year <photo_year>
    """
    serializer_class = UserSerializer
    permission_classes = (IsAuthenticated,)
    renderer_classes = (JPEGRenderer,)

    def retrieve(self, request, *args, **kwargs):
        if 'pk' in kwargs:
            user = User.objects.get(pk=kwargs['pk'])
        else:
            user = request.user

        binary = None
        if 'photo_year' in kwargs:
            photo_year = kwargs['photo_year']
            if photo_year in Grade.names:
                binary = user.photo_binary(photo_year)
        else:
            binary = user.default_photo()
        if not binary:
            default_image_path = os.path.join(settings.PROJECT_ROOT, "static/img/default_profile_pic.png")
            binary = io.open(default_image_path, mode="rb").read()

        response = Response(binary, content_type='image/jpeg')
        return response

class ClassDetail(generics.RetrieveAPIView):
    """API endpoint that retrieves details of a TJHSST class
    """
    serializer_class = ClassSerializer
    permission_classes = (IsAuthenticated,)

    def retrieve(self, request, *args, **kwargs):
        cl = Class(id=kwargs['pk'])

        serializer = self.get_serializer(cl)
        return Response(serializer.data)

class Search(generics.RetrieveAPIView):
    """API endpoint that retrieves the results of a search for Ion users

    Paginated using ?page=<page>
    """

    permission_classes = (IsAuthenticated,)

    def retrieve(self, request, *args, **kwargs):
        query = kwargs['query']
        user_ids = []
        query = query.replace("+", " ")
        query_error, results = get_search_results(query)
        for unserialized_user in results:
            user_ids.append(unserialized_user.id)

        queryset = User.objects.filter(pk__in=user_ids)
        users = self.paginate_queryset(queryset)

        response = []
        for user in users:
            if user.is_student:
                response.append(StudentSerializer(user, context={'request': request}).data)
            else:
                response.append(CounselorTeacherSerializer(user, context={'request': request}).data)

        return self.get_paginated_response(response)


#!/usr/bin/env python3
from tkinter import *
from tkinter import messagebox
import configparser
import xml.etree.ElementTree as ET
import grequests
import requests
import grequests
import xmltodict

# main program ################################################################
def main(*args):
    # barcode
    barcode = gui.get_barcode()
    if barcode == "":
        gui.msgbox(barcode, "Bad barcode")
        return
    gui.clear_barcode()

    # get item record
    url = "https://api-na.hosted.exlibrisgroup.com/almaws/v1/items?item_barcode="+barcode+"&apikey="+apikey
    r = requests.get(url)

    # check for invalid api key
    if r.text == "Invalid API Key":
        gui.msgbox(barcode, "Invalid API Key")
        return

    # check for errors
    errors_exist = check_errors_200(r)
    if errors_exist[0] == True:
        error = errors_exist[1]
        gui.msgbox(barcode, error)
        return

    # get item values
    item_xml    = r.text
    item       = ET.fromstring(item_xml)
    title      = item.find('bib_data/title').text[:50]
    mms_id     = item.find('bib_data/mms_id').text
    holding_id = item.find('holding_data/holding_id').text
    item_pid   = item.find('item_data/pid').text
    desc       = item.find('item_data/description').text

    if desc == None:
        desc = ""

    # other records
    urls = [
        "https://api-na.hosted.exlibrisgroup.com/almaws/v1/bibs/"+mms_id+"?apikey="+apikey,
        "https://api-na.hosted.exlibrisgroup.com/almaws/v1/bibs/"+mms_id+"/holdings?apikey="+apikey,
        "https://api-na.hosted.exlibrisgroup.com/almaws/v1/bibs/"+mms_id+"/holdings/"+holding_id+"/items?limit=10&offset=0&apikey="+apikey,
    ]

    r = getXML(urls)
    bib_xml         = r[0].text
    holdings_xml    = r[1].text
    other_items_xml = r[2].text

    # check for multiple holdings
    holdings = ET.fromstring(holdings_xml)
    h_dict   = holdings.attrib
    holdings_count = int(h_dict['total_record_count'])

    # check for other items
    other_items    = ET.fromstring(other_items_xml)
    item_dict      = other_items.attrib
    items_count = int(item_dict['total_record_count'])

    # check if last item and holding on record
    if last_item_check == "active":
        if holdings_count == 1 and items_count == 1:
           gui.msgbox(title, "LAST ITEM ON RECORD")
           return

    # add statistics note to item
    if add_item_note == "active":
        item_stat_note = item.find('item_data/'+item_note_field)
        item_stat_note.text = item_note

        # make final changes to item
        item_final = ET.tostring(item, encoding="unicode", method="xml")
        url = "https://api-na.hosted.exlibrisgroup.com/almaws/v1/bibs/"+mms_id+"/holdings/"+holding_id+"/items/"+item_pid+"?apikey="+apikey
        r = putXML(url, item_final)

        # check for errors
        errors_exist = check_errors_200(r)
        if errors_exist[0] == True:
            error = errors_exist[1]
            gui.msgbox(title, error)
            return

    # withdraw item and holdings
    if wd_item == "active":
        url     = "https://api-na.hosted.exlibrisgroup.com/almaws/v1/bibs/"+mms_id+"/holdings/"+holding_id+"/items/"+item_pid+"?override=false&holdings=retain&apikey="+apikey
        headers = {'Content-Type': 'application/xml', 'charset':'UTF-8'}
        r = deleteXML(url)

        # check for errors
        errors_exist = check_errors_204(r)
        if errors_exist[0] == True:
            error = errors_exist[1]
            gui.msgbox(title, error)
            return

    # finish
    gui.update_status_success(title+" ("+str(desc)+")")

# functions ###################################################################
def getXML(urls):
    rs = (grequests.get(u) for u in urls)
    r = grequests.map(rs)
    return r

def check_errors_200(r):
    if r.status_code != 200:
        errors = xmltodict.parse(r.text)
        error = errors['web_service_result']['errorList']['error']['errorMessage']
        return True, error
    else:
        return False, "OK"

def check_errors_204(r):
    if r.status_code != 204:
        errors = xmltodict.parse(r.text)
        error = errors['web_service_result']['errorList']['error']['errorMessage']
        return True, error
    else:
        return False, "OK"

def putXML(url, xml):
    headers = {'Content-Type': 'application/xml', 'charset':'UTF-8'}
    r = requests.put(url, data=xml.encode('utf-8'), headers=headers)
    return r

def deleteXML(url):
    headers = {'Content-Type': 'application/xml', 'charset':'UTF-8'}
    r = requests.delete(url, headers=headers)
    return r

# configurations ##############################################################
config = configparser.ConfigParser()
config.read('config.ini')

apikey                         = config['misc']['apikey']
version                        = config['misc']['version']

last_item_check                = config['checks']['last_item']

item_note_field                = config['stats']['item_note_field']
item_note                      = config['stats']['item_note']

add_item_note                  = config['operations']['add_item_note']
wd_item                        = config['operations']['wd_item']

# gui #########################################################################
class gui:
    def __init__(self, master):
        self.master = master
        master.title("Item-Be-Gone "+version)
        master.resizable(0, 0)
        master.minsize(width=600, height=100)
        master.iconbitmap("./images/logo_small.ico")

        logo = PhotoImage(file="./images/logo_large.png")
        self.logo = Label(image=logo)
        self.logo.image = logo
        self.logo.pack()

        self.status_title = Label(height=1, text="Scan barcode to begin.", font="Consolas 12 italic")
        self.status_title.pack(fill="both", side="top")

        self.status_wd = Label(height=1, text="READY", font="Consolas 12 bold", fg="green")
        self.status_wd.pack(fill="both", side="top")

        self.barcode_entry_field = Entry(font="Consolas 16")
        self.barcode_entry_field.focus()
        self.barcode_entry_field.bind('<Key-Return>', main)
        self.barcode_entry_field.pack(fill="both", side="top")

        self.scan_button = Button(text="SCAN", font="Arial 16", command=main)
        self.scan_button.pack(fill="both", side="top")

    def msgbox(self, title, msg):
        messagebox.showerror("Attention", msg)
        self.update_status_failure(title, msg)

    def get_barcode(self):
        barcode = self.barcode_entry_field.get()
        barcode = barcode.replace(" ", "")
        return barcode

    def clear_barcode(self):
        self.barcode_entry_field.delete(0, END)
        self.status_title.config(text="")
        self.status_wd.config(text="")

    def update_status_success(self, title):
        self.status_title.config(text=title)
        self.status_wd.config(text="SUCCESSFULLY WD", fg="green")

    def update_status_failure(self, title, msg):
        self.status_title.config(text=title)
        self.status_wd.config(text=msg, fg="red")

root = Tk()
gui = gui(root)
root.mainloop()


from unittest.mock import Mock, patch
import pytest
import numpy as np

from shfl.model.linear_classifier_model import LinearClassifierModel

@pytest.fixture(name="wrapper_arguments")
def fixture_wrapper_arguments():
    """Returns the component necessary for wrapping a k-means clustering model."""
    n_features = 9
    classes = ["a", "b", "c"]

    return n_features, classes

@pytest.fixture(name="input_data")
def fixture_input_data(wrapper_arguments):
    """Returns a random labeled dataset."""
    n_features, classes = wrapper_arguments
    num_data = 50
    data = np.random.rand(num_data, n_features)
    labels = np.random.choice(classes, size=num_data)

    return data, labels

@pytest.mark.parametrize("classes, n_classes", [(["a", "b"], 1),
                                                (["a", "b", "c"], 3)])
@patch('shfl.model.linear_classifier_model.LogisticRegression')
def test_initialization_binary_classes(mock_classifier, classes, n_classes,
                                       wrapper_arguments):
    """Checks that the linear classifier correctly initializes.

    For a binary classification, the number of classes
    "n_classes" is equal to one. Instead, for a multi-class case,
    the number of classes "n_classes" is equal
    to the actual number of classes."""
    model = Mock()
    mock_classifier.return_value = model

    wrapped_model = LinearClassifierModel(n_features=wrapper_arguments[0],
                                          classes=classes)

    assert hasattr(wrapped_model, "_model")
    assert hasattr(wrapped_model, "_n_features")
    assert model.intercept_.shape[0] == n_classes
    assert model.coef_.shape == (n_classes, wrapper_arguments[0])
    assert np.array_equal(classes, model.classes_)

@pytest.mark.parametrize("n_features, classes", [(9.5, ['a', 'b', 'c']),
                                                 (-1, ['a', 'b', 'c']),
                                                 (9, ['b']),
                                                 (9, ['a', 'b', 'a'])])
@patch('shfl.model.linear_classifier_model.LogisticRegression')
def test_model_wrong_initialization(mock_classifier, n_features, classes):
    """Checks that the linear classification model throws an error if
    not initialized correctly.

    Namely, the number of features must be: integer, non-negative.
    The classes must be: more than one, not repeating."""
    mock_classifier.return_value = Mock()

    with pytest.raises(AssertionError):
        LinearClassifierModel(n_features, classes)

@patch('shfl.model.linear_classifier_model.LogisticRegression')
def test_train(mock_classifier, wrapper_arguments, input_data):
    """Checks that the linear classifier model trains correctly."""
    model = Mock()
    mock_classifier.return_value = model
    wrapped_model = LinearClassifierModel(*wrapper_arguments)

    wrapped_model.train(*input_data)

    model.fit.assert_called_once_with(*input_data)

@patch('shfl.model.linear_classifier_model.LogisticRegression')
def test_train_wrong_data(mock_classifier, wrapper_arguments, input_data, helpers):
    """Checks that the linear classifier model throws an error if wrong
    data are used as input."""
    mock_classifier.return_value = Mock()
    wrapped_model = LinearClassifierModel(*wrapper_arguments)

    helpers.check_wrong_data(wrapped_model, *input_data)

@patch('shfl.model.linear_classifier_model.LogisticRegression')
def test_train_wrong_data_single_feature(mock_classifier, wrapper_arguments, input_data):
    """Checks that the linear classifier model throws an error if wrong
    data are used as input.

    If data contains only one column, then the number of features must be 1."""
    mock_classifier.return_value = Mock()
    wrapped_model = LinearClassifierModel(*wrapper_arguments)
    data, labels = input_data
    wrong_data = np.random.rand(len(data))

    with pytest.raises(AssertionError):
        wrapped_model.train(wrong_data, labels)

@patch('shfl.model.linear_classifier_model.LogisticRegression')
def test_train_wrong_labels(mock_classifier, wrapper_arguments, input_data):
    """Checks that the linear classifier model throws an error if wrong
    labels are used as input."""
    mock_classifier.return_value = Mock()
    wrapped_model = LinearClassifierModel(*wrapper_arguments)
    data, labels = input_data
    wrong_labels = labels
    wrong_labels[0] = "not_initialized_class"

    with pytest.raises(AssertionError):
        wrapped_model.train(data, wrong_labels)

@patch('shfl.model.linear_classifier_model.LogisticRegression')
def test_predict(mock_classifier, wrapper_arguments, input_data):
    """Checks that the linear classifier model predicts correctly."""
    data, labels = input_data
    model = Mock()
    true_prediction = np.random.choice(labels, size=len(data))
    model.predict.return_value = true_prediction
    mock_classifier.return_value = model
    wrapped_model = LinearClassifierModel(*wrapper_arguments)

    output_prediction = wrapped_model.predict(data)

    model.predict.assert_called_once_with(data)
    np.testing.assert_array_equal(output_prediction, true_prediction)

@patch('shfl.model.linear_classifier_model.metrics')
@patch('shfl.model.linear_classifier_model.LogisticRegression')
def test_evaluate(mock_classifier, mock_metrics, wrapper_arguments, input_data):
    """Checks that the linear classifier model evaluates correctly."""
    data, labels = input_data
    model = Mock()
    mock_classifier.return_value = model
    wrapped_model = LinearClassifierModel(*wrapper_arguments)
    wrapped_model.predict = Mock()
    true_prediction = np.random.choice(labels, size=len(data))
    wrapped_model.predict.return_value = true_prediction
    mock_metrics.balanced_accuracy_score.return_value = 0.5
    mock_metrics.cohen_kappa_score.return_value = 0.7

    balanced_accuracy_score, cohen_kappa_score = wrapped_model.evaluate(data, labels)

    wrapped_model.predict.assert_called_once_with(data)
    mock_metrics.balanced_accuracy_score.assert_called_once_with(labels, true_prediction)
    mock_metrics.cohen_kappa_score.assert_called_once_with(labels, true_prediction)
    assert balanced_accuracy_score == 0.5
    assert cohen_kappa_score == 0.7

@patch('shfl.model.linear_classifier_model.LogisticRegression')
def test_evaluate_wrong_labels(mock_classifier, wrapper_arguments, input_data):
    """Checks that the linear classifier model throws an error if wrong
    labels are used as input."""
    mock_classifier.return_value = Mock()
    wrapped_model = LinearClassifierModel(*wrapper_arguments)
    data, labels = input_data
    wrong_labels = labels
    wrong_labels[0] = "not_initialized_class"

    with pytest.raises(AssertionError):
        wrapped_model.evaluate(data, wrong_labels)

@patch('shfl.model.linear_classifier_model.metrics')
@patch('shfl.model.linear_classifier_model.LogisticRegression')
def test_performance(mock_classifier, mock_metrics, wrapper_arguments, input_data):
    """Checks that the linear classifier model calls performance correctly."""
    data, labels = input_data
    model = Mock()
    mock_classifier.return_value = model
    wrapped_model = LinearClassifierModel(*wrapper_arguments)
    wrapped_model.predict = Mock()
    true_prediction = np.random.choice(labels, size=len(data))
    wrapped_model.predict.return_value = true_prediction
    mock_metrics.balanced_accuracy_score.return_value = 0.5

    balanced_accuracy_score = wrapped_model.performance(data, labels)

    wrapped_model.predict.assert_called_once_with(data)
    assert balanced_accuracy_score == 0.5

@patch('shfl.model.linear_classifier_model.LogisticRegression')
def test_get_model_params(mock_classifier, wrapper_arguments):
    """Checks that the linear classifier gets the model's parameters correctly."""
    model = Mock()
    mock_classifier.return_value = model
    wrapped_model = LinearClassifierModel(*wrapper_arguments)

    output_params = wrapped_model.get_model_params()

    np.testing.assert_array_equal(model.intercept_, output_params[0])
    np.testing.assert_array_equal(model.coef_, output_params[1])

@patch('shfl.model.linear_classifier_model.LogisticRegression')
def test_set_model_params(mock_classifier, wrapper_arguments):
    """Checks that the linear classifier sets the model's parameters correctly."""
    n_features, classes = wrapper_arguments
    model = Mock()
    mock_classifier.return_value = model
    wrapped_model = LinearClassifierModel(n_features, classes)
    n_classes = len(classes)
    input_params = (np.random.rand(n_classes),
                    np.random.rand(n_classes, n_features))

    wrapped_model.set_model_params(input_params)

    np.testing.assert_array_equal(model.intercept_, input_params[0])
    np.testing.assert_array_equal(model.coef_, input_params[1])


#find a set of data of Ge that gives the energy gap that is closest to the expected data.
from scipy import optimize
import numpy as np
import pylab as pl

pl.rc('axes', linewidth=2)

# set up your read_array to use later to read in your file
def read_array(filename, dtype, separator=','):
    """ Read a file with an arbitrary number of columns.
        The type of data in each column is arbitrary
        It will be cast to the given dtype at runtime
    """
    cast = np.cast
    data = [[] for dummy in range(len(dtype))]
    for line in open(filename, 'r'):
        fields = line.strip().split(separator)
        for i, number in enumerate(fields):
            data[i].append(number)
    for i in range(len(dtype)):
        data[i] = cast[dtype[i]](data[i])
    return np.rec.array(data, dtype=dtype)

# now read in your file -- the line below gives examples of datatypes
#mydescr = np.dtype([('column1', 'int32'), ('column2Name', 'uint32'), ('col3', 'uint64'), ('c4', 'float32')])
mydescr = np.dtype([('xpos', 'float32'), ('xerr', 'float32'), ('ypos', 'float32'),('yerr', 'float32')])
def column(matrix, i):
    return [row[i] for row in matrix]
def func(x, a, b):
    return a + b*x
temperature = [0.002675,0.003394,0.003619,0.004831]
fileList = ['Ge_boiling.csv', 'Ge_room2.csv', 'Ge_ice.csv', 'Ge_dryice.csv']
total = []
for f in fileList:
    orecarray = read_array(f, mydescr)
    dLen = len(orecarray)
    filetotal = []
#set the range of data that will be analyzed
    if f == fileList[0]:
        p1 = 5
        p2 = dLen-15
        p3 = 10
        t = temperature[0]
    elif f == fileList[1]:
        p1 = 3
        p2 = dLen-5
        p3 = 10
        t = temperature[1]
    elif f == fileList[2]:
        p1 = 5
        p2 = dLen-13
        p3 = 10
        t = temperature[2]
    elif f == fileList[3]:
        p1 = 5
        p2 = dLen-13
        p3 = 10
        t = temperature[3]
    def func2(x, a):
        return a+np.log(np.exp(1.602*(10**(-19))/(1.38*(10**(-23)))*t*x)-1)
    for x in range(p1,p2):
        for y in range(x+p3,p2):
            myrecarray = orecarray[x:y]
            x0    = np.array([-10])
            sigma = myrecarray.yerr
            result = optimize.curve_fit(func2, myrecarray.xpos, myrecarray.ypos, x0, sigma)
            a = result[0][0]
            da = np.sqrt(result[1][0][0])
            #b = result[0][1]
            #db = np.sqrt(result[1][1][1])
            filetotal.append([a,da,x,y])
    total.append(filetotal)
#now try all combinations, calculate Eg for each combination, then select one that
#is closest to 0.68.
allSlopes = []
count = len(total[0])
for i in total[0]:
    count -= 1
    print(count)
    for j in total[1]:
        for k in total[2]:
            for l in total[3]:
                data = []
                data.append(i)
                data.append(j)
                data.append(k)
                data.append(l)

                x0 = np.array([10,-7891])
                sigma = column(data,1)
                result = optimize.curve_fit(func, [0.002675,0.003394,0.003619,0.004831], column(data,0), x0, sigma)
                allSlopes.append([np.abs(result[0][1]+7891),result[0][0],np.sqrt(result[1][0][0]),result[0][1],np.sqrt(result[1][1][1]),data])
eg=[row[0] for row in allSlopes]
r = allSlopes[eg.index(min(eg))] # the wanted one!
print(r)

#plot the select data set
i=0
for f in fileList:
    orecarray = read_array(f, mydescr)
    pl.figure(1)
    pl.errorbar(orecarray.xpos[r[5][i][2]:r[5][i][3]],orecarray.ypos[r[5][i][2]:r[5][i][3]],orecarray.yerr[r[5][i][2]:r[5][i][3]],orecarray.xerr[r[5][i][2]:r[5][i][3]], fmt = 'o')
    i+=1

pl.figure(2)
pl.errorbar([0.002675,0.003394,0.003619,0.004831],column(r[5],0),column(r[5],1),[0.00002675,0.00003394,0.00003619,0.00004831], fmt = 'o')
a = r[1]
b = r[3]
#plot the fit line, slope = -Eg/k
x1 = (-np.amax([0.002675,0.003394,0.003619,0.004831]) + 11*np.amin([0.002675,0.003394,0.003619,0.004831]))*0.1
x2 = (11*np.amax([0.002675,0.003394,0.003619,0.004831]) - np.amin([0.002675,0.003394,0.003619,0.004831]))*0.1
pl.plot([x1, x2], [a+b*x1, a+b*x2])
print('Eg = ',-1*b*8.617*10**(-5)) #print Eg
#print(allSlopes[(np.argmin(allSlopes, axis = 0, ))[0]])
# Change size and font of tick labels
# Again, this doesn't work in interactive mode.
fontsize = 14
ax = pl.gca()
for tick in ax.xaxis.get_major_ticks():
    tick.label1.set_fontsize(fontsize)
    tick.label1.set_fontweight('bold')
for tick in ax.yaxis.get_major_ticks():
    tick.label1.set_fontsize(fontsize)
    tick.label1.set_fontweight('bold')

pl.xlabel('V', fontsize=16, fontweight='bold')
pl.ylabel('lnI', fontsize=16, fontweight='bold')

# save the plot to a file
#pl.savefig('HEP.png', bbox_inches='tight')
# display the plot so you can see it
pl.show()

Semi Conductor Band Gap/Lab Data/2/fitHEP_multi_Geclosest.py


# -*- coding: utf-8 -*-
import logging as logger

from googleapiclient.discovery import build
from httplib2 import Http
from oauth2client import file, client, tools

class GCalendar:
    def __init__(self, cal_id='primary', timezone='Asia/Seoul'):
        self.cal_id = cal_id
        self.timezone = timezone

        # rw calendar
        scope = 'https://www.googleapis.com/auth/calendar'
        store = file.Storage('token.json')
        creds = store.get()
        if not creds or creds.invalid:
            flow = client.flow_from_clientsecrets('credentials.json', scope)
            creds = tools.run_flow(flow, store)
        self.service = build('calendar', 'v3', http=creds.authorize(Http()))

    def event_exists(self, event_id):
        service = self.service
        query = '{}={}'.format('event_id', event_id)
        event = service.events().list(calendarId=self.cal_id,
                                      privateExtendedProperty=query).execute()
        if len(event['items']) == 0:
            return False
        return True

    def create_event(self, event_id, title, location, description, start_dt, end_dt):
        if event_id is None or title is None or location is None or \
                description is None or start_dt is None or end_dt is None:
            logger.warning("Incomplete data. Skipping event...")
            return

        event = {
            'summary': title,
            'location': location,
            'description': description,
            'start': {
                'dateTime': start_dt.strftime('%Y-%m-%dT%H:%M:%S'),
                'timeZone': self.timezone,
            },
            'end': {
                'dateTime': end_dt.strftime('%Y-%m-%dT%H:%M:%S'),
                'timeZone': self.timezone,
            },
            'extendedProperties': {
                'private': {
                    'event_id': event_id
                }
            }
        }

        service = self.service
        event = service.events().insert(calendarId=self.cal_id,
                                        body=event).execute()
        logger.warning('Event created: %s' % (event.get('htmlLink')))


import json
import logging
import os
from typing import Dict

import subprocess
import sys

def install(package):
    subprocess.check_call([sys.executable, "-m", "pip", "install", package])

# install('pandas')
# install('Elasticsearch')

import numpy as np
import pandas as pd
from elasticsearch import Elasticsearch, helpers
import csv

logging.shutdown()
# logging.basicConfig(filename="es.log", level=logging.INFO)
logger = logging.getLogger('es')
logger.setLevel(logging.INFO)
fh = logging.FileHandler(logger.name + '.log', mode='w')
fh.setLevel(logging.INFO)
formatter = logging.Formatter(fmt='%(asctime)s\t\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p')
fh.setFormatter(formatter)
logger.addHandler(fh)

class EsManagement:
    def __init__(self):
        self.es = Elasticsearch([{'host': 'localhost', 'port': 9200}])
        logger.info(self.es.ping())

    def create_index(self, index_name: str, mapping: Dict) -> None:
        """
        Create an ES index.
        :param index_name: Name of the index.
        :param mapping: Mapping of the index
        """
        logger.info(f"Creating index {index_name} with the following schema: {json.dumps(mapping, indent=2)}")
        print(self.es.indices.create(index=index_name, ignore=400, body=mapping))

    def clear_index(self, index_name: str):
        """
        delete an ES index.
        :param index_name: Name of the index.
        """
        logger.info(f"deleting index {index_name}")
        print(self.es.indices.delete(index=index_name, ignore=[400, 404]))

    def load_csv_into_index(self, path: str, index_name: str) -> None:
        """
        load data to an index from a CSV file.
        :param path: The path to the CSV file.
        :param index_name: Name of the index to which documents should be written.
        """
        with open(path, encoding="utf8") as f:
            reader = csv.DictReader(f)
            rows_num = helpers.bulk(self.es, reader, index=index_name)[0]
            logger.info(f"Writing {rows_num} documents to ES index {index_name}")


import unittest
import pyrtl
from pyrtl import transform

class NetWireNumTestCases(unittest.TestCase):
    def setUp(self):
        pyrtl.reset_working_block()

    def assert_num_net(self, num, block=None):
        block = pyrtl.working_block(block)
        self.assertEqual(len(block.logic), num)

    def assert_num_wires(self, num, block=None):
        block = pyrtl.working_block(block)
        self.assertEqual(len(block.wirevector_set), num)

    def num_net_of_type(self, netOp, num, block=None):
        block = pyrtl.working_block(block)
        self.assertEquals(len([net for net in block.logic if net.op == netOp]), num)

    def num_wire_of_type(self, wiretype, num, block=None):
        block = pyrtl.working_block(block)
        self.assertEquals(len(block.wirevector_subset(wiretype)), num)

def insert_random_inversions(rate=0.5):
    """
    an example transform that can be used for testing
    """

    import random

    def randomly_replace(wire):
        if random.random() < rate:
            new_src, new_dst = transform.clone_wire(wire), transform.clone_wire(wire)
            new_dst <<= ~new_src
            return new_src, new_dst
        return wire, wire

    transform.wire_transform(randomly_replace)

class TestWireTransform(NetWireNumTestCases):
    def test_randomly_replace(self):
        a, b = pyrtl.WireVector(3), pyrtl.WireVector(3)
        o = a & b
        insert_random_inversions(1)
        block = pyrtl.working_block()
        self.num_net_of_type('~', 3, block)
        self.num_net_of_type('&', 1, block)

        new_and_net = block.logic_subset('&').pop()
        for arg in new_and_net.args:
            self.assertIsNot(arg, a)
            self.assertIsNot(arg, b)
        self.assertIsNot(new_and_net.dests[0], o)

class TestCopyBlock(NetWireNumTestCases):
    def num_memories(self, mems_expected, block):
        memories = set()
        for net in block.logic_subset('m@'):
            memories.add(net.op_param[1])  # location of the memories object
        self.assertEqual(mems_expected, len(memories))

    def test_blank(self):
        block = transform.copy_block()
        self.assert_num_net(0, block)
        self.assert_num_wires(0, block)

    def test_block(self):
        a = pyrtl.Const(23)
        b = pyrtl.Input(5)
        o = pyrtl.Output(5)
        o <<= ~a & b

        old_block = pyrtl.working_block()
        old_block.sanity_check()
        self.assert_num_wires(5, old_block)
        self.assert_num_net(3, old_block)

        new_block = transform.copy_block()
        new_block.sanity_check()
        self.assert_num_wires(5, new_block)
        self.assert_num_net(3, old_block)

    def test_copy_mem(self):
        ins = [pyrtl.Input(5) for i in range(4)]
        out = pyrtl.Output(5)

        mem1 = pyrtl.MemBlock(5, 5)
        mem2 = pyrtl.MemBlock(5, 5)

        mem1_o1 = mem1[ins[0]]
        mem1[ins[1]] <<= ins[2]
        mem2_o2 = mem2[ins[3]]
        out <<= mem1_o1 & mem2_o2

        old_block = pyrtl.working_block()
        old_block.sanity_check()
        self.num_net_of_type('m', 2, old_block)
        self.num_net_of_type('@', 1, old_block)
        self.num_net_of_type('&', 1, old_block)
        self.num_memories(2, old_block)

        new_block = transform.copy_block()
        self.num_net_of_type('m', 2, new_block)
        self.num_net_of_type('@', 1, new_block)
        self.num_net_of_type('&', 1, new_block)
        self.num_memories(2, new_block)

class TestFastWireReplace(unittest.TestCase):
    def setUp(self):
        pyrtl.reset_working_block()

    def test_replace_multiple_wires(self):
        j, n = pyrtl.Input(8), pyrtl.Output(8)
        o, h = pyrtl.WireVector(), pyrtl.WireVector()
        x, y = pyrtl.WireVector(8), pyrtl.WireVector(8)

        o <<= j
        h <<= o
        n <<= h
        block = pyrtl.working_block()
        src_nets, dst_nets = block.net_connections()
        transform.replace_wire_fast(o, x, x, src_nets, dst_nets)
        transform.replace_wire_fast(h, y, y, src_nets, dst_nets)
        for old_wire in (o, h):
            self.assertNotIn(old_wire, src_nets)
            self.assertNotIn(old_wire, dst_nets)
            self.assertNotIn(old_wire, block.wirevector_set)
        block.sanity_check()

    def test_replace_multiple_wires_2(self):
        j, n = pyrtl.Input(8), pyrtl.Output(8)
        o = pyrtl.WireVector()
        x, y, z = pyrtl.WireVector(8), pyrtl.WireVector(8), pyrtl.WireVector(8)

        o <<= j
        p = ~ j
        h = o & p
        n <<= h
        block = pyrtl.working_block()
        src_nets, dst_nets = block.net_connections()
        transform.replace_wire_fast(o, x, x, src_nets, dst_nets)
        transform.replace_wire_fast(p, z, z, src_nets, dst_nets)
        transform.replace_wire_fast(h, y, y, src_nets, dst_nets)
        for old_wire in (o, h, p):
            self.assertNotIn(old_wire, src_nets)
            self.assertNotIn(old_wire, dst_nets)
            self.assertNotIn(old_wire, block.wirevector_set)
        block.sanity_check()

    def test_wire_used_in_multiple_places(self):
        j, k = pyrtl.Input(8), pyrtl.Input(8)
        n, o = pyrtl.Output(8), pyrtl.Output(8)
        x = pyrtl.WireVector(8)

        r = j & k
        n <<= j | r
        o <<= r ^ k

        block = pyrtl.working_block()
        src_nets, dst_nets = block.net_connections()
        transform.replace_wire_fast(r, x, x, src_nets, dst_nets)

        for old_wire in (r,):
            self.assertNotIn(old_wire, src_nets)
            self.assertNotIn(old_wire, dst_nets)
            self.assertNotIn(old_wire, block.wirevector_set)
        block.sanity_check()

# this code needs mocking from python 3's unittests to work
"""
@mock.patch('transform_examples.pyrtl.probe')
def test_probe(self, probe):
    # Note to readers, this is a rather contrived test
    # If you want to know how to probe a single wirevector, look at the
    # probe function in pyrtl core
    in_wire, in_wire2 = pyrtl.Input(3), pyrtl.Input(3)
    out_wire = pyrtl.Output()
    test_wire = ~(in_wire & in_wire2)
    out_wire <<= test_wire

    def probe_cond(wire):
        return wire is test_wire

    transform_examples.probe_wire_if(probe_cond)
    probe.assert_called_once_with(test_wire)
"""


#This program creates an analog clock that keeps track of the current real world time.

from datetime import datetime
from math import pi, sin, cos
import tkinter

#Class for Clock
class AnalogClock:

    #Member Variables
    rootWindow = tkinter.Tk()
    canvas = tkinter.Canvas(rootWindow, width=600, height=600, borderwidth=0, highlightthickness=0, bg="white")
    time = datetime.now().time() #t.hour, t.min, t.sec

    #Updates the current time for the class object, so the TimeKeeper function can access it.
    #Also deletes the previous lines for the hour, minute, and second hands
    def timeUpdate(self):
        self.canvas.delete("hour", "minute" , "second")
        self.time = datetime.now().time() #t.hour, t.min, t.sec
        self.TimeKeeper()
        self.rootWindow.after(500, self.timeUpdate)

    #Creates clock w/labels and labels the canvas
    def createCircleAndLabels(self):
        self.canvas.create_oval(100, 500, 500, 100, width = 2, fill='white')
        self.canvas.create_oval(295, 305, 305, 295, fill = "black")
        j = 0
        for i in range(0,60):
            angle = ((15-i)/15)*(pi/2.0)
            tipx = cos(angle) * 190
            tipy = sin(angle) * 190
            if i % 5 == 0:
                tipx = cos(angle) * 230
                tipy = sin(angle) * 230
                if j == 0:
                    self.canvas.create_text(300 + tipx, 300 - tipy, text = str(12), font = ("times new roman", 40))
                else:
                    self.canvas.create_text(300 + tipx, 300 - tipy, text = str(j), font = ("times new roman", 40))
                j = j + 1
            else:
                tipx = cos(angle) * 210
                tipy = sin(angle) * 210
                self.canvas

    #Calculates the angle for the new time, and draws the new lines for the hands
    def TimeKeeper(self):
        timeSplit = str(self.time)
        timeSplit = timeSplit.split(":")
        iHr = int(timeSplit[0])
        iMin = int(timeSplit[1])
        iSec = round(float(timeSplit[2]))

        fHr = ((iHr % 12) * 5) + ((iMin/60) * 5)
        lenHourHand = 100
        fMin = ((iMin)) + ((iSec/60))
        lenMinuteAndSecHand = 150
        lenSecHand =  150

        #Draw Hour Hand
        angle = ((15-fHr)/15)*(pi/2.0)
        tipx = cos(angle) * lenHourHand
        tipy = sin(angle) * lenHourHand
        self.canvas.create_line(300, 300, 300 + tipx, 300 - tipy, width = 4, tags = "hour")

        #Draw Minute Hand
        angle = ((15-fMin)/15)*(pi/2.0)
        tipx = cos(angle) * lenMinuteAndSecHand
        tipy = sin(angle) * lenMinuteAndSecHand
        self.canvas.create_line(300, 300, 300 + tipx, 300 - tipy, width = 2, tags = "minute")

        #Draw Second Hand
        angle = ((15-iSec)/15)*(pi/2.0)
        tipx = cos(angle) * lenMinuteAndSecHand
        tipy = sin(angle) * lenMinuteAndSecHand
        self.canvas.create_line(300, 300, 300 + tipx, 300 - tipy, width = 2, fill = "black", tags = "second")

    #Packs the canvas, and recursively calls the timeUpdate function
    def end(self):
        self.canvas.pack()
        self.rootWindow.after(500, self.timeUpdate)
        self.rootWindow.mainloop()

def main():

    #Declare Object, and call its member functions
    MyClock = AnalogClock()
    MyClock.createCircleAndLabels()
    MyClock.TimeKeeper()
    MyClock.end()

if __name__ == "__main__":
    main()


#! /usr/bin/env python
# coding=utf-8
from PIL import Image
import os
from bmfont import FontGenerator
from collections import OrderedDict
from bmfont import Fnt
from packedfont import PackedFont
import sys

def do(name, width=512, height=512):
    print(name)
    pf = PackedFont(open(name, "rb"))
    size = int(pf.f0)
    if size < 13:
        size = 13
    print(pf.f0, pf.f1, pf.f2, size)
    font_gen = FontGenerator()

    texts = TEXTS
    pf.f2 *= 1.5
    if 'mono' in name:
        font_gen.set_font_name("Consolas")
        size += 2
    else:
        font_gen.set_font_name("Arial Unicode MS")
    font_gen.set_font_size(-size)
    font_gen.set_texture_format("png")
    font_gen.set_chars(texts)
    font_gen.set_texture_size(width, height)
    font_gen.set_fixed_height(True)
    font_gen.set_enable_kernings(False)
    font_gen.set_spacing(3)

    if "bold" in name:
        font_gen.set_font_bold(True)

    if 'italic' in name:
        font_gen.set_font_italic(True)

    fnt = font_gen.gen()

    pf.set_chars(u"".join([chr(c["id"]) for c in fnt.chars]))
    pf.textures = []
    for page in fnt.pages:
        im = Image.open(page).transpose(Image.FLIP_TOP_BOTTOM)
        _, _, _, im = im.split()
        pf.textures.append(im)

    pf.glyphs = []
    for c in fnt.chars:
        pf.glyphs.append({
            "index": c["page"],
            "x": c["x"],
            "y": height - c["y"] - c["height"],
            "w": c["width"],
            "h": c["height"],
            "offsetx": c["xoffset"],
            "offsety": size - fnt.common.base - 3,
            "adv": c["xadvance"]
        })

    pf.save(open(os.path.split(name)[1], "wb"))

if __name__ == "__main__":
    TEXTS = open("chars.txt", "r", encoding='utf-8').read()

    codes = [[ord(TEXTS[0]), ord(TEXTS[0])], ]
    for t in TEXTS[1:]:
        t = ord(t)
        if codes[-1][1] + 1 == t:
            codes[-1][1] += 1
        else:
            codes.append([t, t])

    # for i, c in enumerate(codes):
        # print i, c

    for root, dirs, files in os.walk("fonts"):
        for f in files:
            do(os.path.join(root, f))


from json import load
from urllib.request import urlopen
from pprint import pprint
import time

start_time = time.time()
base_url = "https://jsonplaceholder.typicode.com/"

def get_users():
    return load(urlopen(base_url + 'users/'))

def get_all_posts():
    return load(urlopen(base_url + 'posts/'))

def get_all_comments():
    return load(urlopen(base_url + 'comments/'))

def get_user_info(userid, users):
    for user in users:
        if user.get('id') == userid:
            return user

def get_posts(userid, all_posts):
    return (post for post in all_posts if post.get('userId') == userid)

def get_comments_count(postid, all_comments):
    count = 0
    for comment in all_comments:
        if comment.get('postId') == postid:
            count += 1
    return count

users = get_users()
posts = get_all_posts()
comments = get_all_comments()

comment_counts = []
for user in users:
    user_comment_count = []
    for post in get_posts(user.get('id'), posts):
        comments_count_for_post = get_comments_count(post.get('userId'), comments)
        user_comment_count.append(comments_count_for_post)
    comment_counts.append({'userid': user.get('id'), 'commentSum': sum(user_comment_count)})

max_comment_sum = max(comment_counts, key=lambda item: item.get('commentSum'))
all_max_users = []
for comment_count in comment_counts:
    if comment_count.get('commentSum') == max_comment_sum.get('commentSum'):
        all_max_users.append(get_user_info(comment_count.get('userid'), users))
print(comment_counts)
pprint(all_max_users)
print(time.time() - start_time)


from django.urls import path
from . import views
from .views import *
from rest_framework.routers import DefaultRouter
from rest_framework.authtoken.views import obtain_auth_token

# urlpatterns = [
#     path('producto/', ProductoViewSet.as_view({'get': 'list'})),
#     path('marca/', MarcaViewSet.as_view({'get': 'list'})),
#     path('categoria/', CategoriaViewSet.as_view({'get': 'list'})),
#     path('proveedor/', ProveedorViewSet.as_view({'get': 'list'})),
#     path('stock/', StockViewSet.as_view({'get': 'list'})),
#     path('carrito/', CarritoViewSet.as_view({'get': 'list'})),
#     path('producto-agregado/', ProductoAgregadoViewSet.as_view({'get': 'list'}))
# ]

router = DefaultRouter()
router.register("producto", ProductoViewSet, basename="producto")
router.register("marca", MarcaViewSet, basename="marca")
router.register("categoria", CategoriaViewSet, basename="categoria")
router.register("proveedor", ProveedorViewSet, basename="proveedor")
router.register("stock", StockViewSet, basename="stock")
router.register("carrito", CarritoViewSet, basename="carrito")
router.register("producto-agregado", ProductoAgregadoViewSet, basename="producto-agregado")

urlpatterns = router.urls

urlpatterns += [
    path('login/', obtain_auth_token, name='get-token'),
    path('get-user/', views.get_user, name='get-user'),
    path('register-user/', views.register_user, name='register-user'),
    path('add-product/', views.add_product, name='add-product'),
    path('delete-product/', views.delete_product, name='delete-product'),
    path('purchase-cart/', views.purchase_cart, name='purchase-cart'),
]


import os
import socket
import base64
import threading
from time import sleep
from threading import Thread
#from SocketServer import ThreadingMixIn

class ClientThread(threading.Thread):
    def __init__(self, clientAddress, clientsock, file_name, server):
        threading.Thread.__init__(self)
        self.server = server
        self.csocket = clientsock
        self.caddress = clientAddress
        self.imagename = file_name
        # print("New connection added: ", clientAddress)

    def run(self):
        clientsock, clientAddress = self.server.accept()
        print("Connection from send: ", clientAddress)
        while True:
            data = self.csocket.recv(4096)
            print('Server received', data, self.imagename)
            print(self.imagename)
            filenamee = "C:\\Users\\kavya\\Desktop\\files\\test2017\\test2017\\" + self.imagename
            f = open(filenamee, 'rb')
            picture = base64.b64encode(f.read())
            l = f.read(4096)
            while (l):
                print(type(clientsock))
                clientsock.sendall(str.encode(picture))
                print('sent')
                #l = f.read(4096)
            f.close()
            print('Done sending')
            clientsock.close()
            print("Client at ", clientAddress, " disconnected...")

class recievethread(threading.Thread):
    def __init__(self, file_name, clientAddress, clientsock, server):
        threading.Thread.__init__(self)
        self.server = server
        #self.port = port1
        # self.csocket = clientsock
        # self.caddress = clientAddress
        self.file_name = file_name
        # print("New connection added: ", clientAddress)

    def run(self):
        clientsock, clientAddress = self.server.accept()
        print("Connection from recieve: ", clientAddress)
        self.csocket = clientsock
        self.caddress = clientAddress
        print("New connection added: ", clientAddress)

    # def run(self):
    #     print("Connection from : ", self.caddress)
    #     data = self.csocket.recv(2048)
    #     print(data)
    #     while True:
    #         filename = 'C:\\Users\\kavya\\Desktop\\files\\test2017\\test2017\\' + self.file_name
    #         f = open(filename, 'rb')
    #         l = f.read(4096)
    #         while (l):
    #             # print(type(self.csocket))
    #             self.csocket.sendall(bytes(l))
    #             print('ssent')
    #             l = f.read(4096)
    #         f.close()
    #         print('Done sending')
    #         #self.csocket.close()
    #         sleep(2)
        # def recieveimg(self):
        #     print("Client at ", self.caddress, " disconnected...")
        #     #self.csocket.connect((SERVER, PORT))
        datq = self.csocket.recv(4096)
        print(datq)
        with open('C:\\pynq\\Docker Toolbox\\Docker workfolder\\recievedfiles\\received_file', 'wb') as x:
                print('file opened')
                while True:
                    data = clientsock.recv(4096)
                    if not data:
                        print('if not this')
                        break
                    # write data to a file
                    x.write(data)
                print("closing client 2")
        x.close()
        clientsock.close()

class ThreadClose(threading.Thread):
    def __init__(self, clientAddress, clientsocket):
        threading.Thread.__init__(self)
        self.csocket = clientsocket
        self.caddress = clientAddress
        print("ThreadClose:New connection added: ", clientAddress)

    def run(self):
        print("ThreadClose: Connection from : ", self.caddress)
        self.csocket.send(bytes("shutdown", 'utf-8'))
        self.csocket.close()
        print("ThreadClose:Client at ", self.caddress, " disconnected...")

def main():
    counter = 0
    LOCALHOST = socket.gethostname()
    PORT = 8080
    #port1 = 8088
    server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    server.bind((LOCALHOST, PORT))
    file_name = insertdict()
    print("Server started", file_name)
    print("Waiting for client request..")
    while True:
        server.listen(1)
        clientsock, clientAddress = server.accept()
        if (counter < 5):
            newthread = ClientThread(clientAddress, clientsock, file_name, server)
            #newthread = ClientThread( server, file_name, clientAddress, clientsocket)
            print("before spwaning the thread")
            newthread.start()
            # newthread.join()
            print("done with tread exec")
            sleep(1)
            newthread = recievethread (file_name, clientAddress, clientsock, server)
            newthread.start()
            counter += 1
        else:
            threadoff = ThreadClose(clientAddress, clientsock)
            threadoff.start()
            print("closed off client request")

def insertdict():

    direc = 'C:\\Users\\kavya\\Desktop\\files\\test2017\\test2017'
    #ext = '.jpg' # Select your file delimiter
    file_dict = {}  # Create an empty dict
    txt_files = [i for i in os.listdir(direc)]
    file_dict = {txt_files[i]: 'unprocessed' for i in range(0, len(txt_files))}
    return (list(file_dict.keys())[list(file_dict.values()).index('unprocessed')])
      # returns the first value with that key

if __name__ == "__main__":
    main()


import pandas as pd
from time import sleep as sl
import numpy as np
import matplotlib.pyplot as plt
import matplotlib as mpl
from pandas import ExcelWriter

import seaborn as sns
sns.set_palette(sns.color_palette("tab20", 20))

from make_outp_excel_ready import split_hgts
mpl.rcParams['pdf.fonttype'] = 42

def data_metadata_import_merge(datafile,metafile,precombined):
    if precombined != "":
        fulldata = pd.read_excel(precombined, index_col=0)
    else:
        mgt_data = split_hgts(datafile, False)
        metadata = pd.read_excel(metafile,index_col=0, dtype=str,na_values="0")
        fulldata = pd.merge(mgt_data,metadata,left_index=True,right_index=True)

    return fulldata

# def major_st_over_time(dataframe):
#     df = pd.read_csv("/Users/michaelpayne/Documents/UNSW/OneDrive - UNSW/HGT_Paper/figures/dt104_major_ST_over_time/major_st_perc_over_time_cc.txt",sep="\t",header=0).set_index('Year')
#     df = df[4:]
#
#     plot = df.plot.line(figsize=(10,6))
#
#     # ax.legend(loc=1)
#     # plt.ylabel("Strain count")
#     # plt.xticks(df.index)
#     plot.set_yticks([0,20,40,60,80,100])
#     plot.set_xticks(range(1990,2011,2))
#     plot.set_xlim(1990,2018)
#     plot.legend(loc=0)
#     #plt.show
#
#     plot = plot.get_figure()
#     # plot.tight_layout()
#     plot.show()

def change_recursive(change_dict,string,count,no):
    if count == no:
        return string
    else:
        l = string.split("-")
        tocheck = l[count]
        tochange = count-1
        l = l[:tochange] + [change_dict[count][tocheck]] + l[tochange+1:]
        nstring = "-".join(l)
        count += 1
        return change_recursive(change_dict,nstring,count,no)

def get_inconsistent(all,scheme):

    nos = all.groupby(scheme).count()
    nos = (nos["Strain"])
    no = int(scheme[-1])

    ind_lis = list(nos.T.index)

    # print(ind_lis)
    exist = {}
    exist2 = {}
    for pos in range(1,no):
        exist[pos] = {}
        exist2[pos] = {}
        for i in ind_lis:
            l = i.split("-")
            id = l[pos]
            up = l[pos-1]
            if id not in exist[pos]:
                exist[pos][id] = [up]
            else:
                exist[pos][id].append(up)
        for i in exist[pos]:
            idlis = exist[pos][i]
            nums1 = list(set(exist[pos][i]))
            nums = [(x,idlis.count(x)) for x in nums1]
            mx = max(nums, key=lambda item: item[1])

            # exist2[pos][i] = mx[0]
            if len(nums1) > 1:
                # exist2[pos][i] = mx[0]+"*"
                exist2[pos][i] = "/".join(nums1)
            # elif len(nums1) > 1 and len(nums1) <= 3 :
            #     exist2[pos][i] = "/".join(nums1)
            else:
                exist2[pos][i] = mx[0]

    # print(exist2[1])
    conv = dict()
    convr = {}
    ind_lis2 = list(nos.T.index)
    for i in ind_lis2:
        newi = change_recursive(exist2,i,1,no)
        convr[i] = newi
        if newi in conv:
            conv[newi].append(i)
        else:
            conv[newi] = [i]
    # print(convr)
    return convr

def leekit_country(leekit,schemes,outfolder):
    for scheme in schemes:

        scheme = "id_string_"+scheme

        print(leekit.columns)
        df2 = leekit.groupby(["Location (Corrected - Country)", scheme])["Location (Corrected - Country)"].count().unstack(scheme).fillna(0)
        print(df2.columns)
        df2 = df2[df2.sum().sort_values(ascending=False).index]
        print(df2.columns)
        df2 = df2.reindex(["Austria","Germany","Denmark","Netherlands","Luxembourg","Switzerland","France","Spain","Ireland","Poland","Czech Republic","Israel","Morocco","Thailand","Taiwan","New Zealand","Canada","United States","Argentina"])
        print(df2.columns)
        # get_inconsistent(df2)

        # print(list(df2.T.index))

        ###### with singletons separate plot

        # plt = df2.plot.bar(stacked=True,figsize=(10,8),width=0.8).get_figure()
        # # plt.tight_layout()
        # plt.savefig(scheme+"_leekit_country_distribution.pdf")
        # plt.show()

        #####################

        ###### with singletons together plot
        # df2.to_excel(writer, 'leekit_country_' + scheme)
        series = df2.sum(axis=0)
        mask = (series).gt(1)

        # print(series)
        tokeep = list(series[mask].index)
        tocombine = list(series[~mask].index)

        morethanone = df2[tokeep]

        singles = pd.DataFrame(columns=["Singletons"])

        singles["Singletons"] = df2[tocombine].sum(axis=1)

        final = pd.concat([morethanone,singles],axis=1)

        plt2 = final.plot.bar(stacked=True,figsize=(10,8),width=0.8).get_figure()
        # plt2.xlabel("Country")
        # plt2.ylabel("Strain count")
        # plt2.tight_layout()

        plt2.savefig(outfolder+scheme+"_leekit_country_distribution_no1.pdf")

        ##################
def all_source(total,schemes,outfolder):
    schemes+=["MGT92","MGT95","MGT910"]
    for scheme in schemes:
        if len(scheme) == 4:
            scheme = "id_string_"+scheme

        to_keep = ["clinical", "environmental/other"]
        total = total[total["met2"].isin(to_keep)]

        to_keep = ["United Kingdom: Scotland", "United Kingdom: England", "United Kingdom: Wales"]
        total = total[total["Location (Corrected - Country)"].isin(to_keep)]

        df2 = total.groupby(['met2', scheme])['met2'].count().unstack(scheme).fillna(0)

        df2 = df2[df2.sum().sort_values(ascending=False).index]

        # df2 = df2.reindex(["Austria","Germany","Denmark","Netherlands","Luxembourg","Switzerland","France","Spain","Ireland","Poland","Czech Republic","Israel","Morocco","Thailand","Taiwan","New Zealand","Canada","United States","Argentina"])

        # get_inconsistent(df2)

        # print(list(df2.T.index))

        ###### with singletons separate plot

        # plt = df2.plot.bar(stacked=True,figsize=(10,8),width=0.8).get_figure()
        # # plt.tight_layout()
        # plt.savefig(scheme+"_leekit_country_distribution.pdf")
        # plt.show()

        #####################

        ###### with singletons together plot
        # df2.to_excel(writer, 'leekit_country_' + scheme)
        series = df2.sum(axis=0)
        mask = (series).gt(1)

        # print(series)
        tokeep = list(series[mask].index)
        tocombine = list(series[~mask].index)

        morethanone = df2[tokeep]

        singles = pd.DataFrame(columns=["Singletons"])

        singles["Singletons"] = df2[tocombine].sum(axis=1)

        final = pd.concat([morethanone,singles],axis=1)

        plt2 = final.plot.bar(stacked=True,figsize=(10,8),width=0.8).get_figure()
        # plt2.xlabel("Country")
        # plt2.ylabel("Strain count")
        # plt2.tight_layout()

        plt2.savefig(outfolder+scheme+"_with_string_source_distribution_no1.pdf")

        ##################

def mather_UK_time(mather,schemes,outfolder):
    # print(mather)
    to_keep = ["United Kingdom: Scotland","United Kingdom: England","United Kingdom: Wales"]
    mather = mather[mather["Location (Corrected - Country)"].isin(to_keep)]
    # print(mather)
    for scheme in schemes:
        scheme = "id_string_" + scheme
        df2 = mather.groupby(['Collection Year', scheme])['Collection Year'].count().unstack(scheme).fillna(0)
        # print(df2)

        # df2 = df2.reindex(df2.T.sum().sort_values().T.index,axis=1)
        df2 = df2[df2.sum().sort_values(ascending=False).index]
        # df2.to_excel(writer, 'mather_time_' + scheme)
        series = df2.sum(axis=0)
        mask = (series).gt(1)

        # print(series)
        tokeep = list(series[mask].index)
        tocombine = list(series[~mask].index)

        morethanone = df2[tokeep]

        singles = pd.DataFrame(columns=["Singletons"])

        singles["Singletons"] = df2[tocombine].sum(axis=1)

        final = pd.concat([morethanone, singles], axis=1)

        plt2 = final.plot.area(stacked=True, figsize=(8, 6),lw=0,fontsize=12).get_figure()
        # plt2.xlabel("Year")
        # plt2.ylabel("Strain count")
        plt2.tight_layout()

        plt2.savefig(outfolder+scheme + "_mather_time_UK_distribution_no1.pdf")

def major_ST_over_time(all,schemes,outfolder,name,cc_or_st):
    df = pd.DataFrame()
    c = 0
    for scheme in schemes:
        scheme = "id_string_" + scheme
        # all[['Collection Year']].apply(pd.to_numeric, errors='ignore')

        df2 = all.groupby(['Collection Year', scheme])['Collection Year'].count().unstack(scheme).fillna(0)
        # print(df2)
        # df3 = list(df2.sum(axis=0).sort_values(ascending=False).T.index)
        maxst = list(df2.sum().sort_values(ascending=False).index)[0]
        df2 = df2.div(df2.sum(axis=1), axis=0).multiply(100)

        # print(df2)
        # sl(10)

        if c == 0:
            df = df2[maxst]
        else:
            df = pd.concat([df, df2[maxst]], axis=1)

        c+=1
        # df2 = df2.reindex(df3)
    # df = df.drop(["??"])

    # print(df)

    # df.to_csv("/Users/michaelpayne/Documents/UNSW/OneDrive - UNSW/HGT_Paper/figures/dt104_major_ST_over_time/3-major_cc_perc_over_time_rerun2.txt",sep="\t")

    # df.to_excel(writer, 'major_over_time')

    # df = df.drop(df.columns[[7,8]], axis=1)
    # print(df)

    # print(df)

    plot = df.plot.line(figsize=(10, 6))

    # ax.legend(loc=1)
    # plt.ylabel("Strain count")
    # plt.xticks(df.index)
    plot.set_yticks([0, 20, 40, 60, 80, 100])
    if name == "DT104":
        plot.set_xticks(range(1990, 2011, 2))
        plot.set_xlim(1990, 2018)
    elif name == "DT160":
        plot.set_xticks(range(1998, 2012, 2))
        plot.set_xlim(1998, 2012)
    plot.legend(loc=0)
    plot = plot.get_figure()
    plot.tight_layout()
    plot.savefig(outfolder+ cc_or_st + "_" +
        name +"_major_st_perc_over_time_rerun.pdf")

        # print(df2)

def add_singletones(df): ####creating for graph, just a large block for the single STs instead of lots of little blocks.
    df = df[df.sum().sort_values(ascending=False).index]
    series = df.sum(axis=0)
    mask = (series).gt(1)

    # print(series)
    tokeep = list(series[mask].index)
    tocombine = list(series[~mask].index)

    morethanone = df[tokeep]

    singles = pd.DataFrame(columns=["<3"])

    singles["<3"] = df[tocombine].sum(axis=1)

    final = pd.concat([morethanone, singles], axis=1)

    return final

def general_dt104_mgt_summary(dt104,schemes,outfolder,name,cc_or_st):
    df = pd.DataFrame()
    c=0
    for scheme in schemes:
        # scheme = "id_string_" + scheme

        df2 = dt104[scheme].fillna(0).value_counts().to_frame().T
        # print(df2)
        final = add_singletones(df2).T

        # print(df2)
        if c==0:
            df = final
        else:
            df = pd.merge(df,final,how="outer",left_index=True,right_index=True).fillna(0)
        c+=1
    # df.to_excel(writer, 'overall')

    # print(df)
    plt2 = df.T.plot.bar(stacked=True, figsize=(10, 10), width=0.8,legend=False,fontsize=12).get_figure()
    # plt2.xlabel("Country")
    # plt2.ylabel("Strain count")
    # plt2.rcParams.update({'font.size': 12})
    plt2.tight_layout()

    plt2.savefig(outfolder+name+"_"+cc_or_st+"_counts_nostring.pdf")

hgt = "/Users/michaelpayne/Documents/UNSW/OneDrive - UNSW/HGT_Paper/data/10k_run_initial_29-8-18/MGT_stm4_hgt.txt"
meta = "/Users/michaelpayne/Documents/UNSW/OneDrive - UNSW/HGT_Paper/data/old/10krun_final_21-8-18/10k_full_metadata.xlsx"
outfolder = "/Users/michaelpayne/Documents/UNSW/OneDrive - UNSW/HGT_Paper/data/10k_run_initial_29-8-18/graphs/"

annotated_HGT = ""#"/Users/michaelpayne/Documents/UNSW/OneDrive - UNSW/HGT_Paper/data/10k_run_initial_29-8-18/MGT_stm4_initial.xlsx"#"/Users/michaelpayne/Documents/UNSW/OneDrive - UNSW/HGT_Paper/data/10krun_initial_27-8-18/10k_initial.txt"

schemes = ["MGT"+str(x) for x in range(1,10)]
schemes2 = ["ac_MGT"+str(x) for x in range(2,10)]
print(schemes)

all = data_metadata_import_merge(hgt,meta,annotated_HGT)

all[['Collection Year']] = all[['Collection Year']].replace("nan", '0', regex=True)

all[['Collection Year']] = all[['Collection Year']].astype(int)

for i in range(len(schemes)):
    nscheme = "id_string_" + schemes[i]
    # print(nscheme)
    if i == 0:
        all[nscheme] = all[schemes[i]]
        # print(all[nscheme])
    else:
        all[nscheme] = all["id_string_"+schemes[i-1]] + "-" +all[schemes[i]]
        # print(all["id_string_"+schemes[i-1]])
        # print(all[schemes[i]])
        # print(all["id_string_"+schemes[i-1]] + "-" +all[schemes[i]])
        # sl(0.3)

for i in range(len(schemes2)):
    nscheme = "id_string_" + schemes2[i]
    if i == 0:
        all[nscheme] = all[schemes2[i]]
    else:
        all[nscheme] = all["id_string_"+schemes2[i-1]] + "-" + all[schemes2[i]]
print(all.columns)

# for i in range(len(schemes)):
#     nscheme = "id_string_" + schemes[i]
#     changedict = get_inconsistent(all,nscheme)
#     all[nscheme] = all[nscheme].replace(changedict)

# print(all["id_string_MGT5"])

# print(all.describe())
mather = all.loc[all['substudy']=="mathers"]
print(mather.shape)
leekit = all.loc[all['substudy']=="leekit"]
dt104 = all.loc[all['Initial_experiment']=="DT104"]
nz = all.loc[all['Initial_experiment']=="NZ"]
# writer = ExcelWriter('/Users/michaelpayne/Documents/UNSW/OneDrive - UNSW/HGT_Paper/figures/dt104_combined.xlsx')

leekit_country(leekit,schemes,outfolder)
all_source(mather,schemes,outfolder)
schemes = ["MGT"+str(x) for x in range(1,10)]
mather_UK_time(mather,schemes,outfolder)

# sns.set_palette(sns.color_palette("tab10", 9))
#
major_ST_over_time(dt104,schemes,outfolder,"DT104","st")
major_ST_over_time(nz,schemes,outfolder,"DT160","st")
general_dt104_mgt_summary(dt104,schemes,outfolder,"DT104","st")
general_dt104_mgt_summary(nz,schemes,outfolder,"DT160","st")
# writer.save()


from seaborn import scatterplot

class Visualizacao:
    def __init__(self):
        pass

    def variavel_viz(self, variavel):
        """
        Visualizar uma variável do DF
        : param variavel: vetor/coluna do DF
        : return:
        """

        print('Visualização uma variável (NAs removidos)')
        plotaDistribuicaoUniVar(variavel.dropna())

    def missings_viz(
        self, df, visualizar=True, escolhido_tipo=None, df_missings=False
    ):
        """
        Visualizar os missings, plota o tipo de visualizacao
        : param df: pd.DataFrame para visualizar
        : param visualizar: booleano para decidir qual visualizar
        : param escolhido_tipo: inteiro para decidir qual tipo visualizar
        : param df_missings: booleano para retorna Dataframe com percentual de nulos
        : return: pd.DataFrame com nomes das colunas e porcentagem missings
        """

        if visualizar:
            # para quem usar um tema dark na IDE
            from matplotlib.pyplot import style

            style.use('classic')

            # colunas com missings apenas
            cols_miss = df.isnull().any()
            cols_miss = df.columns[cols_miss]

            if escolhido_tipo == None:
                print(
                    'Tipo de visualizacao: ',
                    '\n',
                    'total de missings - 1',
                    '\n',
                    'ordem de aparição - 2',
                    '\n',
                    'correlação - 3',
                    '\n',
                    'dendograma - 4',
                )
                escolhido_tipo = int(input())

            print('Visualização missings')
            # total
            if escolhido_tipo == 1:
                from missingno import bar

                bar(df[cols_miss])
            # ordem aparicao
            elif escolhido_tipo == 2:
                from missingno import matrix

                matrix(df[cols_miss])
            # correlacao
            elif escolhido_tipo == 3:
                from missingno import heatmap

                heatmap(df[cols_miss])
            # dendograma
            elif escolhido_tipo == 4:
                from missingno import dendrogram

                dendrogram(df[cols_miss])

        if df_missings:
            from funcoesProprias import dfExploracao

            print('Cálculo do percentual de missings num DataFrame')
            explora = dfExploracao(df)
            explora = explora.sort_values(['tipos', 'na_perct', 'quantUnicos'])
            return explora

    def correlacao_viz(self, df, colunas=None, anotado=False):
        """
        Matriz de correlação de um DataFrame
        : param df: Dataframe
        : param colunas: lista de colunas a visualizar
        : param anotado: booleano para anotar valor da correlacao
        """
        from seaborn import heatmap

        print('Visualizando correlação de Pearson (NAs removidos)')
        heatmap(df[colunas].dropna().corr(), annot=anotado)

    def regression_viz(self, y_true, y_pred, nome):
        """
        Visualize the quality of regression model
        :param y_true: pd.Series with true label values
        :param y_pred: pd.Series with predicted label values
        :param nome: Name of the file wich will be saved
        :return: Save files in specified path
        """
        residual = y_pred - y_true
        data = pd.DataFrame(
            {'pred': y_pred, 'true': y_true, 'residual': residual}
        )
        plot1 = sns.distplot(data['residual'], bins=50)
        plot2 = sns.scatterplot(x='true', y='residual', data=data)
        plt.savefig(plot1, '../data/' + nome + '_distplot.csv')
        plt.savefig(plot1, '../data/' + nome + 'scatterplot.csv')
        plt.show()

    def pca(self, pca_data, target):
        """Visualiza PCA em 2 Dimensões com Target destacado.

        :param pca_data: Numpy Array são os dados reduzidos.
        :param target: pd.Dataframe com o target.
        :return: None
        """
        scatterplot(x=pca_data[:, 0], y=pca_data[:, 1], hue=target['tem_diab'])


import os
import cv2
import numpy as np
import sklearn
import random
import utils
from keras import layers
from keras.models import Sequential
import matplotlib
import keras
#matplotlib.use("agg")
import matplotlib.pyplot as plt
from keras import backend as K

IMAGE_MEAN, IMAGE_STD = utils.load_image_stats()

def normalize_img(image):
    #return (image - 0.5)*2
    return (image - IMAGE_MEAN) / (IMAGE_STD+1e-8)

def denormalize_img(image):
    return (image * IMAGE_STD) - IMAGE_MEAN

def generator(samples, batch_size=1, augment_data=True):

    num_samples = len(samples)
    while 1: # Loop forever so the generator never terminates
        random.shuffle(samples)
        for offset in range(0, num_samples, batch_size):
            batch_samples = samples[offset:offset+batch_size]

            images = []
            y_train = []
            for batch_sample in batch_samples:
                name = batch_sample[0].split('/')[-1]
                correction = 0
                if augment_data and random.random() > 0.75: # Augment left / right camera
                    if random.random() > 0.5: # left
                        name = batch_sample[1].split('/')[-1]
                        correction = 0.2
                    else: # right
                        name = batch_sample[2].split('/')[-1]
                        correction = -0.2

                filepath = os.path.join("/opt/test/drive2", "IMG", name)
                assert os.path.isfile(filepath), "Is not path: {}".format(filepath)
                center_image = utils.read_img(filepath)
                center_angle = float(batch_sample[3]) # Steering
                center_angle += correction
                throttle = float(batch_sample[4])
                brake = float(batch_sample[5])
                speed = float(batch_sample[6])
                if augment_data and random.random() > 0.5:
                    center_image = np.fliplr(center_image)
                    center_angle = - center_angle

                images.append(center_image)
                y_train.append([center_angle, throttle])

            # trim image to only see section with road
            X_train = np.array(images)
            y_train = np.array(y_train)
            y_train = y_train[:, 0]
            #X_train = (X_train - mean) / std
            # Reshape from BGR to RGB
            #print(X_train.shape, IMAGE_MEAN.shape)
            X_train = normalize_img(X_train)
            #X_train = crop_images(X_train)
            #print(X_train.shape)
            yield sklearn.utils.shuffle(X_train, y_train)

def add_conv2d(model, filters, max_pool=False):
    model.add(layers.Conv2D(filters, 3, padding="same", activation="relu", data_format="channels_last"))
    if max_pool:
        model.add(layers.MaxPool2D(2, strides=[2,2]))
    model.add(layers.BatchNormalization())

def create_model():
    ch, row, col = 3, 160, 320  # Trimmed image format
    top_crop = row * 8 // 20
    print(top_crop)
    model = Sequential()
    local = False
    # Preprocess incoming data, centered around zero with small standard deviation
    #model.add(layers.Dropout(0, input_shape=[160, 320, 3]))
    model.add(layers.Cropping2D(cropping=((50,20), (0,0)), input_shape=[row, col, ch]))
    #model.add(layers.Lambda(lambda x: normalize_data(x)))
    if not local:
        add_conv2d(model, 32, True)
        add_conv2d(model, 32)
        add_conv2d(model, 32)
        add_conv2d(model, 32, True)
        add_conv2d(model, 64)
        add_conv2d(model, 64)
        add_conv2d(model, 64, True)
        add_conv2d(model, 128)
        add_conv2d(model, 128)
        add_conv2d(model, 128, True)
        add_conv2d(model, 256)
        model.add(layers.MaxPool2D(2, strides=[1,2]))
        add_conv2d(model, 256)
        add_conv2d(model, 256, True)
        add_conv2d(model, 512, True)

    #model.add(... finish defining the rest of your model architecture here ...)
    model.add(layers.Flatten())
    model.add(layers.Dropout(0.25))
    model.add(layers.Dense(32, activation="relu"))
    model.add(layers.BatchNormalization())
    model.add(layers.Dense(1))
    model.compile(loss='mse', optimizer=keras.optimizers.Adam(lr=0.001))
    model.summary()
    return model

if __name__ == "__main__":
    samples = utils.read_csv_file('data/driving_log.csv')
    samples += utils.read_csv_file("/opt/test/drive1/driving_log.csv")
    samples += utils.read_csv_file("/opt/test/drive2/driving_log.csv")
    samples += utils.read_csv_file("/opt/test/drive3/driving_log.csv")
    samples += utils.read_csv_file("/opt/test/drive4/driving_log.csv")

    samples = samples[1:] # remove header line

    from sklearn.model_selection import train_test_split
    train_samples, validation_samples = train_test_split(samples, test_size=0.1)

    #for i in generator(train_samples):
    #    print(i[0].shape)

    batch_size = 32
    # compile and train the model using the generator function
    train_generator = generator(train_samples, batch_size=batch_size)
    validation_generator = generator(validation_samples, batch_size=batch_size, augment_data=False)
    #imgs = next(train_generator)[0]
    #print(imgs.min(), imgs.max)
    #imgs2 = imgs - imgs.min(axis=0)
    #imgs2 = imgs2 / imgs2.max(axis=0)
    #plt.imshow(imgs2[0])
    #plt.show()
    model = create_model()
    #imgs = model.predict(imgs)
    #plt.imshow(denormalize_img( imgs[0]))
    #plt.show()
    model.load_weights("my_model_weights.h5")
    """
    model.fit_generator(train_generator,
                        samples_per_epoch=len(train_samples),
                        validation_data=validation_generator,
                        nb_val_samples=len(validation_samples),
                        nb_epoch=3)

    """
    print("Train samples:", len(train_samples))
    #If the above code throw exceptions, try

    model.fit_generator(train_generator, steps_per_epoch= len(train_samples)//batch_size,
    validation_data=validation_generator, validation_steps=len(validation_samples)//batch_size, epochs=40, verbose = 1)
    model.save("my_model.h5")
    model.save_weights("my_model_weights.h5")


# -*- encoding: UTF-8 -*-
import baostock as bs
import pandas as pd
import json
from utils import utils
import logging
import settings
import schedule
import time

from dao.redis_utils import RedisUtils

logging.basicConfig(format='%(asctime)s %(message)s', filename='sequoia.log')
logging.getLogger().setLevel(logging.INFO)

settings.init()
redisUtils = RedisUtils()

def update_pool2redis():
	# 登陆系统 ####
	lg = bs.login()
	# 显示登陆返回信息
	print('login respond error_code:' + lg.error_code + ', error_msg:' + lg.error_msg)
	dt = utils.get_recently_trade_date()
	stock_rs = bs.query_all_stock(day=dt)
	stock_df = stock_rs.get_data()
	# print(stock_df)
	stocks = stock_df.set_index('code').T.to_dict(orient='list')
	sJsonStr = json.dumps(stocks, indent=4, ensure_ascii=False).encode('utf-8')
	r = redisUtils.set("Ashare", sJsonStr)
	print("update Ashare to Redis, status:", r)
	bs.logout()

def update_stock_daily(history=False):
	try:
		et = utils.get_recently_trade_date()
		st = et
		if history:
			st = settings.START_DATE
		print(st, et)
		# 登陆系统 ####
		lg = bs.login()
		# 显示登陆返回信息
		print('login respond error_code:' + lg.error_code + ', error_msg:' + lg.error_msg)
		stock = redisUtils.get("Ashare")
		stockJson = json.loads(stock)
		for code in stockJson:
			name = stockJson[code][1]
			if "ST" in name:
				continue
			print("Downloading :" + code + " , name :" + name)
			k_rs = bs.query_history_k_data_plus(code, settings.STOCK_FIELDS, start_date=st, end_date=et)
			# print(k_rs.get_data())
			stock_records = k_rs.get_data().to_dict('records')
			for record in stock_records:
				rJsonStr = json.dumps(record, indent=4, ensure_ascii=False).encode('utf-8')
				r = redisUtils.sadd(code, rJsonStr)
			print("{} redis add finish".format(code))
		bs.logout()
	except IOError:
		print("Update Data Error ")

if __name__ == '__main__':
	# update_pool2redis()
	update_stock_daily(history=False)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
from django.conf import settings

class Migration(migrations.Migration):

    dependencies = [
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
    ]

    operations = [
        migrations.CreateModel(
            name='CampoSolicitud',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('nombre', models.CharField(max_length=60)),
                ('label', models.CharField(max_length=60)),
                ('type', models.CharField(max_length=30)),
                ('alias', models.CharField(max_length=100)),
                ('value', models.CharField(max_length=255, blank=True)),
                ('campo_local', models.CharField(max_length=100)),
                ('tabla_local', models.CharField(max_length=100)),
                ('campo_banner', models.CharField(max_length=100)),
                ('tabla_banner', models.CharField(max_length=100)),
            ],
            options={
                'verbose_name_plural': 'Campos de Solicitud',
            },
        ),
        migrations.CreateModel(
            name='Campus',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('nombre', models.CharField(max_length=30)),
            ],
        ),
        migrations.CreateModel(
            name='CanalesVenta',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('nombre', models.CharField(max_length=60)),
            ],
        ),
        migrations.CreateModel(
            name='CanalSolicitudAdmision',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('activo', models.BooleanField()),
                ('canal', models.ForeignKey(to='configurador.CanalesVenta')),
            ],
        ),
        migrations.CreateModel(
            name='ConfiguracionCanalSolicitud',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('nombre_campo', models.CharField(max_length=60)),
                ('obligatorio', models.BooleanField(default=False)),
            ],
        ),
        migrations.CreateModel(
            name='Modulo',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('nombre', models.CharField(max_length=40)),
                ('activo', models.BooleanField(default=True)),
            ],
            options={
                'verbose_name_plural': 'Modulos',
            },
        ),
        migrations.CreateModel(
            name='Programa',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('nombre', models.CharField(max_length=30)),
            ],
        ),
        migrations.CreateModel(
            name='SolicitudAdmision',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('nombre_solicitud', models.CharField(max_length=40)),
                ('habilitado', models.BooleanField(default=False)),
                ('campus', models.ForeignKey(to='configurador.Campus')),
                ('nivel', models.ForeignKey(to='configurador.Programa')),
            ],
            options={
                'verbose_name_plural': 'Solicitud de Admision',
            },
        ),
        migrations.CreateModel(
            name='SolicitudGeneral',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('nombre', models.CharField(max_length=30)),
            ],
            options={
                'verbose_name_plural': 'Solicitudes de Admisi\xf3n General',
            },
        ),
        migrations.CreateModel(
            name='TSolicitudGeneral',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('fecha_creacion', models.DateTimeField(auto_now=True)),
                ('fecha_borrado', models.DateTimeField(null=True, blank=True)),
                ('activo', models.BooleanField(default=False)),
                ('campo', models.ForeignKey(to='configurador.CampoSolicitud')),
                ('general', models.ForeignKey(to='configurador.SolicitudGeneral')),
                ('solicitud', models.ForeignKey(to='configurador.SolicitudAdmision')),
                ('usuario_responsable', models.ForeignKey(to=settings.AUTH_USER_MODEL)),
            ],
            options={
                'verbose_name_plural': 'Throgh Solicitud Admision',
            },
        ),
        migrations.AddField(
            model_name='solicitudgeneral',
            name='campo',
            field=models.ManyToManyField(to='configurador.CampoSolicitud', through='configurador.TSolicitudGeneral'),
        ),
        migrations.AddField(
            model_name='canalsolicitudadmision',
            name='configuracion',
            field=models.ForeignKey(to='configurador.ConfiguracionCanalSolicitud'),
        ),
        migrations.AddField(
            model_name='canalsolicitudadmision',
            name='modulos',
            field=models.ManyToManyField(to='configurador.Modulo'),
        ),
        migrations.AddField(
            model_name='camposolicitud',
            name='modulo',
            field=models.ForeignKey(to='configurador.Modulo'),
        ),
    ]


import os
from app import viewSet, views
from django.contrib import admin
from django.views.static import serve
from django.conf.urls import url, include
from rest_framework.routers import DefaultRouter

BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# 创建路由器并注册我们的视图。
router = DefaultRouter()
router.register(r'api/roles', viewSet.RoleViewSet)
router.register(r'api/users', viewSet.UserViewSet)
router.register(r'api/events', viewSet.EventViewSet)
router.register(r'api/bomAnas', viewSet.BomAnaViewSet)
router.register(r'api/sections', viewSet.SectionViewSet)
router.register(r'api/projects', viewSet.ProjectViewSet)
router.register(r'api/problems', viewSet.ProblemViewSet)
router.register(r'api/progresses', viewSet.ProgressesViewSet)
router.register(r'api/departments', viewSet.DepartmentViewSet)

# API URL现在由路由器自动确定。
# 另外，我们还要包含可浏览的API的登录URL。
urlpatterns = [
    url(r'^upload/', views.upload),
    url(r'^', include(router.urls)),
    url(r'^admin/', admin.site.urls),
    url(r'^api/bomAna/', views.bomAna),
    url(r'^multUpload/', views.multUpload),
    url(r'^api/bomCheck/', views.bomCheck),
    url(r'^api/updatePwd/', views.updatePwd),
    url(r'^api/loginCheck/', views.loginCheck),
    url(r'^api/exportData/', views.exportData),
    url(r'^api/materials/', views.getMaterials),
    url(r'^api/updateProject/', views.updateProject),
    url(r'^api/analyseMaterial/', views.analyseMaterial),
    url(r'^static/(?P<path>.*)$', serve,
        {'document_root': BASE_DIR+'/static/appendix'}),
    url(r'^api-auth/', include('rest_framework.urls', namespace='rest_framework'))
]