"""
    This script aims to evaluate the performance of model on CTX images, from E174°N-20° to E180°N-14°.
    The ground-truth labels were obtained from Robbins crater dataset after mapping to the CTX images.
    By running this script, Precision, recall and average IoU can be obtained.

    NOTE: The CTX tiles to be detected is in the ".CTX/cropped_images/" directory. And the ground-truth
        labels are stored in './CTX/selected_craters_XY.txt'. Both of tiles and labels can be generated
        after running 'CTX_pre-processing.py'.
"""

import numpy as np
import torch
from yolo import YOLO
from PIL import Image
import os
from utils.utils import non_max_suppression,letterbox_image,yolo_correct_boxes
import cv2
import argparse

def arg_parse():
    """
    Parse arguments to detect craters on CTX images
    """
    parser = argparse.ArgumentParser(description='evaluating on CTX images')
    parser.add_argument("--min", type=float, default=1.0, help="the size of the smallest crater to be detected")
    parser.add_argument("--max", type=float, default=32.0, help="the size of the largest crater to be detected")
    parser.add_argument("--resized_size", type=int, default=1248, help="the size of resized CTX images")

    return parser.parse_args()

def size_filter(prediction, min, max, resized_size):
    """
    This function is designed for selecting the detected craters of a certain range.

    Parameters
    ----------
    prediction: np.array, 2-d array of prediction to be filtered
    min: float or int, the lower limit of the size range
    max: float or int, the higher limit of the size range
    resized_size: int, the size of resized CTX images

    Returns
    ----------
    prediction: np.array, the filtered array with predictions in the size range
    """
    # 23710 is the size of original images, 0.005 means the resolution of original images is 5 m/pixel
    prediction = prediction[((prediction[:, 3] - prediction[:, 1]) + \
                            (prediction[:, 2] - prediction[:, 0])) / 2 > (min/(23710 * 0.005 /resized_size))]
    prediction = prediction[((prediction[:, 3] - prediction[:, 1]) + \
                            (prediction[:, 2] - prediction[:, 0])) / 2 < (max/(23710 * 0.005 /resized_size))]
    return prediction

def bbox_iou(box1, box2):
    """
    This function is referenced and modified from:
        https://github.com/eriklindernoren/PyTorch-YOLOv3/blob/master/utils/utils.py with GNU GENERAL PUBLIC LICENSE.

    This function for calculating IoU is widely used by projects relating to object detection.
    I modified it as my own version which can best fit this evaluation process.

    Parameters
    ----------
    box1, box2: 2-D tensor, box1:(Tensor[N, 4]), box2:(Tensor[M, 4]).
                Representing the a collection of bboxes with xyxy format.

    Returns
    ----------
    iou_all: np.array, 2-D numpy array with shape of (len(box1), len(box2)). iou_all[i, j] represents the IoU between
            box1[i] and box2[j].
    """

    b1_x1, b1_y1, b1_x2, b1_y2 = box1[:, 0], box1[:, 1], box1[:, 2], box1[:, 3]
    b2_x1, b2_y1, b2_x2, b2_y2 = box2[:, 0], box2[:, 1], box2[:, 2], box2[:, 3]

    for i in range(len(box1)):
        inter_rect_x1 = torch.max(b1_x1[i], b2_x1)
        inter_rect_y1 = torch.max(b1_y1[i], b2_y1)
        inter_rect_x2 = torch.min(b1_x2[i], b2_x2)
        inter_rect_y2 = torch.min(b1_y2[i], b2_y2)

        for j in range(len(box2)):
            inter_area = torch.clamp(inter_rect_x2[j] - inter_rect_x1[j] + 1, min=0) * \
                        torch.clamp(inter_rect_y2[j] - inter_rect_y1[j] + 1, min=0)

            b1_area = (b1_x2[i] - b1_x1[i] + 1) * (b1_y2[i] - b1_y1[i] + 1)
            b2_area = (b2_x2[j] - b2_x1[j] + 1) * (b2_y2[j] - b2_y1[j] + 1)

            iou = inter_area / (b1_area + b2_area - inter_area + 1e-16)

            if j == 0:
                iou_i = np.array([[iou.item()]])
            else:
                iou_i = np.append(iou_i, iou.item())

        if i == 0:
            iou_all = np.array([iou_i])
        else:
            iou_all = np.concatenate((iou_all, [iou_i]), axis=0)

    return iou_all

class ctx_Eval(YOLO):
    def ctx_evaluate(self, image):
        """
        Obtaining the array of predictions for CTX images.

        Parameters
        ----------
        image: PIL.open(image_path), the 'image_path" is the path of the ctx images.

        Returns
        ----------
        pred[:, 2:]: np.array, the array of all the predictions, but each prediction only contains the coordinates of
                    top-left and bottom-right corners of the prediction bounding box.
        """

        self.confidence = 0.5**8
        image_shape = np.array(np.shape(image)[0:2])

        crop_img = np.array(letterbox_image(image, (self.model_image_size[0], self.model_image_size[1])))
        photo = np.array(crop_img, dtype=np.float32)
        photo /= 255.0
        photo = np.transpose(photo, (2, 0, 1))
        photo = photo.astype(np.float32)
        images = []
        images.append(photo)

        images = np.asarray(images)
        images = torch.from_numpy(images)
        if self.cuda:
            images = images.cuda()

        with torch.no_grad():
            outputs = self.net(images)
            output_list = []
            for i in range(3):
                output_list.append(self.yolo_decodes[i](outputs[i]))
            output = torch.cat(output_list, 1)
            batch_detections = non_max_suppression(output, self.config["yolov3"]["classes_num"],
                                                   conf_thres=self.confidence,
                                                   nms_thres=0.1)
        batch_detections = batch_detections[0].cpu().numpy()

        top_index = batch_detections[:, 4] * batch_detections[:, 5] > self.confidence
        top_conf = batch_detections[top_index, 4] * batch_detections[top_index, 5]
        top_label = np.array(batch_detections[top_index, -1], np.int32)
        top_bboxes = np.array(batch_detections[top_index, :4])

        top_xmin, top_ymin, top_xmax, top_ymax = \
            np.expand_dims(top_bboxes[:, 0], -1), np.expand_dims(top_bboxes[:, 1], -1), \
            np.expand_dims(top_bboxes[:, 2], -1), np.expand_dims(top_bboxes[:, 3], -1)

        boxes = yolo_correct_boxes(top_ymin, top_xmin, top_ymax, top_xmax,
                                   np.array([self.model_image_size[0], self.model_image_size[1]]), image_shape)

        for i, c in enumerate(top_label):
            predicted_class = self.class_names[c]
            score = float(top_conf[i])
            score = float('%.4f' % score)

            top, left, bottom, right = boxes[i]
            if i == 0:
                pred = np.array([[int(predicted_class), score, int(left), int(top), int(right), int(bottom)]])
            else:
                pred_i = np.array([[int(predicted_class), score, int(left), int(top), int(right), int(bottom)]])
                pred = np.concatenate((pred, pred_i), axis=0)

        return pred[:, 2:]

if __name__ == "__main__":
    args = arg_parse()
    ctx_eval = ctx_Eval()
    path = "./CTX/cropped_images/"
    imgs = os.listdir(path)

    prediction_all = np.array([[]])
    for img in imgs:
        if not img.startswith('.'):
            image = Image.open(path + img)
            prediction = ctx_eval.ctx_evaluate(image)
            # get the position of each tile in panoramic view from the filename of each tile
            row = int(img[12:14])
            column = int(img[15:17])
            # covert the coordinates to fit the panoramic view
            prediction[:, 0] = prediction[:, 0] + 416 * column
            prediction[:, 2] = prediction[:, 2] + 416 * column
            prediction[:, 1] = prediction[:, 1] + 416 * row
            prediction[:, 3] = prediction[:, 3] + 416 * row
            # combine all the predictions from each tile as a whole
            prediction_all = np.append(prediction_all, prediction)

    prediction_all = np.reshape(prediction_all, (-1, 4))
    # filter out the detections out of the size range
    prediction_all = size_filter(prediction_all, args.min, args.max, args.resized_size)
    np.savetxt("./CTX/predictions.txt", prediction_all, fmt='%f', delimiter=' ')

    # draw the prediction bounding boxes on the panoramic view for visualization
    image = cv2.imread('./CTX/gt_stacked_CTX.png')
    for i in range(len(prediction_all)):
        x1 = prediction_all[i, 0]
        y1 = prediction_all[i, 1]
        x2 = prediction_all[i, 2]
        y2 = prediction_all[i, 3]
        cv2.rectangle(image, (int(x1), int(y1)), (int(x2), int(y2)), (0, 0, 255), 2)
    cv2.imwrite("./CTX/detections.png", image)

    # get the ground-truth labels
    robbins_path = "./CTX/selected_craters_XY.txt"
    f = open(robbins_path, 'r')
    line = f.readline()
    cnt = 0
    while line:
        a = line.split()
        if cnt == 0:
            targets = np.array([[float(a[0]), float(a[1]), float(a[2]), float(a[3])]])
        else:
            target_i = np.array([[float(a[0]), float(a[1]), float(a[2]), float(a[3])]])
            targets = np.concatenate((targets, target_i), axis=0)
        cnt += 1
        line = f.readline()

    print("Please wait. Evaluating...")
    # the IoU threshold for TP detection on the CTX images,
    # we use 0.2 and the reason is mentioned in the 5.2.1 of the report
    Iou_thres = 0.2
    target_num = len(targets)
    pred_num = len(prediction_all)
    prediction_all = torch.from_numpy(prediction_all).type(torch.LongTensor)
    targets = torch.from_numpy(targets).type(torch.LongTensor)
    iou = bbox_iou(targets, prediction_all)
    iou_valid = iou[iou > Iou_thres]
    tp_pred = len(iou_valid)

    # print the result
    print("The number of targets: ", target_num)
    print("The number of predictions: ", pred_num)
    print("The number of TP predictions: ", tp_pred)
    print("Average IoU: ", "%.2f%%" % (np.sum(iou_valid) / len(iou_valid) * 100))
    print("Precision: ", "%.2f%%" % (tp_pred / pred_num * 100))
    print("Recall: ", "%.2f%%" % (tp_pred / target_num * 100))


import os
import testinfra.utils.ansible_runner
import pytest
import pytest_rpc.helpers as helpers

"""ASC-298: Verify all swift rings are mounted using swift-recon tool

See RPC 10+ Post-Deployment QC process document
"""

testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('os-infra_hosts')[:1]

@pytest.mark.test_id('4b0691b8-8b9e-11e8-8fb0-a860b622fd2c')
@pytest.mark.jira('ASC-298')
def test_verify_swift_ring_mounted(host):
    """Verify all swift rings are mounted using swift-recon.
    """

    result = helpers.run_on_swift('swift-recon --unmounted', host)
    swift_data = helpers.parse_swift_recon(result.stdout)

    assert len(swift_data) > 1, "swift-recon did not return expected data"

    # assert no output found for unmounted drives
    assert len(swift_data[1]) == 1, ('Unmounted swift drives found:\n{}'
                                     .format('\n'.join(swift_data[1])))


from django.contrib.auth import get_user_model
from django.utils.translation import gettext_lazy as _
from rest_framework import serializers

from .fields import MarkdownField
from .models import Project, Repository, Task
from .validators import CaseInsensitiveUniqueTogetherValidator

User = get_user_model()

class HashidPrimaryKeyRelatedField(serializers.PrimaryKeyRelatedField):
    def to_representation(self, value):
        if self.pk_field is not None:
            return self.pk_field.to_representation(value.pk)
        return str(value.pk)

class FullUserSerializer(serializers.ModelSerializer):
    id = serializers.CharField(read_only=True)

    class Meta:
        model = User
        fields = (
            "id",
            "username",
            "email",
            "is_staff",
            "valid_token_for",
            "org_name",
            "org_type",
            "is_devhub_enabled",
            "sf_username",
        )

class MinimalUserSerializer(serializers.ModelSerializer):
    id = serializers.CharField(read_only=True)

    class Meta:
        model = User
        fields = ("id", "username")

class RepositorySerializer(serializers.ModelSerializer):
    id = serializers.CharField(read_only=True)
    description = MarkdownField(allow_blank=True)

    class Meta:
        model = Repository
        fields = (
            "id",
            "name",
            "repo_url",
            "description",
            "is_managed",
            "slug",
            "old_slugs",
        )

class ProjectSerializer(serializers.ModelSerializer):
    id = serializers.CharField(read_only=True)
    description = MarkdownField(allow_blank=True)
    repository = serializers.PrimaryKeyRelatedField(
        queryset=Repository.objects.all(), pk_field=serializers.CharField()
    )
    branch_url = serializers.SerializerMethodField()

    class Meta:
        model = Project
        fields = (
            "id",
            "name",
            "description",
            "slug",
            "old_slugs",
            "repository",
            "branch_url",
        )
        validators = (
            CaseInsensitiveUniqueTogetherValidator(
                queryset=Project.objects.all(),
                fields=("name", "repository"),
                message=_("A project with this name already exists."),
            ),
        )

    def get_branch_url(self, obj):
        return f"{obj.repository.repo_url}/tree/{obj.branch_name}"

class TaskSerializer(serializers.ModelSerializer):
    id = serializers.CharField(read_only=True)
    description = MarkdownField(allow_blank=True)
    project = serializers.PrimaryKeyRelatedField(
        queryset=Project.objects.all(), pk_field=serializers.CharField()
    )
    assignee = HashidPrimaryKeyRelatedField(
        queryset=User.objects.all(), allow_null=True
    )

    class Meta:
        model = Task
        fields = (
            "id",
            "name",
            "description",
            "project",
            "assignee",
            "slug",
            "old_slugs",
        )
        validators = (
            CaseInsensitiveUniqueTogetherValidator(
                queryset=Task.objects.all(),
                fields=("name", "project"),
                message=_("A task with this name already exists."),
            ),
        )


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt  # 绘图库
import matplotlib as mpl

def set_plot_param():
    """Set my own customized plotting parameters"""

    mpl.rc('axes', edgecolor='black')
    mpl.rc('axes', labelcolor='dimgrey')
    mpl.rc('xtick', color='dimgrey')
    mpl.rc('xtick', labelsize=16)
    mpl.rc('ytick', color='dimgrey')
    mpl.rc('ytick', labelsize=16)
    mpl.rc('axes', titlesize=18)
    mpl.rc('axes', labelsize=16)
    mpl.rc('legend', fontsize='large')
    mpl.rc('text', color='dimgrey')

csvframe_noi = pd.read_csv("NOI_final.csv",header=None)
csvframe_ctl = pd.read_csv("CTL_final.csv",header=None)
csvframe_irr = pd.read_csv("IRR_final.csv",header=None)
array_noi = np.array(csvframe_noi)
array_ctl = np.array(csvframe_ctl)
array_irr = np.array(csvframe_irr)

LHF_noi = array_noi[:,0]
LHF_ctl = array_ctl[:,0]
LHF_irr = array_irr[:,0]
LHF_ctl_noi = LHF_ctl - LHF_noi
LHF_irr_noi = LHF_irr - LHF_noi
LHF_irr_ctl = LHF_irr - LHF_ctl

SHF_noi = array_noi[:,1]
SHF_ctl = array_ctl[:,1]
SHF_irr = array_irr[:,1]
SHF_ctl_noi = SHF_ctl - SHF_noi
SHF_irr_noi = SHF_irr - SHF_noi
SHF_irr_ctl = SHF_irr - SHF_ctl

LW_noi = array_noi[:,2]
LW_ctl = array_ctl[:,2]
LW_irr = array_irr[:,2]
LW_ctl_noi = LW_ctl - LW_noi
LW_irr_noi = LW_irr - LW_noi
LW_irr_ctl = LW_irr - LW_ctl

SW_noi = array_noi[:,3]
SW_ctl = array_ctl[:,3]
SW_irr = array_irr[:,3]
SW_ctl_noi = SW_ctl - SW_noi
SW_irr_noi = SW_irr - SW_noi
SW_irr_ctl = SW_irr - SW_ctl

E_noi = array_noi[:,4] / 28.94 * 365
E_ctl = array_ctl[:,4] / 28.94 * 365
E_irr = array_irr[:,4] / 28.94 * 365
E_ctl_noi = E_ctl - E_noi
E_irr_noi = E_irr - E_noi
E_irr_ctl = E_irr - E_ctl

T_noi = array_noi[:,5] * 31536000
T_ctl = array_ctl[:,5] * 31536000
T_irr = array_irr[:,5] * 31536000
T_ctl_noi = T_ctl - T_noi
T_irr_noi = T_irr - T_noi
T_irr_ctl = T_irr - T_ctl

R_noi = array_noi[:,6] * 31536000
R_ctl = array_ctl[:,6] * 31536000
R_irr = array_irr[:,6] * 31536000
R_ctl_noi = R_ctl - R_noi
R_irr_noi = R_irr - R_noi
R_irr_ctl = R_irr - R_ctl

SM_noi = array_noi[:,7]
SM_ctl = array_ctl[:,7]
SM_irr = array_irr[:,7]
SM_ctl_noi = SM_ctl - SM_noi
SM_irr_noi = SM_irr - SM_noi
SM_irr_ctl = SM_irr - SM_ctl

plot_LHF_ctl_noi = [LHF_ctl_noi[0], LHF_ctl_noi[7], LHF_ctl_noi[20], LHF_ctl_noi[36], LHF_ctl_noi[38], LHF_ctl_noi[39]]
plot_LHF_irr_noi = [LHF_irr_noi[0], LHF_irr_noi[7], LHF_irr_noi[20], LHF_irr_noi[36], LHF_irr_noi[38], LHF_irr_noi[39]]
plot_LHF_irr_ctl = [LHF_irr_ctl[0], LHF_irr_ctl[7], LHF_irr_ctl[20], LHF_irr_ctl[36], LHF_irr_ctl[38], LHF_irr_ctl[39]]

plot_SHF_ctl_noi = [SHF_ctl_noi[0], SHF_ctl_noi[7], SHF_ctl_noi[20], SHF_ctl_noi[36], SHF_ctl_noi[38], SHF_ctl_noi[39]]
plot_SHF_irr_noi = [SHF_irr_noi[0], SHF_irr_noi[7], SHF_irr_noi[20], SHF_irr_noi[36], SHF_irr_noi[38], SHF_irr_noi[39]]
plot_SHF_irr_ctl = [SHF_irr_ctl[0], SHF_irr_ctl[7], SHF_irr_ctl[20], SHF_irr_ctl[36], SHF_irr_ctl[38], SHF_irr_ctl[39]]

plot_LW_ctl_noi = [LW_ctl_noi[0], LW_ctl_noi[7], LW_ctl_noi[20], LW_ctl_noi[36], LW_ctl_noi[38], LW_ctl_noi[39]]
plot_LW_irr_noi = [LW_irr_noi[0], LW_irr_noi[7], LW_irr_noi[20], LW_irr_noi[36], LW_irr_noi[38], LW_irr_noi[39]]
plot_LW_irr_ctl = [LW_irr_ctl[0], LW_irr_ctl[7], LW_irr_ctl[20], LW_irr_ctl[36], LW_irr_ctl[38], LW_irr_ctl[39]]

plot_SW_ctl_noi = [SW_ctl_noi[0], SW_ctl_noi[7], SW_ctl_noi[20], SW_ctl_noi[36], SW_ctl_noi[38], SW_ctl_noi[39]]
plot_SW_irr_noi = [SW_irr_noi[0], SW_irr_noi[7], SW_irr_noi[20], SW_irr_noi[36], SW_irr_noi[38], SW_irr_noi[39]]
plot_SW_irr_ctl = [SW_irr_ctl[0], SW_irr_ctl[7], SW_irr_ctl[20], SW_irr_ctl[36], SW_irr_ctl[38], SW_irr_ctl[39]]

plot_E_ctl_noi = [E_ctl_noi[0], E_ctl_noi[7], E_ctl_noi[20], E_ctl_noi[36], E_ctl_noi[38], E_ctl_noi[39]]
plot_E_irr_noi = [E_irr_noi[0], E_irr_noi[7], E_irr_noi[20], E_irr_noi[36], E_irr_noi[38], E_irr_noi[39]]
plot_E_irr_ctl = [E_irr_ctl[0], E_irr_ctl[7], E_irr_ctl[20], E_irr_ctl[36], E_irr_ctl[38], E_irr_ctl[39]]

plot_T_ctl_noi = [T_ctl_noi[0], T_ctl_noi[7], T_ctl_noi[20], T_ctl_noi[36], T_ctl_noi[38], T_ctl_noi[39]]
plot_T_irr_noi = [T_irr_noi[0], T_irr_noi[7], T_irr_noi[20], T_irr_noi[36], T_irr_noi[38], T_irr_noi[39]]
plot_T_irr_ctl = [T_irr_ctl[0], T_irr_ctl[7], T_irr_ctl[20], T_irr_ctl[36], T_irr_ctl[38], T_irr_ctl[39]]

plot_R_ctl_noi = [R_ctl_noi[0], R_ctl_noi[7], R_ctl_noi[20], R_ctl_noi[36], R_ctl_noi[38], R_ctl_noi[39]]
plot_R_irr_noi = [R_irr_noi[0], R_irr_noi[7], R_irr_noi[20], R_irr_noi[36], R_irr_noi[38], R_irr_noi[39]]
plot_R_irr_ctl = [R_irr_ctl[0], R_irr_ctl[7], R_irr_ctl[20], R_irr_ctl[36], R_irr_ctl[38], R_irr_ctl[39]]

plot_SM_ctl_noi = [SM_ctl_noi[0], SM_ctl_noi[7], SM_ctl_noi[20], SM_ctl_noi[36], SM_ctl_noi[38], SM_ctl_noi[39]]
plot_SM_irr_noi = [SM_irr_noi[0], SM_irr_noi[7], SM_irr_noi[20], SM_irr_noi[36], SM_irr_noi[38], SM_irr_noi[39]]
plot_SM_irr_ctl = [SM_irr_ctl[0], SM_irr_ctl[7], SM_irr_ctl[20], SM_irr_ctl[36], SM_irr_ctl[38], SM_irr_ctl[39]]

fig = plt.figure(figsize=[18,22], dpi=1000)
fig.subplots_adjust(hspace=0.2, wspace=0.2, left = 0.1, right = 0.95, top = 0.95, bottom = 0.05)
set_plot_param()
ax1 = fig.add_subplot(3,2,2)
ax1.text(0.01,1.02,'b',color='dimgrey',fontsize=20, transform=ax1.transAxes, weight = 'bold')
total_width, n = 0.8, 4
width = total_width / n
x = list(range(6))
for i in range(len(x)):
    x[i] = x[i] -0.4
plt.bar(x, plot_LHF_ctl_noi, width=width, label='LHF',fc = 'tomato')
for i in range(len(x)):
    x[i] = x[i] + width
plt.bar(x, plot_SHF_ctl_noi, width=width, label='SHF',fc = 'dodgerblue')
for i in range(len(x)):
    x[i] = x[i] + width
plt.bar(x, plot_SW_ctl_noi, width=width, label='SWup',fc = 'chartreuse')
for i in range(len(x)):
    x[i] = x[i] + width
plt.bar(x, plot_LW_ctl_noi, width=width, label='LWup',fc = 'orange')
plt.xticks([0, 1, 2, 3, 4, 5 ], [r'GLO', r'NCA', r'MED', r'EAS', r'SAS', r'SEA'], fontsize=20)
plt.yticks(fontsize=20)
plt.ylim((-4, 7))
plt.legend(loc='upper left', fontsize=20,frameon=False)
ax1.set_title('CTL - NOI',loc='right',fontsize = 22)
plt.ylabel(r"Changes ($\mathregular{W/m^2}$)", fontsize=20)

ax1 = fig.add_subplot(3,2,1)
ax1.text(0.01,1.02,'a',color='dimgrey',fontsize=20, transform=ax1.transAxes, weight = 'bold')
total_width, n = 0.8, 4
width = total_width / n
x = list(range(6))
for i in range(len(x)):
    x[i] = x[i] -0.4
plt.bar(x, plot_E_ctl_noi, width=width, label='E',fc = 'tomato')
for i in range(len(x)):
    x[i] = x[i] + width
plt.bar(x, plot_T_ctl_noi, width=width, label='TR',fc = 'dodgerblue')
for i in range(len(x)):
    x[i] = x[i] + width
plt.bar(x, plot_R_ctl_noi, width=width, label='R',fc = 'chartreuse')
for i in range(len(x)):
    x[i] = x[i] + width
plt.bar(x, plot_SM_ctl_noi, width=width, label='TSW',fc = 'orange')
plt.xticks([0, 1, 2, 3, 4, 5 ], [r'GLO', r'NCA', r'MED', r'EAS', r'SAS', r'SEA'], fontsize=20)
plt.yticks(fontsize=20)
plt.ylim((-90, 100))
plt.legend(loc='upper left', fontsize=20,frameon=False)
ax1.set_title('CTL - NOI',loc='right',fontsize = 22)
plt.ylabel(r"Changes (mm/year) / ($\mathregular{kg/m^2}$)", fontsize=20)

ax1 = fig.add_subplot(3,2,4)
ax1.text(0.01,1.02,'d',color='dimgrey',fontsize=20, transform=ax1.transAxes, weight = 'bold')
total_width, n = 0.8, 4
width = total_width / n
x = list(range(6))
for i in range(len(x)):
    x[i] = x[i] -0.4
plt.bar(x, plot_LHF_irr_noi, width=width, label='LHF',fc = 'tomato')
for i in range(len(x)):
    x[i] = x[i] + width
plt.bar(x, plot_SHF_irr_noi, width=width, label='SHF',fc = 'dodgerblue')
for i in range(len(x)):
    x[i] = x[i] + width
plt.bar(x, plot_SW_irr_noi, width=width, label='SWup',fc = 'chartreuse')
for i in range(len(x)):
    x[i] = x[i] + width
plt.bar(x, plot_LW_irr_noi, width=width, label='LWup',fc = 'orange')
plt.xticks([0, 1, 2, 3, 4, 5 ], [r'GLO', r'NCA', r'MED', r'EAS', r'SAS', r'SEA'], fontsize=20)
plt.yticks(fontsize=20)
plt.ylim((-4, 7))
plt.legend(loc='upper left', fontsize=20,frameon=False)
ax1.set_title('IRR - NOI',loc='right',fontsize = 22)
plt.ylabel(r"Changes ($\mathregular{W/m^2}$)", fontsize=20)

ax1 = fig.add_subplot(3,2,6)
ax1.text(0.01,1.02,'f',color='dimgrey',fontsize=20, transform=ax1.transAxes, weight = 'bold')
total_width, n = 0.8, 4
width = total_width / n
x = list(range(6))
for i in range(len(x)):
    x[i] = x[i] -0.4
plt.bar(x, plot_LHF_irr_ctl, width=width, label='LHF',fc = 'tomato')
for i in range(len(x)):
    x[i] = x[i] + width
plt.bar(x, plot_SHF_irr_ctl, width=width, label='SHF',fc = 'dodgerblue')
for i in range(len(x)):
    x[i] = x[i] + width
plt.bar(x, plot_SW_irr_ctl, width=width, label='SWup',fc = 'chartreuse')
for i in range(len(x)):
    x[i] = x[i] + width
plt.bar(x, plot_LW_irr_ctl, width=width, label='LWup',fc = 'orange')
plt.xticks([0, 1, 2, 3, 4, 5 ], [r'GLO', r'NCA', r'MED', r'EAS', r'SAS', r'SEA'], fontsize=20)
plt.yticks(fontsize=20)
plt.ylim((-4, 7))
plt.legend(loc='upper left', fontsize=20,frameon=False)
ax1.set_title('IRR - CTL',loc='right',fontsize = 22)
plt.ylabel(r"Changes ($\mathregular{W/m^2}$)", fontsize=20)

ax1 = fig.add_subplot(3,2,3)
ax1.text(0.01,1.02,'c',color='dimgrey',fontsize=20, transform=ax1.transAxes, weight = 'bold')
total_width, n = 0.8, 4
width = total_width / n
x = list(range(6))
for i in range(len(x)):
    x[i] = x[i] -0.4
plt.bar(x, plot_E_irr_noi, width=width, label='E',fc = 'tomato')
for i in range(len(x)):
    x[i] = x[i] + width
plt.bar(x, plot_T_irr_noi, width=width, label='TR',fc = 'dodgerblue')
for i in range(len(x)):
    x[i] = x[i] + width
plt.bar(x, plot_R_irr_noi, width=width, label='R',fc = 'chartreuse')
for i in range(len(x)):
    x[i] = x[i] + width
plt.bar(x, plot_SM_irr_noi, width=width, label='TSW',fc = 'orange')
plt.xticks([0, 1, 2, 3, 4, 5 ], [r'GLO', r'NCA', r'MED', r'EAS', r'SAS', r'SEA'], fontsize=20)
plt.yticks(fontsize=20)
plt.ylim((-90, 100))
plt.legend(loc='upper left', fontsize=20,frameon=False)
ax1.set_title('IRR - NOI',loc='right',fontsize = 22)
plt.ylabel(r"Changes (mm/year) / ($\mathregular{kg/m^2}$)", fontsize=20)

ax1 = fig.add_subplot(3,2,5)
ax1.text(0.01,1.02,'e',color='dimgrey',fontsize=20, transform=ax1.transAxes, weight = 'bold')
total_width, n = 0.8, 4
width = total_width / n
x = list(range(6))
for i in range(len(x)):
    x[i] = x[i] -0.4
plt.bar(x, plot_E_irr_ctl, width=width, label='E',fc = 'tomato')
for i in range(len(x)):
    x[i] = x[i] + width
plt.bar(x, plot_T_irr_ctl, width=width, label='TR',fc = 'dodgerblue')
for i in range(len(x)):
    x[i] = x[i] + width
plt.bar(x, plot_R_irr_ctl, width=width, label='R',fc = 'chartreuse')
for i in range(len(x)):
    x[i] = x[i] + width
plt.bar(x, plot_SM_irr_ctl, width=width, label='TSW',fc = 'orange')
plt.xticks([0, 1, 2, 3, 4, 5 ], [r'GLO', r'NCA', r'MED', r'EAS', r'SAS', r'SEA'], fontsize=20)
plt.yticks(fontsize=20)
plt.ylim((-90, 100))
plt.legend(loc='upper left', fontsize=20,frameon=False)
ax1.set_title('IRR - CTL',loc='right',fontsize = 22)
plt.ylabel(r"Changes (mm/year) / ($\mathregular{kg/m^2}$)", fontsize=20)

plt.savefig('Region-impacts.png')
#plt.show()
print('test')


from django.urls import path

from . import views,deploy,chart,attendance,penality

urlpatterns = [
    path('showpenality/', penality.showpenality, name="showpenality"),
    path('takepenality/',  penality.takepenality, name="takepenality"),

    path('showattendance/', attendance.showattendance, name="showattendance"),
    path('takeattendance/', attendance.takeattendance, name="takeattendance"),

    path('report', chart.report, name='report'),
    path('report-chart/', chart.report_chart, name='report-chart'),
    path('', views.index, name='index'),

    path('vehicle_location', views.vehicle_location,name='vehicle_location'),
    path('location/<str:mid>', views.location,name='location'),

    path('deployment', deploy.deploy,name="deploy"),
    path('driver/register', views.driver_register, name='driver_register'),
    path('driver/new', views.driver_register_save, name='driver_save'),
    path('driver/', views.driver_list, name='driver_list'),

    path('subcity/register', views.subcity_register, name='subcity_register'),
    path('subcity/new', views.subcity_save, name='subcity_save'),
    path('subcity/', views.subcity_list, name='subcity_list'),
    path('subcity/edit/<int:id>', views.subcity_edit,name='subcityEdit'),
    path('subcity/update/<int:id>', views.subcity_update,name='subcityUpdate'),
    path('subcity/delete/<int:id>', views.subcity_destroy,name='subcityDelete'),

    path('station/register', views.station_register, name='station_register'),
    path('station/new', views.station_save, name='station_save'),
    path('station/', views.station_list, name='station_list'),
    path('station/edit/<int:id>', views.station_edit,name='stationEdit'),
    path('station/update/<int:id>', views.station_update,name='stationUpdate'),
    path('station/delete/<int:id>', views.station_destroy,name='stationDelete'),

    path('route/register', views.route_register, name='route_register'),
    path('route/new', views.route_save, name='route_save'),
    path('route/', views.route_list, name='route_list'),
    path('route/edit/<int:id>', views.route_edit,name='routeEdit'),
    path('route/update/<int:id>', views.route_update,name='routeUpdate'),
    path('route/delete/<int:id>', views.route_destroy,name='routeDelete'),

    path('vehicle/register', views.vehicle_register, name='vehicle_register'),
    path('vehicle/new', views.vehicle_save, name='vehicle_save'),
    path('vehicle/', views.vehicle_list, name='vehicle_list'),
    path('Assigned_vehicle_list/', views.Assigned_vehicle_list, name='Assigned_vehicle_list'),
    path('vehicle/edit/<int:id>', views.vehicle_edit,name='vehicleEdit'),
    path('vehicle/update/<int:id>', views.vehicle_update,name='vehicleUpdate'),
    path('vehicle/delete/<int:id>', views.vehicle_destroy,name='vehicleDelete'),

    path('machine/register', views.machine_register, name='machine_register'),
    path('machine/new', views.machine_save, name='machine_save'),
    path('machine/', views.machine_list, name='machine_list'),
    path('machine/edit/<int:id>', views.machine_edit,name='machineEdit'),
    path('machine/update/<int:id>', views.machine_update,name='machineUpdate'),
    path('machine/delete/<int:id>', views.machine_destroy,name='machineDelete'),
    path('machine/data', views.machine_data_list, name='machine_data_list'),

]


from django.urls import path
from . import views
# app_name = 'library_api'
urlpatterns = [
    path('', views.home, name='home'),
    path('login/', views.login, name='login'),
    path('logout/', views.logout, name='logout'),
    path('register/', views.register, name='register'),
    path('all_books/', views.all_books, name='all_books'),
    path('create_record/', views.create_book_record, name='create_book_record'),
    path('update_record/<pk>', views.update_book_record, name='update_book_record'),
    path('delete_record/<pk>', views.delete_record, name='delete_record'),
    path('author-records/', views.author_books, name='author_books'),
]


from . import inventory
from .. models import Inventory
from flask import render_template
from flask_login import login_required
from ..utils import helpers, constants

@inventory.route(constants.INVENTORY_URL, methods=[constants.GET, constants.POST])
@login_required
def list_inventory():
    """
    List inventory
    """
    # Initialised the list to resolve following UnboundLocalError
    # UnboundLocalError: local variable 'inventory_list' referenced before assignment
    inventory_list = []
    helpers.check_manager()
    try:
        inventory_list = helpers.get_item_list(Inventory)
    except Exception as e:
        helpers.notify_client(str(e))

    return render_template(constants.INVENTORY_HTML,
                           inventory=inventory_list,
                           title=constants.INVENTORY)


import sys
import numpy as np
import torch
from torch import nn
from torch.autograd import Variable
from torchvision.datasets import CIFAR10
import torchvision.transforms as transforms

def vgg_block(num_convs, in_channels, out_channels):
    net = [nn.Conv2d(in_channels, out_channels, kernel_size=3, padding=1), nn.ReLU(True)]

    for i in range(num_convs - 1):  # 定义后面的许多层
        net.append(nn.Conv2d(out_channels, out_channels, kernel_size=3, padding=1))
        net.append(nn.ReLU(True))

    net.append(nn.MaxPool2d(2, 2))  # 定义池化层
    return nn.Sequential(*net)

# 下面我们定义一个函数对这个 vgg block 进行堆叠
def vgg_stack(num_convs,
              channels):  # vgg_net = vgg_stack((1, 1, 2, 2, 2), ((3, 64), (64, 128), (128, 256), (256, 512), (512, 512)))
    net = []
    for n, c in zip(num_convs, channels):
        in_c = c[0]
        out_c = c[1]
        net.append(vgg_block(n, in_c, out_c))
    return nn.Sequential(*net)

# vgg类
class vgg(nn.Module):
    def __init__(self):
        super(vgg, self).__init__()
        self.feature = vgg_net
        self.fc = nn.Sequential(
            nn.Linear(512, 100),
            nn.ReLU(True),
            nn.Linear(100, 10)
        )

    def forward(self, x):
        x = self.feature(x)
        x = x.view(x.shape[0], -1)
        x = self.fc(x)
        return x

# 然后我们可以训练我们的模型看看在 cifar10 上的效果
def data_tf(x):
    x = np.array(x, dtype='float32') / 255
    x = (x - 0.5) / 0.5
    x = x.transpose((2, 0, 1))  ## 将 channel 放到第一维，只是 pytorch 要求的输入方式
    x = torch.from_numpy(x)
    return x

transform = transforms.Compose([transforms.ToTensor(),
                                transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)),
                                ])

def get_acc(output, label):
    total = output.shape[0]
    _, pred_label = output.max(1)
    num_correct = (pred_label == label).sum().item()
    return num_correct / total

def train(net, train_data, valid_data, num_epochs, optimizer, criterion):
    if torch.cuda.is_available():
        net = net.cuda()
    for epoch in range(num_epochs):
        train_loss = 0
        train_acc = 0
        net = net.train()
        for im, label in train_data:
            if torch.cuda.is_available():
                im = Variable(im.cuda())
                label = Variable(label.cuda())
            else:
                im = Variable(im)
                label = Variable(label)
            # forward
            output = net(im)
            loss = criterion(output, label)
            # forward
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

            train_loss += loss.item()
            train_acc += get_acc(output, label)

        if valid_data is not None:
            valid_loss = 0
            valid_acc = 0
            net = net.eval()
            for im, label in valid_data:
                if torch.cuda.is_available():
                    with torch.no_grad():
                        im = Variable(im.cuda())
                        label = Variable(label.cuda())
                else:
                    with torch.no_grad():
                        im = Variable(im)
                        label = Variable(label)
                output = net(im)
                loss = criterion(output, label)
                valid_loss += loss.item()
                valid_acc += get_acc(output, label)
            epoch_str = (
                    "Epoch %d. Train Loss: %f, Train Acc: %f, Valid Loss: %f, Valid Acc: %f, "
                    % (epoch, train_loss / len(train_data),
                       train_acc / len(train_data), valid_loss / len(valid_data),
                       valid_acc / len(valid_data)))
        else:
            epoch_str = ("Epoch %d. Train Loss: %f, Train Acc: %f, " %
                         (epoch, train_loss / len(train_data),
                          train_acc / len(train_data)))

        # prev_time = cur_time
        print(epoch_str)

if __name__ == '__main__':
    # 作为实例，我们定义一个稍微简单一点的 vgg11 结构，其中有 8 个卷积层
    vgg_net = vgg_stack((1, 1, 2, 2, 2), ((3, 64), (64, 128), (128, 256), (256, 512), (512, 512)))
    print(vgg_net)

    train_set = CIFAR10('./data', train=True, transform=transform, download=False)
    train_data = torch.utils.data.DataLoader(train_set, batch_size=64, shuffle=True)
    test_set = CIFAR10('./data', train=False, transform=transform, download=False)
    test_data = torch.utils.data.DataLoader(test_set, batch_size=128, shuffle=False)

    net = vgg()
    optimizer = torch.optim.SGD(net.parameters(), lr=1e-1)
    criterion = nn.CrossEntropyLoss()  # 损失函数为交叉熵

    train(net, train_data, test_data, 50, optimizer, criterion)
    torch.save(net, 'vgg_model.pth')


import threading
import logging
import time

def thread1(event):
    logging.info("wait for the event to start")
    event_is_set = event.wait()
    if event_is_set:
        logging.info('Im doing someting because event is set')
    logging.info('event set: %s', event_is_set)

def thread2(event, timeout):
    while not event.is_set(): # event set olmadigi surece while'da kal
        logging.info("wait for the event to start")
        event_is_set = event.wait(timeout)
        logging.info('event set: %s', event_is_set)
        if event_is_set:
            logging.info('processing event')
        else:
            logging.info('doing something else')

if __name__ == "__main__":
    logging.basicConfig(level=logging.DEBUG,
                        format='(%(threadName)-9s) %(message)s', )

    e = threading.Event()
    t1 = threading.Thread(name="blocking", target=thread1, args=(e, ))
    t1.start()

    t2 = threading.Thread(name="non-blocking", target=thread2, args=(e, 2))
    t2.start()

    logging.info('Waiting before calling Event.set()')
    time.sleep(3)
    e.set()
    logging.info('Event is set')


import sqlite3
import sys
import numpy as np
import pandas as pd
from numpy import random
import os

# random seed
random.seed(1)

# サンプリングの割合
sampling_ratio = 0.1

# オプションの処理
if(len(sys.argv) == 2 and sys.argv[1] in ["training", "tuning"]):
    dataset = sys.argv[1]
else:
    print("usage: python3 random_sampling.py [training|tuning]")
    sys.exit()

# DBに接続
base_path = os.path.normpath(os.path.join(os.path.abspath(os.path.dirname(__file__)) , "../../"))
con = sqlite3.connect(os.path.join(base_path, "tables/track2.sqlite3"))

# queryの組み立て
query = "SELECT id FROM training WHERE age IS NOT NULL AND gender IN (1,2);"
print(query)

# 実行して結果をデータフレームとして取り出し
df = pd.read_sql(query, con)
size = df.shape[0]

# indexの組み立て
if(dataset == "tuning"):
    start_index = int(size * sampling_ratio)
    last_index = int(size * (sampling_ratio + sampling_ratio ** 2))
else:
    start_index = 0
    last_index = int(size * sampling_ratio)
indices = np.sort(random.permutation(size)[start_index:last_index])

# csvとして出力
output_path = os.path.join(base_path, "modeling/dataset/{dataset}_ids.csv".format(dataset = dataset))
df.take(indices).to_csv(output_path, index = False)

# 終了処理
con.close()


#coding=utf-8

import argparse
import os
import time

from skipgram import build_model, train_op, train
from dataset import Dataset

def parse_args():
    # parse ht arguments
    parser = argparse.ArgumentParser(description='metapath2vec')
    parser.add_argument('--walks', type=str, required=True, help='text file that has a random walk in each line.')
    parser.add_argument('--types', type=str, required=True, help='text file that has node types.')
    parser.add_argument('--epochs', type=int, default=2, help='number of epochs.')
    # parser.add_argument('--batch', type=int, default=1, help='batch size.')
    parser.add_argument('--lr', type=float, default=0.01, help='learning rate.')
    parser.add_argument('--log', type=str, required=True, help='log directory.')
    parser.add_argument('--log-interval', type=int, default=-1, help='log intervals. -1 means per epoch')
    parser.add_argument('--max-keep-model', default=10, type=int, help='number of models to keep saving')
    parser.add_argument('--embedding-dim', default=100, type=int, help='embedding dimensions')
    parser.add_argument('--negative-samples', default=5, type=int, help='number of negative samples')
    parser.add_argument('--care-type', default=1, type=int,
                        help='care type or not. if 1, it cares (i.e. heterogeneous negative sampling). If 0, it does not care (i.e. normal negative sampling).')
    parser.add_argument('--window', default=5, type=int, help='context window size')

    return parser.parse_args()

def main(args):
    if os.path.isdir(args.log):
        print('%s already exist. Are you sure to override? I will wait for 5 seconds. Ctrl-C to abort.' % args.log)
        time.sleep(5)
        os.system('rm -rf %s/' % args.log)
    else:
        os.makedirs(args.log)
        print('make the log directory %s success!' % args.log)

    t0 = time.time()
    dataset = Dataset(random_walk_txt=args.walks, node_type_mapping_txt=args.types, window_size=args.window)
    center_node_placeholder, context_node_placeholder, negative_sample_placeholder, loss = build_model(BATCH_SIZE=1, VOCAB_SIZE=len(dataset.nodeid2index), EMBED_SIZE=args.embedding_dim, NUM_SAMPLED=args.negative_samples)
    optimizer = train_op(loss, LEARNING_RATE=args.lr)
    train(center_node_placeholder, context_node_placeholder, negative_sample_placeholder, loss, dataset, optimizer, NUM_EPOCHS=args.epochs, BATCH_SIZE=1, NUM_SAMPLED=args.negative_samples, care_type=args.care_type, LOG_DIRECTORY=args.log, LOG_INTERVAL=args.log_interval, MAX_KEEP_MODEL=args.max_keep_model)
    t1 = time.time()
    deltaT = t1 - t0
    deltaT = deltaT / 60.0
    print('Totally spend time is: %f min' % deltaT)

if __name__ == '__main__':
    args = parse_args()
    main(args)


# encoding: utf8
from __future__ import division

import os
import logging
import math

logs = set()

def init_log(name, log_dir, level=logging.INFO):
    if (name, level) in logs:
        return
    logs.add((name, level))
    logger = logging.getLogger(name)
    logger.setLevel(level)
    formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(filename)s(#%(lineno)d): %(message)s')

    ch = logging.StreamHandler()
    ch.setLevel(level)
    ch.setFormatter(formatter)
    logger.addHandler(ch)

    if not os.path.exists(log_dir):
        os.makedirs(log_dir)
    log_path = os.path.join(log_dir, 'log.log')

    f_handler = logging.FileHandler(log_path)
    f_handler.setLevel(level)
    f_handler.setFormatter(formatter)
    logger.addHandler(f_handler)

# init_log('global')

def print_speed(i, i_time, n):
    """print_speed(index, index_time, total_iteration)"""
    logger = logging.getLogger('global')
    average_time = i_time
    remaining_time = (n - i) * average_time
    remaining_day = math.floor(remaining_time / 86400)
    remaining_hour = math.floor(remaining_time / 3600 - remaining_day * 24)
    remaining_min = math.floor(remaining_time / 60 - remaining_day * 1440 - remaining_hour * 60)
    logger.info('Progress: %d / %d [%d%%], Speed: %.3f s/iter, ETA %d:%02d:%02d (D:H:M)\n' % (
        i, n, i / n * 100, average_time, remaining_day, remaining_hour, remaining_min))

def main():
    for i, lvl in enumerate([logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR, logging.CRITICAL]):
        log_name = str(lvl)
        init_log(log_name, './', lvl)
        logger = logging.getLogger(log_name)
        print('****cur lvl:{}'.format(lvl))
        logger.debug('debug')
        logger.info('info')
        logger.warning('warning')
        logger.error('error')
        logger.critical('critiacal')

if __name__ == '__main__':
    main()


import os
import sys
import numpy as np
from warnings import filterwarnings

filterwarnings("ignore")

import utils
import utils_data
import utils_models

####################################### Settings #######################################
datasets = ['scenes', 'birds', 'flowers', 'cars', 'aircrafts']
n_classes = {'scenes': 67, 'flowers': 102, 'birds': 200, 'cars': 196, 'aircrafts': 90}

data_dir = sys.argv[1]
data_size = int(sys.argv[2])
valid_split = float(sys.argv[3])
test_split = float(sys.argv[4])
seed = int(sys.argv[5])
ds = sys.argv[6]

result_filename = "result.txt"
data_shape = (data_size, data_size, 3)
batch_size = 16

features_extractor_name = 'VGG16'
autoencoder_hidden_layer_sizes = [50, 100, 200, 400]
autoencoder_weight_decays = [0.005, 0.0005]
autoencoder_learning_rates = [0.1, 0.01, 0.001]
autoencoder_epsilons = [1e-07, 1e-08]
autoencoder_objective_loss = 'binary_crossentropy'
autoencoder_epochs = 25
########################################################################################

features_mean = np.load(os.path.join(os.getcwd(), str(data_size) + '_mean.npy'))
features_std = np.load(os.path.join(os.getcwd(), str(data_size) + '_std.npy'))
features_extractor = utils_models.build_features_extractor(model_name=features_extractor_name, input_shape=data_shape)

for task_id, dataset in enumerate(datasets):
    if ds == dataset:
        if task_id == 0:
            open_mode = 'w'
        else:
            open_mode = 'a'
        utils.write_on_file(result_filename, open_mode,
                            "[%s] Loading data..." % dataset.upper())
        print("[%s] Loading data..." % dataset.upper())

        X_train, _, X_valid, _, _, _ = utils_data.load_data(data_dir=data_dir,
                                                            dataset=dataset,
                                                            data_size=data_size,
                                                            valid_split=valid_split,
                                                            test_split=test_split,
                                                            seed=seed)
        utils.write_on_file(result_filename, 'a', "[%s] Starting autoencoder's cross-validation..." % dataset.upper())
        print("[%s] Starting autoencoder's cross-validation..." % dataset.upper())

        autoencoder = utils.autoencoder_cross_validation(features_extractor=features_extractor,
                                                         batch_size=batch_size, X_train=X_train, X_valid=X_valid,
                                                         features_mean=features_mean, features_std=features_std,
                                                         hidden_layer_sizes=autoencoder_hidden_layer_sizes,
                                                         weight_decays=autoencoder_weight_decays,
                                                         learning_rates=autoencoder_learning_rates,
                                                         epsilons=autoencoder_epsilons, epochs=autoencoder_epochs,
                                                         objective_loss=autoencoder_objective_loss, dataset=ds)
        autoencoder.save(os.path.join('./', dataset + '_autoencoder.h5'))


# example resource link: https://pytorch.org/tutorials/beginner/blitz/cifar10_tutorial.html

import sys
import os

fid = open("../../.envPath","r")
envPath = fid.readline()
envPath = envPath[:-1]
fid.close
del fid
print("EnvPath:"+envPath)

sys.path.append(os.path.abspath(envPath+"/src/io"))

from load_Cul10_Semi import *
import torch
import numpy as np
import h5py
from torchvision import transforms, datasets

'''
PARAMETER SETTING
'''
print("parameter setting...")
paraDict = {
        ### network parameters
        "nbBatch": 256,
        "nbEpoch": 728,
        "learningRate": 1e-4,

        ### data loading parameters
        "trainData": "lcz42", # training data could be the training data of LCZ42 data, or data of one of the cultural-10 city
        # "trainData": "moscow",
        "testData": "cul10",  # testing data could be all the data of the cultural-10 cities, or one of them.
        # "testData": "munich",

        "normalization":"no", # "ms": mean-std normalization, patch-wise
        "datFlag":2, # data selection: sentinel-1, sentinel-2, or both

        ### model name
        "modelName":'ResNet',
        #"modelName":'LeNet_conv5',
        #"modelName":'LeNet', # model name
        #"modelName":'Sen2LCZ',#'LeNet', # model name
        #"Sen2LCZ_drop_out": 0.2,

        }

cudaNow = torch.device('cuda:0')

nbBatch = paraDict["nbBatch"]
nbEpoch = paraDict["nbEpoch"]
learnRate = paraDict["learningRate"]
datFlag = paraDict["datFlag"]
modelName = paraDict["modelName"]

'''
initial folder saving outputs
'''
outcomeDir = initialOutputFolder(paraDict)
print("Experiments outcomes are saving in the directory: "+outcomeDir)
## record parameters
recordExpParameters(outcomeDir,paraDict)

'''
STEP ONE: data loading
'''
trainDataSet,testDataSet = lczIterDataSet(envPath,paraDict["trainData"],paraDict["testData"],datFlag,paraDict["normalization"],transform=transforms.Compose([ToTensor()]))

s1_source_data_loader = torch.utils.data.DataLoader(trainDataSet, batch_size=nbBatch, shuffle=True)
s2_source_data_loader = torch.utils.data.DataLoader(trainDataSet, batch_size=nbBatch, shuffle=True)
target_data_loader = torch.utils.data.DataLoader(testDataSet, batch_size=nbBatch, shuffle=True)

'''
STEP TWO: initial a resnet model
'''
sys.path.append(os.path.abspath(envPath+"/src/model"))
import resnetModel
if modelName == 'ResNet':
    student1 = resnetModel.resnet18(pretrained=False, inChannel=trainDataSet.nbChannel()).to(cudaNow)
    student2 = resnetModel.resnet18(pretrained=False, inChannel=trainDataSet.nbChannel()).to(cudaNow)
elif  modelName == 'LeNet':
    student1 = resnetModel.LeNet(inChannel=trainDataSet.nbChannel(), nbClass = trainDataSet.label.shape[1]).to(cudaNow)
    student2 = resnetModel.LeNet(inChannel=trainDataSet.nbChannel(), nbClass = trainDataSet.label.shape[1]).to(cudaNow)
elif modelName == 'Sen2LCZ':
    student1 = resnetModel.Sen2LCZ(in_Channel=10, nb_class=17, nb_kernel=16, depth=17, bn_flag=1, drop_rate=paraDict["Sen2LCZ_drop_out"]).to(cudaNow)
    student2 = resnetModel.Sen2LCZ(in_Channel=10, nb_class=17, nb_kernel=16, depth=17, bn_flag=1, drop_rate=paraDict["Sen2LCZ_drop_out"]).to(cudaNow)
elif modelName == 'LeNet_conv5':
    student1 = resnetModel.LeNet_conv_5(inChannel=trainDataSet.nbChannel(), nbClass = trainDataSet.label.shape[1]).to(cudaNow)
    student2 = resnetModel.LeNet_conv_5(inChannel=trainDataSet.nbChannel(), nbClass = trainDataSet.label.shape[1]).to(cudaNow)
else:
    display('model not defined')

'''
STEP THREE: Define a loss function and optimizer
'''
import torch.optim as optim
import torch.nn as nn
classification_loss = nn.CrossEntropyLoss()
consistency_loss = nn.MSELoss()
optimizer_stu1 = optim.Adam(student1.parameters(), lr=learnRate)
optimizer_stu2 = optim.Adam(student2.parameters(), lr=learnRate)

'''
STEP FOUR: Train the network
'''
import modelOperDataLoader
student1, student2, cla_loss_train_student1, cla_acc_train_student1, cla_loss_test_student1, cla_acc_test_student1, cla_loss_train_student2, cla_acc_train_student2, cla_loss_test_student2, cla_acc_test_student2, con_loss_train_student1, con_loss_train_student2, cla_averacc_test_student1, cla_averacc_test_student2  = modelOperDataLoader.train_unlabel_ensemble(student1, student2, cudaNow, s1_source_data_loader, s2_source_data_loader, target_data_loader, classification_loss, consistency_loss, optimizer_stu1, optimizer_stu2, nbEpoch)

'''
STEP FIVE: Test the network
'''
# pred,_,acc = modelOperation.test(resnet,cudaNow,dat_te,lab_te,criterion = criterion,numBatch=nbBatch)
# print('Accuracy of the network on the %d test samples: %d %%' % ( dat_te.shape[0], acc))

'''
STEP SIX: Save the trained model
'''
'''
codes for saving and loading models:
	Save:
		torch.save(model.state_dict(), PATH)
	Load:
		model = TheModelClass(*args, **kwargs)
		model.load_state_dict(torch.load(PATH))
		model.eval()
'''

torch.save(student1.state_dict(), os.path.join(outcomeDir,'student1_model'))
torch.save(student2.state_dict(), os.path.join(outcomeDir,'student2_model'))

# saveing training history
fid = h5py.File(os.path.join(outcomeDir,'training_history.h5'),'w')
fid.create_dataset('cla_loss_train_student1',data=cla_loss_train_student1)
fid.create_dataset('cla_loss_train_student2',data=cla_loss_train_student2)
fid.create_dataset('con_loss_train_student1',data=con_loss_train_student1)
fid.create_dataset('con_loss_train_student2',data=con_loss_train_student2)
fid.create_dataset('cla_acc_train_student1',data=cla_acc_train_student1)
fid.create_dataset('cla_acc_train_student2',data=cla_acc_train_student2)
fid.create_dataset('cla_loss_test_student1',data=cla_loss_test_student1)
fid.create_dataset('cla_loss_test_student2',data=cla_loss_test_student2)
fid.create_dataset('cla_acc_test_student1',data=cla_acc_test_student1)
fid.create_dataset('cla_acc_test_student2',data=cla_acc_test_student2)
fid.create_dataset('cla_averacc_test_student1',data=cla_averacc_test_student1)
fid.create_dataset('cla_averacc_test_student2',data=cla_averacc_test_student2)
fid.close()

'''
STEP SEVEN: Predict with the student1 model
'''
confusion_matrix,oa,aa,ka,pa,ua = modelOperDataLoader.confusionMatrix(student1, cudaNow, target_data_loader, classification_loss)

# predModel_s1.load_state_dict(torch.load(modelPath,map_location=cudaNow))
# confusion_matrix,oa,aa,ka,pa,ua = modelOperDataLoader.confusionMatrix(predModel_s1, cudaNow, target_data_loader, classification_loss)

# save accuracy
fid = h5py.File(os.path.join(outcomeDir,'student1_test_accuracy.h5'),'w')
fid.create_dataset('confusion_matrix',data=confusion_matrix)
fid.create_dataset('oa',data=oa)
fid.create_dataset('aa',data=aa)
fid.create_dataset('ka',data=ka)
fid.create_dataset('pa',data=pa)
fid.create_dataset('ua',data=ua)
fid.close()
# plot confusion matrix
cm_disp_obj = modelOperDataLoader.ConfusionMatrixDisplay(confusion_matrix,np.linspace(1,confusion_matrix.shape[0],confusion_matrix.shape[0]))
cm_disp = cm_disp_obj.plot()
cm_disp.figure_.savefig(os.path.join(outcomeDir,'student1_confusion_matrix.png'))

'''
STEP SEVEN: Predict with the trained student2 model
'''
confusion_matrix,oa,aa,ka,pa,ua = modelOperDataLoader.confusionMatrix(student2, cudaNow, target_data_loader, classification_loss)

# save accuracy
fid = h5py.File(os.path.join(outcomeDir,'student2_test_accuracy.h5'),'w')
fid.create_dataset('confusion_matrix',data=confusion_matrix)
fid.create_dataset('oa',data=oa)
fid.create_dataset('aa',data=aa)
fid.create_dataset('ka',data=ka)
fid.create_dataset('pa',data=pa)
fid.create_dataset('ua',data=ua)
fid.close()
# plot confusion matrix
cm_disp_obj = modelOperDataLoader.ConfusionMatrixDisplay(confusion_matrix,np.linspace(1,confusion_matrix.shape[0],confusion_matrix.shape[0]))
cm_disp = cm_disp_obj.plot()
cm_disp.figure_.savefig(os.path.join(outcomeDir,'student2_confusion_matrix.png'))

'''
plot outcomes
'''
pth = modelOperDataLoader.plotTrainHistory(outcomeDir,paraDict["modelName"])
pth.plotHistory()


# coding:utf-8

import pyliblinear
from sklearn.utils import shuffle
from sklearn.datasets import dump_svmlight_file
from sklearn.svm import LinearSVC
from sklearn import metrics
from sklearn.model_selection import KFold
import numpy as np
from sklearn.feature_extraction.text import CountVectorizer
import os

feature_dir = "/home/iiip/Info/capsnet/n-gram/data/feature(1,3).txt"
ngram = CountVectorizer(ngram_range=(1,3))

def metrics_result(actual,predict):
    accuracy = round(metrics.accuracy_score(actual,predict),3)
    return accuracy

def get_dataset():
    corpus = []
    label_list = []
    datadir = "/home/iiip/Info/capsnet/data/20_newsgroups_new"
    for labelname in os.listdir(datadir):
        labeldir = os.path.join(datadir,labelname)
        for textname in os.listdir(labeldir):
            text = os.path.join(labeldir,textname)
            with open(text,"r") as f:
                content = f.read()
                corpus.append(content)
            label_list.append(labelname)
    data = ngram.fit_transform(corpus)
    return data,label_list

#data是ngram表示特征的sparse matrix
#label_list是每篇文档所属类别
data,label_list = get_dataset()

#将每篇文章所属类别转换成integer
def get_label(label_list):
    label_list = label_list
    label = []
    label2index = {}
    index = 0
    datadir = "/home/iiip/Info/capsnet/data/20_newsgroups_new"
    for labelname in os.listdir(datadir):
        label2index[labelname] = index
        index+=1
    for i in label_list:
        label.append(label2index[i])
    return label

label = get_label(label_list)
data,label = shuffle(data,label)  #打乱数据
print(ngram.ngram_range)
dump_svmlight_file(data,label,feature_dir) #将数据转换成libsvm格式并保存本地

# 训练，预测代码，在Python终端运行
# y, x = svm_read_problem('/home/iiip/Info/capsnet/n-gram/data/feature(1,2).txt', return_scipy = True)
# m2 = train(y[:16000], x[:16000, :], '-c 1 -s 0')
# p_label, p_acc, p_val = predict(y[16000:], x[16000:, :], m2)


from pulse_templates.coherent_control.single_qubit_gates.phase_offsets_charac import phase_offset_charac
from core_tools.sweeps.pulse_lib_wrappers.PSB_exp import run_qubit_exp
from good_morning.fittings.fit_rabi_osc import fit_ramsey
from core_tools.sweeps.sweeps import scan_generic, do0D, do1D
from dev_V2.six_qubit_QC.system import six_dot_sample
from core_tools.utility.variable_mgr.var_mgr import variable_mgr
import qcodes as qc
import numpy as np
from pulse_lib.segments.utility.looping import linspace
from pulse_templates.coherent_control.single_qubit_gates.single_qubit_gates import single_qubit_gate_spec

def get_target(obj, name):
	name = name.split('.')

	for operator in name:
		obj = getattr(obj, operator)

	return obj

def CROT_cali_pi_meas(pair, target, ancilla, freqs, old_time, flip):

	s = six_dot_sample(qc.Station.default.pulse)
	var_mgr = variable_mgr()

	pair = str(int(pair))
	s.init(pair[0], pair[1])

	s.wait(100).add(s.manip)
	s.pre_pulse.add(s.manip)
	s.wait(100).add(s.manip)

	gate_set = getattr(s, f'q{pair}')
	target_gate_set = getattr(s, f'q{pair[target-1]}')
	ancilla_gate_set = getattr(s, f'q{pair[ancilla-1]}')

	if flip:
		ancilla_gate_set.X180.add(s.manip)
		# target_gate_set.X180.add(s.manip)

	scan_range = single_qubit_gate_spec(f'qubit{target}_MW',
                                    linspace(freqs[0]*1e9, freqs[1]*1e9, 2, axis= 1, name='freq', unit='Hz'),
                                    linspace(0,old_time*4, 100, axis= 0, name='time', unit='ns'), getattr(var_mgr, f'q{pair[target-1]}_MW_power'), padding=2)

	gate_to_test = getattr(gate_set, f'CROT{target}{ancilla}')

	gate_to_test.add(s.manip, gate_spec = scan_range)
	s.wait(100).add(s.manip)
	s.read(pair[target-1])
	sequence, minstr, name = run_qubit_exp(f'crot-z_crot_cal_pi_q{pair}_target{pair[target-1]}', s.segments(), s.measurement_manager)

	return sequence, minstr, name

def CROT_pi_calib(pair, target, ancilla, plot=False):
	'''
	calibrates a single qubit phase for a target qubit

	Args:
		target (int) : qubit number to target (1-6)
		ancillary (list) : qubit that need to be initialized to make this work
	'''
	var_mgr = variable_mgr()
	pair = str(int(pair))

	old_crot = getattr(var_mgr, f'crot{pair[target-1]}{pair[ancilla-1]}')
	old_z_crot = getattr(var_mgr, f'z_crot{pair[target-1]}{pair[ancilla-1]}')
	crot_time = getattr(var_mgr, f'pi_crot{pair[target-1]}_m3dbm')
	z_crot_time = getattr(var_mgr, f'pi_z_crot{pair[target-1]}_m3dbm')

	gate_time= max(crot_time,z_crot_time)

	sequence, minstr, name = CROT_cali_pi_meas(pair, target, ancilla, [old_z_crot,old_crot], gate_time, 0)
	ds_one = scan_generic(sequence, minstr, name=name).run()
	sequence, minstr, name = CROT_cali_pi_meas(pair, target, ancilla, [old_z_crot,old_crot], gate_time, 1)
	ds_two = scan_generic(sequence, minstr, name=name).run()

	frequency_one = ds_one(f'read{pair[target-1]}').x()
	time_one = ds_one(f'read{pair[target-1]}').y()*1e-9
	probabilities_one = ds_one(f'read{pair[target-1]}').z()
	index_one = np.argmax([[np.std(probabilities_one[0]),np.std(probabilities_one[1])]])
	Pi_time_one = fit_ramsey(time_one[5:], probabilities_one[index_one][5:], plot=plot)

	frequency_two = ds_two(f'read{pair[target-1]}').x()
	time_two = ds_two(f'read{pair[target-1]}').y()*1e-9
	probabilities_two = ds_two(f'read{pair[target-1]}').z()
	index_two = np.argmax([[np.std(probabilities_two[0]),np.std(probabilities_two[1])]])
	Pi_time_two = fit_ramsey(time_two, probabilities_two[index_two], plot=plot)

	if index_one>index_two:
		pi_crot =[Pi_time_two,Pi_time_one]
	else:
		pi_crot =[Pi_time_one,Pi_time_two]
	old_z_crot = getattr(var_mgr, f'pi_z_crot{pair[target-1]}_m3dbm')
	setattr(var_mgr, f'pi_z_crot{pair[target-1]}_m3dbm', round(pi_crot[0]))
	old_crot = getattr(var_mgr, f'pi_crot{pair[target-1]}_m3dbm')
	setattr(var_mgr, f'pi_crot{pair[target-1]}_m3dbm', round(pi_crot[1]))

	print(f'calibrated z_crot_pi for qubit pair {pair}, target {pair[target-1]} '+
		f'Old z_crot_pi : {round(old_z_crot,6)} \n New z_crot_pi : {round(pi_crot[0])} \n')
	print(f'calibrated crot_pi for qubit pair {pair}, target {pair[target-1]} '+
		f'Old crot_pi : {round(old_crot,6)} \n New crot_pi : {round(pi_crot[1])} \n')


__author__ = 'maury'

import os
import pandas as pd
from py2neo import Graph
from pandas import DataFrame
from generalSettings import CSV_DIRECTORY, DATA_DIRECTORY
from tools.persistence import storageCSV,storageData,readingData

class Recommender:
    def __init__(self):
        self.graph=Graph()
        self.matrix=self.makeMatrix()

    def makeMatrix(self):
        # Recupero di TUTTI i rates (USER_RATING e RATING) forniti dai diversi utenti per le diverse Attrazioni e ritorno DataFrame
        if os.path.exists(CSV_DIRECTORY+"ratesMatrix.csv") and os.path.exists(DATA_DIRECTORY+"ratesMatrix"):
            # Vado a recuperare la matrice di rates memorizzato in precedenza
            matrix=readingData("ratesMatrix")
            # Vado a costruire la matrice dei rates vera e propria sotto forma di DataFrame
            print("\nCaricata la matrice  di rates (DataFrame)!")
            # Vado a calcolare il valore di densita della matrice
            self.sparsita=self.calcolaSparsita(matrix)
            print("\nSPARSITA: {}".format(self.sparsita))
            return matrix
        else:
            if os.path.exists(CSV_DIRECTORY+"ratings.csv") and os.path.exists(DATA_DIRECTORY+"ratings"):
                # Vado a recuperare il DataFrame memorizzato in precedenza
                rates=readingData("ratings")
                # Vado a costruire la matrice dei rates vera e propria sotto forma di DataFrame
                print("\nCreato la matrice (DataFrame) dei rates degli utenti!")
                return self.createRateMatrix(rates)
            else:
                # Recupero i RATES e costruisco il Dataframe da utilizzare
                self.retriveRates()
                pd.read_csv(CSV_DIRECTORY+"ratings.csv",index_col=0)
                print("\nPrelevato e creato il DataFrame di tutti i rates degli utenti!")
                return False

    def retriveRates(self):
        # Recupero tutti i rates forniti dagli utenti tramite reviews che hanno stilato per le varie attrazioni
        query="MATCH (u:User)-[w:WROTE]->(rev:Review)-[r:RATING]->(a:Attraction) \
        RETURN a.attrazioneID as URLAttrazione,a.nome as NomeAttrazione,COLLECT([u.utente,u.nome,r.rate]) as Valori, COUNT(a.attrazioneID) as NumRates \
        ORDER BY (a.nome)"
        datiReviews=self.graph.cypher.execute(query)

        totRatings=0
        dati=[]
        for datoReviews in datiReviews:
            dati.append({"URLAttrazione":datoReviews["URLAttrazione"],"NomeAttrazione":datoReviews["NomeAttrazione"],"Valori":datoReviews["Valori"],"NumRates":datoReviews["NumRates"]})
            totRatings+=datoReviews["NumRates"]

        print("Numero di RATINGS da REVIEWS: {}".format(totRatings))

        query="MATCH (u:User)-[r:USER_RATING]->(a:Attraction) \
        RETURN a.attrazioneID as URLAttrazione,a.nome as NomeAttrazione,COLLECT([u.utente,u.nome,r.rate]) as Valori, COUNT(a.attrazioneID) as NumRates \
        ORDER BY (a.nome)"
        datiRatings=self.graph.cypher.execute(query)

        # Recupero di tutti i rates forniti dagli utenti senza reviews per le varie attrazioni
        for datoRatings in datiRatings:
            trovato=False
            for dato in dati:
                if(dato["URLAttrazione"]==datoRatings["URLAttrazione"]):
                    dato["Valori"]=dato["Valori"]+datoRatings["Valori"]
                    dato["NumRates"]=dato["NumRates"]+datoRatings["NumRates"]
                    trovato=True
            if not trovato:
                dati.append({"URLAttrazione":datoRatings["URLAttrazione"],"NomeAttrazione":datoRatings["NomeAttrazione"],"Valori":datoRatings["Valori"],"NumRates":datoRatings["NumRates"]})
            totRatings+=datoRatings["NumRates"]

        print("Numero di RATINGS TOTALI: {}".format(totRatings))

        # Vado a creare il DataFrame dai dati recuperati e ne creo il CSV corrispondente
        urls=[dato["URLAttrazione"] for dato in dati]
        valori=[]
        for dato in dati:
            valori.append([dato["NomeAttrazione"],dato["Valori"],dato["NumRates"],totRatings])
        rates=pd.DataFrame(valori,index=urls,columns=['NomeAttrazione','Valori','NumRates','NumRatingsGlobale'])
        storageCSV(rates.to_csv(),"ratings.csv")
        storageData(rates,"ratings")
        return rates

    def createRateMatrix(self,listaRates):
        """
        :param listaRates: Lista delle varie attrazioni con associati rates dei vari utenti
        :type listaRates DataFrame
        :return: Dataframe della matrice dei rates dei vari utenti
        """
        # nomiAttrs=listaRates["NomeAttrazione"].values
        urlsAttrs=listaRates.index.values
        i=0
        totRates=0
        dati={}
        for rates in listaRates["Valori"].values:
            print(rates)
            utenti=[rate[0] for rate in rates]
            voti=[rate[2] for rate in rates]
            totRates+=len(voti)
            diz=dict(zip(utenti,voti))
            dati[urlsAttrs[i]]=diz
            i+=1

        print("\nTOTRATES: {}".format(totRates))

        # Inserisco tutte le attrazioni per cui non è presente alcun rate
        query="MATCH (n:Attraction) RETURN n.attrazioneID as attrazioneID"
        attrs=self.graph.cypher.execute(query)
        for attr in attrs:
            if attr["attrazioneID"] not in dati.keys():
                dati[attr["attrazioneID"]]={}

        # print("\nDizionario dei dati del DataFrame: {}".format(dati))
        matrix=DataFrame(dati)
        storageCSV(matrix.to_csv(),"ratesMatrix.csv")
        storageData(matrix,"ratesMatrix")
        return matrix

    def calcolaSparsita(self,matrix):
        """
        Calcolo la sparsita della matrice dei rates creata
        :param matrix: matrice dei rates
        :type matrix: DataFrame
        :return:
        """
        numRates=sum(matrix.count().values)
        print("Num Users: {} - Num Attrs: {} - Num Ratings: {}".format(len(matrix.index),len(matrix.columns),numRates))
        return 1-(numRates)/(len(matrix.index)*len(matrix.columns))


import transaction

from pyramid.response import Response
from pyramid.config import Configurator
from pyramid.httpexceptions import HTTPNotFound, HTTPAccepted, HTTPCreated, HTTPBadRequest
from pyramid.view import notfound_view_config, view_config
from pyramid.events import NewRequest
from pyramid.settings import asbool
from pyramid_xmlrpc import parse_xmlrpc_request, xmlrpc_response, xmlrpc_view

from sqlalchemy import func
from sqlalchemy.orm.exc import NoResultFound
from sqlalchemy.exc import DataError

from . import models, health

def get_query_filters(model, request_params):
    filters = {}
    for name, value in request_params.items():
        if hasattr(model, name):
            filters[name] = value
    return filters

@notfound_view_config(append_slash=True)
def notfound(request):
    return HTTPNotFound('Not found')

@view_config(route_name='index')
def index(request):
    return Response("Balaio's HTTP server.")

@view_config(route_name='ArticlePkg', request_method='GET', renderer="gtw")
def package(request):
    """
    Get a single object and return a serialized dict
    """

    article = request.db.query(models.ArticlePkg).get(request.matchdict['id'])

    if article is None:
        return HTTPNotFound()

    return article.to_dict()

@view_config(route_name='list_package', request_method='GET', renderer="gtw")
def list_package(request):
    """
    Return a dict content the total param and the objects list
    Example: {'total': 12, 'limit': 20, offset: 0, 'objects': [object, object,...]}
    """
    limit = request.params.get('limit', request.registry.settings.get('http_server', {}).get('limit', 20))
    offset = request.params.get('offset', 0)

    filters = get_query_filters(models.ArticlePkg, request.params)
    articles = request.db.query(models.ArticlePkg).filter_by(**filters).limit(limit).offset(offset)

    return {'limit': limit,
            'offset': offset,
            'filters': filters,
            'total': request.db.query(func.count(models.ArticlePkg.id)).filter_by(**filters).scalar(),
            'objects': [article.to_dict() for article in articles]}

@view_config(route_name='Attempt', request_method='GET', renderer="gtw")
def attempt(request):
    """
    Get a single object and return a serialized dict
    """
    attempt = request.db.query(models.Attempt).get(request.matchdict['id'])

    if not attempt:
        return HTTPNotFound()

    return attempt.to_dict()

@view_config(route_name='list_attempts', request_method='GET', renderer="gtw")
def attempts(request):
    """
    Return a dict content the total param and the objects list
    Example: {'total': 12, 'limit': 20, offset:0, 'objects': [object, object,...]}
    """
    limit = request.params.get('limit', request.registry.settings.get('http_server', {}).get('limit', 20))
    offset = request.params.get('offset', 0)

    filters = get_query_filters(models.Attempt, request.params)
    attempts = request.db.query(models.Attempt).filter_by(**filters).limit(limit).offset(offset)

    return {'limit': limit,
            'offset': offset,
            'filters': filters,
            'total': request.db.query(func.count(models.Attempt.id)).filter_by(**filters).scalar(),
            'objects': [attempt.to_dict() for attempt in attempts]}

@view_config(route_name='proceed_to_checkout')
def proceed_to_checkout(request):
    """
    XML-RPC method to set attribute proceed_to_checkout=True
    return ``True`` if the operation succeeded ``False`` otherwise
    """
    params, method = parse_xmlrpc_request(request)

    if len(params) != 0:
        attempt = request.db.query(models.Attempt).get(params[0])
    else:
        _return = False

    if attempt:
        attempt.proceed_to_checkout = True
        try:
            transaction.commit()
            _return = True
        except:
            transaction.abort()
            _return = False
    else:
        _return = False

    return xmlrpc_response(_return)

@view_config(route_name='rpc_status')
@xmlrpc_view
def rpc_status(context):
    """
    XML-RPC method to test service
    """
    return True

@view_config(route_name='status', request_method='GET', renderer='json')
def health_status(request):
    """
    Display the health status of the system.
    """
    health_status = request.registry.health_status
    report = health_status.latest_report
    items = {k.__class__.__name__: v for k, v in report.items()}

    return {'meta': {'last_refresh': health_status.since()},
            'results': items}

@view_config(route_name='list_attempt_members', request_method='GET', renderer='json')
def list_files_from_attempt(request):
    """
    List all files bound to an Attempt.
    """
    attempt_id = request.matchdict.get('attempt_id', None)
    try:
        attempt = request.db.query(models.Attempt).get(attempt_id)
    except DataError:
        return HTTPNotFound()

    if attempt is None:
        return HTTPNotFound()

    return attempt.analyzer.get_classified_members()

@view_config(route_name='get_attempt_member', request_method='GET', renderer='json')
def get_file_from_attempt(request):
    """
    Get a portion of a package bound to an Attempt.

    Get a specific member, by name (raw):
    `/api/:api_id/files/:attempt_id/:target?file=:member&raw=true`

    Get a specific member, by name:
    `/api/:api_id/files/:attempt_id/:target.zip?file=:member`

    Get more than one specific members, by name:
    `/api/:api_id/files/:attempt_id/:target.zip?file=:member&file=:member2`

    Get the full package:
    `/api/:api_id/files/:attempt_id/:target.zip?full=true`
    """
    has_body = False

    attempt_id = request.matchdict.get('attempt_id', None)
    target = request.matchdict.get('target', None)

    try:
        attempt = request.db.query(models.Attempt).get(attempt_id)
    except DataError:
        return HTTPNotFound()

    if attempt is None:
        return HTTPNotFound()

    is_full = asbool(request.GET.get('full', False))
    is_raw = asbool(request.GET.get('raw', False))

    if is_full and is_raw:
        return HTTPBadRequest()

    response = Response(status_code=200)

    # Get the full package.
    if is_full:
        response.content_type = 'application/zip'
        response.app_iter = open(attempt.filepath, 'rb')
        has_body = True

    elif is_raw:
        member_name = request.GET.get('file')
        response.content_type = 'text/xml'
        response.app_iter = attempt.analyzer.get_fp(member_name)
        has_body = True

    else:
        response.content_type = 'application/zip'

        # Get partial portions of the package.
        files = [member for attr, member in request.GET.items() if attr == 'file']

        try:
            if files:
                response.app_iter = attempt.analyzer.subzip(*files)
                has_body = True
        except ValueError:
            return HTTPBadRequest()

    return response if has_body else HTTPBadRequest()

def main(config, engine):
    """
    Returns a pyramid app.

    :param config: an instance of :class:`utils.Configuration`.
    :param engine: sqlalchemy engine.
    """
    def bind_db(event):
        event.request.db = event.request.registry.Session()

    def update_health_status(event):
        event.request.registry.health_status.update()

    config_pyrmd = Configurator(settings=dict(config.items()))
    config_pyrmd.add_route('index', '/')
    config_pyrmd.add_route('status', '/status/')

    # get
    config_pyrmd.add_route('ArticlePkg', '/api/v1/packages/{id}/')
    config_pyrmd.add_route('Attempt', '/api/v1/attempts/{id}/')

    # lists
    config_pyrmd.add_route('list_package', '/api/v1/packages/')
    config_pyrmd.add_route('list_attempts', '/api/v1/attempts/')

    # XML RPC status
    config_pyrmd.add_route('rpc_status', '/api/v1/_rpc/status/')
    # XML RPC to mark ``proceed_to_checkout=True``
    config_pyrmd.add_route('proceed_to_checkout', '/api/v1/_rpc/proceed_to_checkout/')

    # files
    config_pyrmd.add_route('list_attempt_members', '/api/v1/files/{attempt_id}/')
    config_pyrmd.add_route('get_attempt_member', '/api/v1/files/{attempt_id}/{target}')

    config_pyrmd.add_renderer('gtw', factory='.renderers.GtwFactory')

    #DB session bound to each request
    config_pyrmd.registry.Session = models.ScopedSession
    config_pyrmd.registry.Session.configure(bind=engine)
    config_pyrmd.add_subscriber(bind_db, NewRequest)

    # Health check is available globally on the application
    # at `request.registry.health_check` and is updated
    # periodically.
    check_list = health.CheckList(refresh=1)
    check_list.add_check(health.DBConnection(engine))
    check_list.add_check(health.NotificationsOption(config))
    check_list.add_check(health.Monitor())
    check_list.add_check(health.Validator())

    config_pyrmd.registry.health_status = check_list
    config_pyrmd.add_subscriber(update_health_status, NewRequest)

    config_pyrmd.scan(package='.httpd')

    return config_pyrmd.make_wsgi_app()


from common import Modules, data_strings, load_yara_rules, PEParseModule, ModuleMetadata
import re
import zlib
from struct import unpack

from Crypto.Cipher import DES
import pefile

def run(raw_data):
    try:
        coded_config = get_codedconfig(raw_data)
        if coded_config[0:4] == '\x08\x00\x00\x00':
            config_dict = decrypt_new(coded_config)
        else:
           config_dict = decrypt_old(coded_config)
        return config_dict
    except KeyboardInterrupt:
        raise
    except:
        return

def decrypt_new(coded_config):
    key = coded_config[4:12]
    data = coded_config[16:]
    raw_config = decrypt_des(key, data)
    # if the config is over a certain size it is compressed. Indicated by a non Null byte
    if raw_config[1] == '\x00':
        return parse_config(raw_config, 'new')
    else:
        # remove the string lengths and deflate the remainder of the stream
        deflate_config = deflate_contents(raw_config)
        return parse_config(deflate_config, 'new')

def decrypt_old(coded_config):
    key = '\x01\x03\x05\x08\x0d\x15\x22\x37'
    data = coded_config[1:]
    new_data = decrypt_des(key, data)
    if new_data[0] != '\x00':
        deflate_config = deflate_contents(new_data)
        return parse_config(deflate_config, 'old')

def deflate_contents(data):
    new_data = data[5:]
    return zlib.decompress(new_data, -15)

# Returns only printable chars
def string_print(line):
    try:
        return ''.join((char for char in line if 32 < ord(char) < 127))
    except KeyboardInterrupt:
        raise
    except:
        return line

# returns pretty config
def parse_config(raw_config, ver):
    config_dict = {}
    if ver == 'new':
        #config_dict['BuildTime'] = unpack(">Q", re.search('BuildTime(.*?)\x0c', raw_config).group()[10:-1])[0]
        config_dict['Version'] = re.search('Version\x0c(.*?)\x0c', raw_config).group()[8:-1]
        config_dict['Mutex'] = re.search('Mutex(.*?)\x0c', raw_config).group()[6:-1].encode('hex')
        config_dict['Group'] = re.search('DefaultGroup\x0c(.*?)\x0c', raw_config).group()[14:-1]
        config_dict['Domain1'] = re.search('PrimaryConnectionHost\x0c(.*?)\x0c', raw_config).group()[23:-1]
        config_dict['Domain2'] = re.search('BackupConnectionHost\x0c(.*?)\x0c', raw_config).group()[22:-1]
        config_dict['Port'] = unpack("<H", re.search('ConnectionPort(.*?)\x0c', raw_config).group()[15:-1])[0]
        config_dict['RunOnStartup'] = re.search('RunOnStartup(.*?)\x0c', raw_config).group()[13:-1].encode('hex')
        config_dict['RequestElevation'] = re.search('RequestElevation(.*?)\x0c', raw_config).group()[17:-1].encode('hex')
        config_dict['BypassUAC'] = re.search('BypassUserAccountControl(.*?)\x0c', raw_config).group()[25:-1].encode('hex')
        config_dict['ClearZoneIdentifier'] = re.search('ClearZoneIdentifier(.*?)\x0c', raw_config).group()[20:-1].encode('hex')
        config_dict['ClearAccessControl'] = re.search('ClearAccessControl(.*?)\x0c', raw_config).group()[19:-1].encode('hex')
        config_dict['SetCriticalProcess'] = re.search('SetCriticalProcess(.*?)\x0c', raw_config).group()[19:-1].encode('hex')
        config_dict['FindLanServers'] = re.search('FindLanServers(.*?)\x0c', raw_config).group()[15:-1].encode('hex')
        config_dict['RestartOnException'] = re.search('RestartOnException(.*?)\x0c', raw_config).group()[19:-1].encode('hex')
        config_dict['EnableDebugMode'] = re.search('EnableDebugMode(.*?)\x0c', raw_config).group()[16:-1].encode('hex')
        config_dict['ConnectDelay'] = unpack("<i", re.search('ConnectDelay(.*?)\x0c', raw_config).group()[13:-1])[0]
    else:
        config_dict['Domain'] = re.search('HOST\x0c(.*?)\x0c', raw_config).group()[6:-1]
        config_dict['Port'] = unpack("<H", re.search('PORT(.*?)\x0c', raw_config).group()[5:-1])[0]
        config_dict['Group'] = re.search('GROUP\x0c(.*?)\x0c', raw_config).group()[7:-1]
        config_dict['ConnectDelay'] = unpack("<i", re.search('DELAY(.*?)\x0c', raw_config).group()[6:-1])[0]
        config_dict['OfflineKeyLog'] = str(re.search('OFFLINE_KEYLOGGING(.*?)\x0c', raw_config).group()[19:-1].encode('hex'))
    return config_dict

# This gets the encoded config from a stub
def get_codedconfig(data):

    coded_config = None
    pe = pefile.PE(data=data)
    for entry in pe.DIRECTORY_ENTRY_RESOURCE.entries:
        if str(entry.name) == "RC_DATA" or "RCData":
            new_dirs = entry.directory
            for res in new_dirs.entries:
                data_rva = res.directory.entries[0].data.struct.OffsetToData
                size = res.directory.entries[0].data.struct.Size
                data = pe.get_memory_mapped_image()[data_rva:data_rva+size]
                coded_config = data
                return coded_config

def decrypt_des(key, data):
    iv = key
    cipher = DES.new(key, DES.MODE_CBC, iv)
    return cipher.decrypt(data)

class Nanocore(PEParseModule):
    def __init__(self):
        md = ModuleMetadata(
            module_name="nanocore",
            bot_name="NanoCore",
            description="RAT",
            authors=["kevthehermit"],  # https://github.com/kevthehermit/RATDecoders/blob/master/NanoCore.py
            version="1.0.0",
            date="March 27, 2015",
            references=[]
        )
        PEParseModule.__init__(self, md)
        self.yara_rules = None
        pass

    def _generate_yara_rules(self):
        if self.yara_rules is None:
            self.yara_rules = load_yara_rules("nanocore.yara")
        return self.yara_rules

    def get_bot_information(self, file_data):
        results = run(file_data)
        if results is None:
            results = {}
        if 'Port' in results and ('Domain' in results or 'Domain1' in results or 'Domain2' in results):
            results['c2s'] = []
            if 'Domain' in results:
                results['c2s'].append({"c2_uri": "tcp://{0}:{1}".format(results['Domain'], results['Port'])})
            if 'Domain1' in results and 'Port' in results:
                results['c2s'].append({"c2_uri": "tcp://{0}:{1}".format(results['Domain1'], results['Port'])})
            if 'Domain2' in results and 'Port' in results:
                results['c2s'].append({"c2_uri": "tcp://{0}:{1}".format(results['Domain2'], results['Port'])})

        return results

Modules.list.append(Nanocore())


#Repunit divisibility
#Problem 129

import time
import fractions

def A(n):
    k = 2
    while not pow(10,k,9* n) == 1:
        k += 1
    return k

def solveBrute(n):
    i = getNext(n)
    while not A(i) >= n:
        i = getNext(i)
    return i

def getNext(i):
    i += 1
    while not fractions.gcd(i, 10) == 1:
        i += 1
    return i

tStart = time.time()
print(solveBrute(10**6))
print("Run Time = " + str(time.time() - tStart))


import pygame
import random
#dimesion fenetre#
largeur=720
hauteur=480

#police#
pygame.font.init()
ma_police=pygame.font.SysFont('Comic Sans MS',30)

ecran=pygame.display.set_mode((largeur,hauteur))
clock=pygame.time.Clock()
FPS=20
#Couleurs RGB (rouge vert bleu)#
Green=(0,255,0)
Black=(0,0,0)
Red=(255,0,0)

#serpent#
serpent=[[largeur/2,hauteur/2]]
taille=10
speed=10
x=0
y=0
#pomme#
pomme_x=random.randint(1,(largeur-10)//10)*10
pomme_y=random.randint(1,(hauteur-10)//10)*10

game_over=0
while True:
    clock.tick(FPS)
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            quit()
        elif event.type==pygame.KEYDOWN:
            #changement de direction#
            if event.key == pygame.K_LEFT and x==0:
                x=-speed
                y=0
            elif event.key == pygame.K_RIGHT and x==0:
                x=speed
                y=0
            elif event.key == pygame.K_UP and y==0:
                x=0
                y=-speed
            elif event.key == pygame.K_DOWN and y==0:
                x=0
                y=speed
            elif event.key==pygame.K_r:
                x=0
                y=0
                serpent=[[largeur/2,hauteur/2]]
                game_over =0

    ecran.fill(Black)
    if game_over==0:
        #collision Murs#
        if serpent[0][0]<0 or serpent[0][0]>largeur-taille or serpent[0][1]<0 or serpent[0][1]>hauteur-taille:
            game_over= 1
        #collision pomme#
        if serpent[0][0]== pomme_x and serpent[0][1] ==pomme_y:
            pomme_x=random.randint(1,(largeur-10)//10)*10
            pomme_y=random.randint(1,(hauteur-10)//10)*10
            #ajout queue du serpent#
            serpent.append([serpent[-1][0]+x,serpent[-1][1]+y])
        #mise a jouer des position du serpent#

        serpent.insert(0,[serpent[0][0]+x,serpent[0][1]+y])
        serpent.pop()

        #collision queue#
        if serpent[0] in serpent [1:]:
            game_over=1

        #mise a jour ecran#
        for i in range(len(serpent)):
            pygame.draw.rect(ecran,Green,(serpent[i][0],serpent[i][1],taille,taille))
        pygame.draw.rect(ecran,Red,(pomme_x,pomme_y,taille,taille))
    elif game_over==1:
        textsurface=ma_police.render("GAME OVER",False,Red)
        ecran.blit(textsurface,(largeur//2,hauteur//2))

    pygame.display.update()