text:
"""Inherited model class."""

from swarms.lib.model import Model
from swarms.lib.time import SimultaneousActivation
# RandomActivation, StagedActivation
from swarms.lib.space import Grid
from swarms.utils.jsonhandler import JsonData
from swarms.utils.results import Experiment
from swarms.utils.db import Connect
from simagent import SimForgAgent, SimCTAgent, SimNMAgent
from swarms.lib.objects import Hub, Sites, Food, Debris, Obstacles
import os
import imp
import datetime
import numpy as np
from swarms.utils.ui import UI

filename = os.path.join(imp.find_module("swarms")[1] + "/utils/world.json")

class SimForgModel(Model):
    """A environemnt to model swarms."""

    def __init__(
            self, N, width, height, grid=10, iter=100000,
            xmlstrings=None, seed=None, viewer=False, pname=None):
        """Initialize the attributes."""
        if seed is None:
            super(SimForgModel, self).__init__(seed=None)
        else:
            super(SimForgModel, self).__init__(seed)

        self.runid = datetime.datetime.now().strftime(
            "%s") + str(self.random.randint(1, 1000, 1)[0])

        if pname is None:
            self.pname = os.getcwd() + '/' + self.runid + "SForagingSimulation"
        else:
            self.pname = pname + '/' + self.runid + "SForagingSimulation"

        self.width = width
        self.height = height
        self.stepcnt = 1
        self.iter = iter
        self.xmlstrings = xmlstrings

        self.viewer = viewer

        # Create db connection
        connect = Connect('swarm', 'swarm', 'swarm', 'localhost')
        self.connect = connect.tns_connect()

        # Fill out the experiment table
        self.experiment = Experiment(
            self.connect, self.runid, N, seed, 'Simuation Single Foraging',
            iter, width, height, grid, phenotype=xmlstrings[0])

        self.experiment.insert_experiment_simulation()

        self.sn = self.experiment.sn

        # Create a folder to store results
        os.mkdir(self.pname)

        self.num_agents = N

        self.grid = Grid(width, height, grid)

        self.schedule = SimultaneousActivation(self)

        # self.site = Sites(id=1, location=(5, 5), radius=11, q_value=0.5)

        # self.grid.add_object_to_grid(self.site.location, self.site)

        # self.hub = Hub(id=1, location=(0, 0), radius=11)

        # self.grid.add_object_to_grid(self.hub.location, self.hub)

        self.agents = []

        bound = np.ceil((self.num_agents * 1.0) / len(self.xmlstrings))

        j = 0
        # Create agents
        for i in range(self.num_agents):
            # print (i, j, self.xmlstrings[j])
            a = SimForgAgent(i, self, xmlstring=self.xmlstrings[j])
            self.schedule.add(a)
            # Add the agent to a random grid cell
            # x = self.random.randint(
            #    -self.grid.width / 2, self.grid.width / 2)
            x = 0
            # y = self.random.randint(
            #    -self.grid.height / 2, self.grid.height / 2)
            y = 0

            a.location = (x, y)
            self.grid.add_object_to_grid((x, y), a)
            a.operation_threshold = 2  # self.num_agents // 10
            self.agents.append(a)

            if (i + 1) % bound == 0:
                j += 1

        # Add equal number of food source
        # for i in range(20):
        #    f = Food(i, location=(-29, -29), radius=5)
        #    self.grid.add_object_to_grid(f.location, f)
            # print (i,x,y)

    def create_environment_object(self, jsondata, obj):
        """Create env from jsondata."""
        name = obj.__name__.lower()
        temp_list = []
        i = 0
        for json_object in jsondata[name]:
            location = (json_object["x"], json_object["y"])
            if "q_value" in json_object:
                temp_obj = obj(
                    i, location, json_object["radius"], q_value=json_object[
                        "q_value"])
            else:
                temp_obj = obj(i, location, json_object["radius"])

            self.grid.add_object_to_grid(location, temp_obj)
            temp_list.append(temp_obj)
            i += 1
        return temp_list

    def build_environment_from_json(self):
        """Build env from jsondata."""
        jsondata = JsonData.load_json_file(filename)
        # Create a instance of JsonData to store object that
        # needs to be sent to UI
        self.render = JsonData()
        self.render.objects = {}

        for name in jsondata.keys():
            obj = eval(name.capitalize())
            self.render.objects[name] = self.create_environment_object(
                jsondata, obj)

        self.hub = self.render.objects['hub'][0]
        try:
            self.site = self.render.objects['sites'][0]
            self.foods = []
            for i in range(self.num_agents * 1):
                f = Food(
                    i, location=self.site.location, radius=self.site.radius)
                f.agent_name = None
                self.grid.add_object_to_grid(f.location, f)
                self.foods.append(f)
        except KeyError:
            pass

        if self.viewer:
            self.ui = UI(
                (self.width, self.height), [self.hub], self.agents,
                [self.site], food=self.foods)

    def step(self):
        """Step through the environment."""
        # Gather info from all the agents
        # self.gather_info()
        # Next step
        self.schedule.step()
        # Increment the step count
        self.stepcnt += 1
        if self.viewer:
            self.ui.step()

    def find_higest_performer(self):
        """Find the best agent."""
        fitness = self.agents[0].individual[0].fitness
        fittest = self.agents[0]
        for agent in self.agents:
            if agent.individual[0].fitness > fitness:
                fittest = agent
        return fittest

    def find_higest_food_collector(self):
        """Find the best agent to collect food."""
        fitness = self.agents[0].food_collected
        fittest = self.agents[0]
        for agent in self.agents:
            if agent.food_collected > fitness:
                fittest = agent
        return fittest

    def detect_food_moved(self):
        """Detect food moved."""
        grid = self.grid
        food_loc = self.site.location
        neighbours = grid.get_neighborhood(food_loc, 10)
        food_objects = grid.get_objects_from_list_of_grid('Food', neighbours)

        # print (food_objects)
        return food_objects

    def food_in_hub(self):
        """Find amount of food in hub."""
        grid = self.grid
        food_loc = self.hub.location
        neighbours = grid.get_neighborhood(food_loc, 10)
        food_objects = grid.get_objects_from_list_of_grid('Food', neighbours)
        return len(food_objects)

    def food_in_loc(self, loc):
        """Find amount of food in hub."""
        grid = self.grid
        neighbours = grid.get_neighborhood(loc, 10)
        food_objects = grid.get_objects_from_list_of_grid('Food', neighbours)
        return food_objects

class SimCTModel(Model):
    """A environemnt to model swarms."""

    def __init__(
            self, N, width, height, grid=10, iter=100000,
            xmlstrings=None, seed=None, viewer=False, pname=None,
            expname='COTSimulation', agent='SimCTAgent'):
        """Initialize the attributes."""
        if seed is None:
            super(SimCTModel, self).__init__(seed=None)
        else:
            super(SimCTModel, self).__init__(seed)

        self.runid = datetime.datetime.now().strftime(
            "%s") + str(self.random.randint(1, 1000, 1)[0])

        if pname is None:
            self.pname = os.getcwd() + '/' + self.runid + expname
        else:
            self.pname = pname + '/' + self.runid + expname

        self.width = width
        self.height = height
        self.stepcnt = 1
        self.iter = iter
        self.xmlstrings = xmlstrings

        self.viewer = viewer

        # Create db connection
        connect = Connect('swarm', 'swarm', 'swarm', 'localhost')
        self.connect = connect.tns_connect()

        # Fill out the experiment table
        self.experiment = Experiment(
            self.connect, self.runid, N, seed, expname,
            iter, width, height, grid, phenotype=xmlstrings[0])

        self.experiment.insert_experiment_simulation()

        self.sn = self.experiment.sn

        # Create a folder to store results
        os.mkdir(self.pname)

        self.num_agents = N

        self.grid = Grid(width, height, grid)

        self.schedule = SimultaneousActivation(self)

        self.agents = []

        bound = np.ceil((self.num_agents * 1.0) / len(self.xmlstrings))

        j = 0
        # Create agents
        for i in range(self.num_agents):
            # print (i, j, self.xmlstrings[j])
            a = eval(agent)(i, self, xmlstring=self.xmlstrings[j])
            self.schedule.add(a)
            # Add the agent to a random grid cell
            # x = self.random.randint(
            #    -self.grid.width / 2, self.grid.width / 2)
            x = 0
            # y = self.random.randint(
            #    -self.grid.height / 2, self.grid.height / 2)
            y = 0

            a.location = (x, y)
            self.grid.add_object_to_grid((x, y), a)
            a.operation_threshold = 2  # self.num_agents // 10
            self.agents.append(a)

            if (i + 1) % bound == 0:
                j += 1

    def create_environment_object(self, jsondata, obj):
        """Create env from jsondata."""
        name = obj.__name__.lower()
        temp_list = []
        i = 0
        for json_object in jsondata[name]:
            location = (json_object["x"], json_object["y"])
            if "q_value" in json_object:
                temp_obj = obj(
                    i, location, json_object["radius"], q_value=json_object[
                        "q_value"])
            else:
                temp_obj = obj(i, location, json_object["radius"])

            self.grid.add_object_to_grid(location, temp_obj)
            temp_list.append(temp_obj)
            i += 1
        return temp_list

    def build_environment_from_json(self):
        """Build env from jsondata."""
        jsondata = JsonData.load_json_file(filename)
        # Create a instance of JsonData to store object that
        # needs to be sent to UI
        self.render = JsonData()
        self.render.objects = {}

        for name in jsondata.keys():
            obj = eval(name.capitalize())
            self.render.objects[name] = self.create_environment_object(
                jsondata, obj)

        self.hub = self.render.objects['hub'][0]
        try:
            self.foods = []
            self.site = self.render.objects['sites'][0]
            food_radius = self.random.randint(20, 30)
            for i in range(self.num_agents):
                f = Food(
                    i, location=self.site.location,
                    radius=food_radius)
                f.agent_name = None
                self.grid.add_object_to_grid(f.location, f)
                self.foods.append(f)
        except KeyError:
            pass

        if self.viewer:
            self.ui = UI(
                (self.width, self.height), [self.hub], self.agents,
                [self.site], food=self.foods)

    def step(self):
        """Step through the environment."""
        # Gather info from all the agents
        # self.gather_info()
        # Next step
        self.schedule.step()
        # Increment the step count
        self.stepcnt += 1
        if self.viewer:
            self.ui.step()

    def find_higest_performer(self):
        """Find the best agent."""
        fitness = self.agents[0].individual[0].fitness
        fittest = self.agents[0]
        for agent in self.agents:
            if agent.individual[0].fitness > fitness:
                fittest = agent
        return fittest

    def find_higest_food_collector(self):
        """Find the best agent to collect food."""
        fitness = self.agents[0].food_collected
        fittest = self.agents[0]
        for agent in self.agents:
            if agent.food_collected > fitness:
                fittest = agent
        return fittest

    def detect_food_moved(self):
        """Detect food moved."""
        grid = self.grid
        food_loc = self.site.location
        neighbours = grid.get_neighborhood(food_loc, 10)
        food_objects = grid.get_objects_from_list_of_grid('Food', neighbours)

        # print (food_objects)
        return food_objects

    def food_in_hub(self):
        """Find amount of food in hub."""
        grid = self.grid
        food_loc = self.hub.location
        neighbours = grid.get_neighborhood(food_loc, 10)
        food_objects = grid.get_objects_from_list_of_grid('Food', neighbours)
        return len(food_objects)

    def food_in_loc(self, loc):
        """Find amount of food in hub."""
        grid = self.grid
        neighbours = grid.get_neighborhood(loc, 10)
        food_objects = grid.get_objects_from_list_of_grid('Food', neighbours)
        return food_objects

class SimNMModel(Model):
    """A environemnt to model swarms."""

    def __init__(
            self, N, width, height, grid=10, iter=100000,
            xmlstrings=None, seed=None, viewer=False, pname=None,
            expname='NMSimulation', agent='SimNMAgent'):
        """Initialize the attributes."""
        if seed is None:
            super(SimNMModel, self).__init__(seed=None)
        else:
            super(SimNMModel, self).__init__(seed)

        # self.runid = datetime.datetime.now().strftime(
        #    "%s") + str(self.random.randint(1, 1000, 1)[0])

        self.runid = datetime.datetime.now().timestamp()
        self.runid = str(self.runid).replace('.', '')

        if pname is None:
            self.pname = os.getcwd() + '/' + self.runid + expname
        else:
            self.pname = pname + '/' + self.runid + expname

        self.width = width
        self.height = height
        self.stepcnt = 1
        self.iter = iter
        self.xmlstrings = xmlstrings

        self.viewer = viewer

        # Create db connection
        connect = Connect('swarm', 'swarm', 'swarm', 'localhost')
        self.connect = connect.tns_connect()

        # Fill out the experiment table
        self.experiment = Experiment(
            self.connect, self.runid, N, seed, expname,
            iter, width, height, grid, phenotype=xmlstrings[0])

        self.experiment.insert_experiment_simulation()

        self.sn = self.experiment.sn

        # Create a folder to store results
        os.mkdir(self.pname)

        self.num_agents = N

        self.grid = Grid(width, height, grid)

        self.schedule = SimultaneousActivation(self)

        self.agents = []

        bound = np.ceil((self.num_agents * 1.0) / len(self.xmlstrings))

        j = 0
        # Create agents
        for i in range(self.num_agents):
            # print (i, j, self.xmlstrings[j])
            a = eval(agent)(i, self, xmlstring=self.xmlstrings[j])
            self.schedule.add(a)
            # Add the agent to a random grid cell
            # x = self.random.randint(
            #    -self.grid.width / 2, self.grid.width / 2)
            x = 0
            # y = self.random.randint(
            #    -self.grid.height / 2, self.grid.height / 2)
            y = 0

            a.location = (x, y)
            self.grid.add_object_to_grid((x, y), a)
            a.operation_threshold = 2  # self.num_agents // 10
            self.agents.append(a)

            if (i + 1) % bound == 0:
                j += 1

    def create_environment_object(self, jsondata, obj):
        """Create env from jsondata."""
        name = obj.__name__.lower()
        temp_list = []
        i = 0
        for json_object in jsondata[name]:
            location = (json_object["x"], json_object["y"])
            if "q_value" in json_object:
                temp_obj = obj(
                    i, location, json_object["radius"], q_value=json_object[
                        "q_value"])
            else:
                temp_obj = obj(i, location, json_object["radius"])

            self.grid.add_object_to_grid(location, temp_obj)
            temp_list.append(temp_obj)
            i += 1
        return temp_list

    def build_environment_from_json(self):
        """Build env from jsondata."""
        jsondata = JsonData.load_json_file(filename)
        # Create a instance of JsonData to store object that
        # needs to be sent to UI
        self.render = JsonData()
        self.render.objects = {}

        for name in jsondata.keys():
            obj = eval(name.capitalize())
            self.render.objects[name] = self.create_environment_object(
                jsondata, obj)

        self.hub = self.render.objects['hub'][0]
        try:
            self.debris = []
            for i in range(self.num_agents):
                dx, dy = self.random.randint(1, 10, 2)
                dx = self.hub.location[0] + dx
                dy = self.hub.location[1] + dy
                d = Debris(
                    i, location=(dx, dy),
                    radius=5)
                d.agent_name = None
                self.grid.add_object_to_grid(d.location, d)
                self.debris.append(d)
        except KeyError:
            pass

        # Create a place for the agents to drop the derbis
        try:
            self.obstacles = []
            for i in range(1):
                dx, dy = self.random.randint(5, 10, 2)
                dx = self.hub.location[0] + 25 + dx
                dy = self.hub.location[1] + 25 + dy
                o = Obstacles(id=i, location=(dx, dy), radius=10)
                self.grid.add_object_to_grid(o.location, o)
                self.obstacles.append(o)
        except AttributeError:
            pass

        if self.viewer:
            self.ui = UI(
                (self.width, self.height), [self.hub], self.agents,
                [], food=[], derbis=self.debris)

    def step(self):
        """Step through the environment."""
        # Gather info from all the agents
        # self.gather_info()
        # Next step
        self.schedule.step()
        # Increment the step count
        self.stepcnt += 1
        if self.viewer:
            self.ui.step()

    def find_higest_performer(self):
        """Find the best agent."""
        fitness = self.agents[0].individual[0].fitness
        fittest = self.agents[0]
        for agent in self.agents:
            if agent.individual[0].fitness > fitness:
                fittest = agent
        return fittest

    def find_higest_debris_collector(self):
        """Find the best agent to collect debris."""
        fitness = self.agents[0].debris_collected
        fittest = self.agents[0]
        for agent in self.agents:
            if agent.food_collected > fitness:
                fittest = agent
        return fittest

    def detect_debris_moved(self):
        """Detect debris moved."""
        grid = self.grid
        debris_loc = self.hub.location
        neighbours = grid.get_neighborhood(debris_loc, 30)
        debris_objects = grid.get_objects_from_list_of_grid(
            'Debris', neighbours)

        return debris_objects

    def debris_around_hub(self):
        """Find amount of debris around hub."""
        grid = self.grid
        food_loc = self.hub.location
        neighbours = grid.get_neighborhood(food_loc, 20)
        food_objects = grid.get_objects_from_list_of_grid(
            'Debris', neighbours)
        return len(food_objects)

    def debris_in_loc(self, loc):
        """Find amount of debris in a location."""
        grid = self.grid
        neighbours = grid.get_neighborhood(loc, 10)
        debris_objects = grid.get_objects_from_list_of_grid(
            'Debris', neighbours)
        return debris_objects

    def debris_cleaned(self):
        """Find amount of debris cleaned."""
        grid = self.grid
        debris_objects = []
        for obstacle in self.obstacles:
            neighbours = grid.get_neighborhood(
                obstacle.location, obstacle.radius)
            debris_objects += grid.get_objects_from_list_of_grid(
                'Debris', neighbours)
        return list(set(debris_objects))

max_stars_repo_name: aadeshnpn/swarm

max_issues_repo_name: aadeshnpn/swarm

max_forks_repo_name: aadeshnpn/swarm


text:
#!/usr/bin/python
# -*- coding: UTF-8 -*-
"""
Tencent is pleased to support the open source community by making Metis available.
Copyright (C) 2018 THL A29 Limited, a Tencent company. All rights reserved.
Licensed under the BSD 3-Clause License (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
https://opensource.org/licenses/BSD-3-Clause
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
"""

import numpy as np
from app.config.errorcode import *
from app.config.common import *

def is_standard_time_series(time_series, window=180):
    """
    Check the length of time_series. If window = 180, then the length of time_series should be 903.
    The mean value of last window should be larger than 0.

    :param time_series: the time series to check, like [data_c, data_b, data_a]
    :type time_series: pandas.Series
    :param window: the length of window
    :return: True or False
    :return type: boolean
    """
    if len(time_series) == 5 * window + 3 and np.mean(time_series[(4 * window + 2):]) > 0:
        return True
    else:
        return False

def split_time_series(time_series, window=180):
    """
    Spilt the time_series into five parts. Each has a length of window + 1

    :param time_series: [data_c, data_b, data_a]
    :param window: the length of window
    :return: spilt list [[data_c_left], [data_c_right], [data_b_left], [data_b_right], [data_a]]
    """
    data_c_left = time_series[0:(window + 1)]
    data_c_right = time_series[window:(2 * window + 1)]
    data_b_left = time_series[(2 * window + 1):(3 * window + 2)]
    data_b_right = time_series[(3 * window + 1):(4 * window + 2)]
    data_a = time_series[(4 * window + 2):]
    split_time_series = []
    split_time_series.append(data_c_left)
    split_time_series.append(data_c_right)
    split_time_series.append(data_b_left)
    split_time_series.append(data_b_right)
    split_time_series.append(data_a)
    return split_time_series

def normalize_time_series(split_time_series):
    """
    Normalize the split_time_series.

    :param split_time_series: [[data_c_left], [data_c_right], [data_b_left], [data_b_right], [data_a]]
    :return: all list / mean(split_time_series)
    """
    value = np.mean(split_time_series[4])
    if value > 1:
        normalized_data_c_left = list(split_time_series[0] / value)
        normalized_data_c_right = list(split_time_series[1] / value)
        normalized_data_b_left = list(split_time_series[2] / value)
        normalized_data_b_right = list(split_time_series[3] / value)
        normalized_data_a = list(split_time_series[4] / value)
    else:
        normalized_data_c_left = split_time_series[0]
        normalized_data_c_right = split_time_series[1]
        normalized_data_b_left = split_time_series[2]
        normalized_data_b_right = split_time_series[3]
        normalized_data_a = split_time_series[4]
    normalized_split_time_series = []
    normalized_split_time_series.append(normalized_data_c_left)
    normalized_split_time_series.append(normalized_data_c_right)
    normalized_split_time_series.append(normalized_data_b_left)
    normalized_split_time_series.append(normalized_data_b_right)
    normalized_split_time_series.append(normalized_data_a)
    return normalized_split_time_series

def build_ret_data(ret_code, data=""):
    return {"code": ret_code, "msg": ERR_CODE[ret_code], "data": data}

def validate_value(data):
    if isinstance(data, unicode):
        if len(data) > INPUT_LEN_ENG_MAX:
            return CHECK_PARAM_FAILED
    elif isinstance(data, str):
        if len(data) > INPUT_LEN_ENG_MAX:
            return CHECK_PARAM_FAILED
    elif isinstance(data, list):
        if len(data) > INPUT_LIST_LEN_MAX:
            return CHECK_PARAM_FAILED
        for item in data:
            ret_code = validate_value(item)
            if ret_code != 0:
                return ret_code
    return 0

def check_value(data):
    if 'attrId' in data:
        ret_code = validate_value(data['attrId'])
        if ret_code != 0:
            return CHECK_PARAM_FAILED, "attrId too long"
    if 'attrName' in data:
        ret_code = validate_value(data['attrName'])
        if ret_code != 0:
            return CHECK_PARAM_FAILED, "attrName too long"
    if 'viewId' in data:
        ret_code = validate_value(data['viewId'])
        if ret_code != 0:
            return CHECK_PARAM_FAILED, "viewId too long"
    if 'viewName' in data:
        ret_code = validate_value(data['viewName'])
        if ret_code != 0:
            return CHECK_PARAM_FAILED, "viewName too long"
    if 'itemPerPage' in data:
        if data['itemPerPage'] > INPUT_ITEM_PER_PAGE_MAX:
            return CHECK_PARAM_FAILED, "itemPerPage too big"
    if 'beginTime' in data:
        if len(str(data['beginTime'])) > INPUT_LEN_ENG_MAX:
            return CHECK_PARAM_FAILED, "beginTime too long"
    if 'endTime' in data:
        if len(str(data['endTime'])) > INPUT_LEN_ENG_MAX:
            return CHECK_PARAM_FAILED, "endTime too long"
    if 'updateTime' in data:
        if len(str(data['updateTime'])) > INPUT_LEN_ENG_MAX:
            return CHECK_PARAM_FAILED, "updateTime too long"
    if 'source' in data:
        ret_code = validate_value(data['source'])
        if ret_code != 0:
            return CHECK_PARAM_FAILED, "source too long"
    if 'trainOrTest' in data:
        ret_code = validate_value(data['source'])
        if ret_code != 0:
            return CHECK_PARAM_FAILED, "trainOrTest too long"
    if 'positiveOrNegative' in data:
        ret_code = validate_value(data['positiveOrNegative'])
        if ret_code != 0:
            return CHECK_PARAM_FAILED, "positiveOrNegative too long"
    if 'window' in data:
        if len(str(data['window'])) > INPUT_LEN_ENG_MAX:
            return CHECK_PARAM_FAILED, "window"
    if 'dataTime' in data:
        if len(str(data['dataTime'])) > INPUT_LEN_ENG_MAX:
            return CHECK_PARAM_FAILED, "dataTime too long"
    if 'dataC' in data:
        if len(str(data['dataC'])) > VALUE_LEN_MAX:
            return CHECK_PARAM_FAILED, "dataC too long"
    if 'dataB' in data:
        if len(str(data['dataB'])) > VALUE_LEN_MAX:
            return CHECK_PARAM_FAILED, "dataB too long"
    if 'dataA' in data:
        if len(str(data['dataA'])) > VALUE_LEN_MAX:
            return CHECK_PARAM_FAILED, "dataA too long"
    return 0, ""

ext: py

max_stars_repo_name: wanghuaway/Metis

max_issues_repo_name: wanghuaway/Metis

max_forks_repo_name: wanghuaway/Metis

reason: import numpy


text:
"""Get grid array from nonogram image"""
# imports
import os
import cv2
import numpy as np
import pytesseract

CUSTOM_CONFIG = r'--oem 3 --psm 6 outputbase digits'

def get_image(path):
    """Get image given path"""
    return cv2.imread(path)

def get_image_name(path):
    """Get image name"""
    return os.path.basename(path)

def get_grayscale(image):
    """Grayscale image"""
    return cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

def base_transformation(image):
    """Complete base transformations"""
    # make image gray
    gray = get_grayscale(image)

    # apply Gaussian Blur to reduce noise
    image_blurred = cv2.GaussianBlur(gray, (3, 3), 0)

    return image_blurred

def get_largest_contour_bounding_box(edges):
    """Get largest contour"""
    # find contours
    contours, _ = cv2.findContours(
        edges, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)

    # finds largest contour
    max_contour = max(contours, key=cv2.contourArea)

    x_coord, y_coord, width, height = cv2.boundingRect(max_contour)
    return (x_coord, y_coord, width, height)

def transform_image(image):
    """Transform image to detect edges"""
    # apply base transformations
    image_blurred = base_transformation(image)

    # get edges using Canny method
    edges = cv2.Canny(image_blurred, 100, 300, apertureSize=3)

    # finds largest contour
    x_coord, y_coord, width, height = get_largest_contour_bounding_box(edges)

    # creates image with contour
    cropped_image = image[y_coord:y_coord + height, x_coord:x_coord + width]

    return cropped_image

def get_binary_image(image):
    """Converts image to black and white"""
    gray_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    (thresh, black_white_image) = cv2.threshold(
        gray_image, 150, 255, cv2.THRESH_BINARY_INV)
    return thresh, black_white_image

def get_num_rows_cols_from_image(image):
    """Get cell dimensions"""
    # transform image
    gray_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    (_, black_white_image) = cv2.threshold(
        gray_image, 150, 255, cv2.THRESH_BINARY)
    kernel = np.ones((1, 5), np.uint8)
    erosion = cv2.erode(black_white_image, kernel, iterations=1)

    for row in range(erosion.shape[0]):
        count = 0
        temp_color = erosion[row][0]
        for col in range(erosion.shape[1]):
            current_color = erosion[row][col]
            if temp_color != current_color:
                if temp_color == 255:
                    temp_color = 0
                else:
                    temp_color = 255
                count += 1
        if count > 2:
            pixel_row_for_col_counting = row + 1
            break

    temp_color = 0
    count = 0
    for col in range(erosion.shape[1]):
        current_color = erosion[pixel_row_for_col_counting][col]
        if temp_color != current_color:
            if temp_color == 255:
                temp_color = 0
            else:
                temp_color = 255
            count += 1

    num_cols = int(round(count / 2))
    approximate_col_side_length = erosion.shape[1] / num_cols
    num_rows = int(round(erosion.shape[0] / approximate_col_side_length))
    return num_rows, num_cols

def draw_improved_grid_lines(image, num_rows, num_cols):
    """Draws single pixel grid lines with no noise to get contours from"""
    row_height = image.shape[0] / num_rows
    col_width = image.shape[1] / num_cols

    for row in range(num_rows + 1):
        cv2.line(image, (0, int(row * row_height)),
                 (image.shape[1], int(row * row_height)), (255, 0, 0), 1, 1)

    for col in range(num_cols + 1):
        cv2.line(image, (int(col * col_width), 0),
                 (int(col * col_width), image.shape[0]), (255, 0, 0), 1, 1)

    cv2.line(image, (0, image.shape[0] -
                     1), (image.shape[1], image.shape[0] -
                          1), (255, 0, 0), 1, 1)
    cv2.line(image, (image.shape[1] -
                     1, 0), (image.shape[1] -
                             1, image.shape[0]), (255, 0, 0), 1, 1)

    return image

def check_cell_for_number(cell):
    """Checks cell for number"""
    pytesseract.pytesseract.tesseract_cmd = 'Tesseract-OCR/tesseract'
    number = pytesseract.image_to_string(cell, config=CUSTOM_CONFIG)
    if number != "":
        number = int(number)
    return number

def remove_noise(image):
    """Remove noise"""
    return cv2.medianBlur(image, 3)

def opening(image):
    """Opening"""
    kernel = np.ones((1, 3), np.uint8)
    return cv2.morphologyEx(image, cv2.MORPH_OPEN, kernel)

def get_array_from_grid(image, num_rows, num_cols):
    """Gets cells from grid"""
    image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    image = cv2.threshold(
        image, 150, 255, cv2.THRESH_BINARY)[1]
    row_height = image.shape[0] / num_rows
    col_width = image.shape[1] / num_cols
    final_array = []
    for row in range(num_rows):
        for col in range(num_cols):
            cell = image[int(row *
                             row_height):int(min((row +
                                                  1) *
                                                 row_height, image.shape[0])),
                         int(col *
                             col_width) + 4:int(min((col +
                                                     1) *
                                                    col_width, image.shape[1])) - 2]

            rnoise = remove_noise(cell)
            processed_cell = opening(rnoise)
            number = check_cell_for_number(processed_cell)
            final_array.append(number)
    return final_array

def organize_array_by_rows(unformatted_array, num_cols):
    """Take unformatted array and make grid array"""
    num_rows = int(len(unformatted_array) / num_cols)
    array = []
    for row in range(num_rows):
        array.append(unformatted_array[row * num_cols:(row + 1) * num_cols])
    return array

def fix_array(array):
    """Have user manually fix any errors from OCR"""
    fixed = False
    while not fixed:
        for row in array:
            for col in row:
                print(str(col) + (2 - len(str(col))) * " " + "|", end=" ")
            print("\n{arg}\n".format(arg=((len(row) * 4) * "-")))
        is_fixed = input("Enter Y if grid is correct. Else enter N. \n")
        if is_fixed.lower() == "y":
            fixed = True
        elif is_fixed.lower() == "n":
            row = input("Enter row of incorrect cell \n")
            if not row.isdigit():
                print("Digit not entered, will start again \n")
                continue
            row = int(row)
            if not 0 < row < len(array):
                print("Digit out of bounds, start again")
                continue
            for index, col in enumerate(array[row - 1]):
                print("Position - " + str(index + 1) + ", Value - " + str(col))
            col = input("Enter position of incorrect cell \n")
            if not col.isdigit():
                print("Digit not entered, will start again \n")
                continue
            col = int(col)
            if not 0 < col < len(array[row]):
                print("Digit out of bounds, start again")
                continue
            print("Incorrect cell currently reads: " +
                  str(array[row - 1][int(col) - 1]) + "\n")
            new_cell = input(
                "Enter correct cell entry. If blank, press enter \n")
            if not new_cell.isdigit() and new_cell != "":
                print("Entry is not digit or blank, will start again \n")
                continue
            if new_cell.isdigit():
                new_cell = int(new_cell)
            array[row - 1][int(col) - 1] = new_cell
        else:
            print('Invalid input, will start again \n')
    return array

def get_num_rows_cols(array):
    """Get number of rows that have col data."""
    row_number = 0
    is_last_row = False
    while not is_last_row:
        is_last_row = check_if_last_row(array, row_number)
        row_number += 1
    return row_number

def check_if_last_row(array, row_number):
    """Loop through row for get_num_rows_cols."""
    for cell in enumerate(array[row_number]):
        if array[row_number][cell] != '':
            if array[row_number + 1][cell] != '':
                return False
    return True

def get_row_col_array(array, rows_skipped):
    """Get col array from grid."""
    ver_hor_array = []
    total_rows = len(array)
    for row in range(rows_skipped, total_rows):
        individual_row_array = []
        for cell in enumerate(array[row]):
            if array[row][cell] != '':
                individual_row_array.append(array[row][cell])
        ver_hor_array.append(individual_row_array)
    return ver_hor_array

def get_ver_hor_array_as_string(ver_hor_array):
    """Get vertical or horizontal array as string.

    Args:
        ver_hor_array: either ver or hor array

    Returns:
        stringified version
    """
    ver_hor_string = '['
    for row_num, _ in enumerate(ver_hor_array):
        ver_hor_string = '{arg}['.format(arg=ver_hor_string)
        for cell in enumerate(ver_hor_array[row_num]):
            ver_hor_string = '{verhorstr}{indstr},'.format(
                verhorstr=ver_hor_string,
                indstr=str(ver_hor_array[row_num][cell]),
            )
        ver_hor_string = ver_hor_string[:-1]
        ver_hor_string = '{str}],'.format(str=ver_hor_string)
    ver_hor_string = ver_hor_string[:-1]
    return '{str}]'.format(str=ver_hor_string)

def get_teal_string(vertical, horizontal):
    """Format string for use in teal online tool.

    Args:
        vertical: vertical axis array
        horizontal: horizontal axis array

    Returns:
        Returns teal string for use online
    """
    ver_string = get_ver_hor_array_as_string(vertical)
    hor_string = get_ver_hor_array_as_string(horizontal)
    beginning_str = '{"ver":'
    mid_str = ',"hor":'
    end_str = '}'
    return '{begstr}{verarg}{midstr}{horarg}{endstr}'.format(
        begstr=beginning_str,
        midstr=mid_str,
        endstr=end_str,
        verarg=ver_string,
        horarg=hor_string,
    )

def main(image_path):
    """Output array from image.

    Args:
        image_path: path to image

    Returns:
        teal string to be used online
    """

if __name__ == '__main__':
    nonogram_image_path = 'tests/input_images/image1.jpg'
    nonogram_image = get_image(nonogram_image_path)
    transformed_image = transform_image(nonogram_image)
    number_of_rows, number_of_cols = get_num_rows_cols_from_image(
        transformed_image,
    )
    finished_array = get_array_from_grid(
        transformed_image,
        number_of_rows,
        number_of_cols,
    )
    grid_array = organize_array_by_rows(finished_array, number_of_cols)
    corrected_grid_array = fix_array(grid_array)
    transposed_array = list(map(list, zip(*corrected_grid_array)))
    num_rows = get_num_rows_cols(grid_array)
    num_cols = get_num_rows_cols(transposed_array)
    ver_array = get_row_col_array(grid_array, num_cols)
    hor_array = get_row_col_array(transposed_array, num_rows)
    teal_string = get_teal_string(ver_array, hor_array)


text:
import numpy as np
from matplotlib import pyplot as plt
import pandas as pd
from scipy.optimize import fsolve
inputFilePath = 'Rawdata.xlsx'
dfIn = pd.read_excel(inputFilePath)
Theta=[]
for i in range (35000):
    Theta.append(dfIn.loc[i,0])

min=np.min(Theta)
max=np.max(Theta)

thetaf = [((ele)*3.14)/(max)+1.57 for ele in Theta]
thetaprimef=[]
# thetaprimef = [((ele)*4*3.14)/((max))-(2*3.14) for ele in Theta]
for ele in Theta:
    if(ele>0):
       thetaprimef.append(((ele)*4*3.14)/((max))-(2*3.14))
    else:
        thetaprimef.append(((ele)*4*3.14)/((max))+(2*3.14))

dfOut = pd.DataFrame(thetaprimef)

outputFilePath = 'thetaprimef.xlsx'

dfOut.to_excel(outputFilePath, index=False)

'''plt.plot(thetaf)
plt.xlabel('Time [non-dim]')
plt.ylabel('Natural logarithm of data difference [a.u.]')
plt.title('Derivative')
plt.figure(dpi=300)
plt.show()'''

max_stars_repo_path: 4.Chaos IX - Road Generation/Utils/Theta1.py

max_stars_repo_name: AidinMomtaz/Chaos-Projects

max_issues_repo_name: AidinMomtaz/Chaos-Projects

max_forks_repo_name: AidinMomtaz/Chaos-Projects

reason: import numpy,from scipy


text:
import argparse
import random
from crowd_dataset import CrowdDataset
import matplotlib
matplotlib.use('Agg')
from matplotlib import pyplot as plt
import cv2
import numpy as np
import os
import random, string
import math
import pickle
from collections import OrderedDict
import torch
from torch import nn as nn, optim as optim
from torch.autograd import Variable

import datetime
import scipy.stats as ss

from pdb import set_trace as bp
from models import BSDR_Net
from models import load_rot_model_blocks, check_BN_no_gradient_change
from models import check_conv_no_gradient_change, set_batch_norm_to_eval
from models import load_net
from noisy_gts import create_noisy_gt
from models import NRN
import warnings
warnings.filterwarnings("ignore")

parser = argparse.ArgumentParser(description='PyTorch BSDR Training')
parser.add_argument('--epochs', default=1000, type=int, metavar='N',
                    help='number of total epochs to run')
parser.add_argument('--gpu', default=1, type=int,
                    help='GPU number')
parser.add_argument('-b', '--batch-size', default=4, type=int, metavar='N',
                    help='mini-batch size (default: 4),only used for train')
parser.add_argument('--patches', default=1, type=int, metavar='N',
                    help='number of patches per image')
parser.add_argument('--dataset', default="parta", type=str,
                    help='dataset to train on')
parser.add_argument('--lr', '--learning-rate', default=1e-3, type=float,
                    metavar='LR', help='initial learning rate')
parser.add_argument('--momentum', default=0.9, type=float,
                    metavar='M', help='momentum')
parser.add_argument('--weight-decay', '--wd', default=1e-4, type=float, metavar='W',
                    help='weight decay (default: 1e-4)')
parser.add_argument("--use-noisygt", type=bool, nargs='?',
                        const=True, default=False,
                        help="Activate nice mode.")
parser.add_argument('--count-thresh', default='1250', type=str, help="Define the count threshold for categories")

def log(f, txt, do_print=1):
    txt = str(datetime.datetime.now()) + ': ' + txt
    if do_print == 1:
        print(txt)
    f.write(txt + '\n')

# Get the filename for the model stored after 'epochs_over' epochs got over
def get_filename(net_name, epochs_over):
    return net_name + "_epoch_" + str(epochs_over) + ".pth"

def save_checkpoint(state, fdir, name='checkpoint.pth'):
    filepath = os.path.join(fdir, name)
    torch.save(state, filepath)

def print_graph(maps, title, save_path):
    fig = plt.figure()
    st = fig.suptitle(title)
    for i, (map, args) in enumerate(maps):
        plt.subplot(1, len(maps), i + 1)
        if len(map.shape) > 2 and map.shape[0] == 3:
            # bp()
            plt.imshow(map.transpose((1, 2, 0)).astype(np.uint8),aspect='equal', **args)
            # bp()
        else:
            # bp()

            plt.imshow(map, aspect='equal', **args)
            plt.colorbar()
            # bp()
            plt.axis('off')
    plt.savefig(save_path + ".png", bbox_inches='tight', pad_inches = 0)
    fig.clf()
    plt.clf()
    plt.close()

excluded_layers = ['conv4_1', 'conv4_2', 'conv5_1']

def train_function(Xs, Ys, network, optimizer):
    """
    Training Prediction and Loss computation
    Parameters
    ----------
    Xs : input images (B,3,h,w)
    Ys : ground truth (B,1,h/8,w/8)
    network : BSDR object
    optimizer : training optimizer
    """

    network = network.cuda()
    optimizer.zero_grad()

    X = torch.autograd.Variable(torch.from_numpy(Xs)).cuda()
    Y = torch.autograd.Variable(torch.FloatTensor(Ys)).cuda()

    outputs = network(X)

    loss = 0.0
    loss_criterion = nn.MSELoss(size_average=True)
    loss =  loss_criterion(outputs, Y)

    assert(loss.grad_fn != None)
    loss.backward()
    optimizer.step()
    return loss.item()

@torch.no_grad()
def test_function(X, Y, network):
    """
    Evaluation of network on test and valid set
    Parameters
    ----------
    X : input images (B,3,h,w)
    Y : ground truth (B,1,h/8,w/8)
    network : BSDR object
    """
    X = torch.autograd.Variable(torch.from_numpy(X)).cuda()
    Y = torch.autograd.Variable(torch.from_numpy(Y)).cuda()

    network = network.cuda()
    network.eval()
    output = network(X) # (B,1,h,w)
    loss = 0.0
    loss_criterion = nn.MSELoss(size_average=True)
    # bp()

    loss = loss_criterion(output, Y)

    count_error = torch.abs(torch.sum(Y.view(Y.size(0), -1), dim=1) - torch.sum(output.view(output.size(0), -1), dim=1))
    network.train()
    network = set_batch_norm_to_eval(network)
    return loss.item(), output.cpu().detach().numpy(), count_error.cpu().detach().numpy()

def test_network(dataset, set_name, network, print_output=False):
    """
    Main loop for evaluation of BSDR network
    Parameters
    ----------
    dataset : dataset object for retrieving data from test/valid set
    set-name : choose the test / valid set
    network : BSDR object
    print_output : determine to dump predictions
    """
    if isinstance(print_output, str):
        print_path = print_output
    elif isinstance(print_output, bool) and print_output:
        print_path = model_save_dir+'/dump'
    else:
        print_path = None

    loss_list = []
    count_error_list = []
    for idx, data in enumerate(dataset.test_get_data(set_name)):
        image_name, Xs, Ys = data
        image = Xs[0].transpose((1, 2, 0))
        image = cv2.resize(image, (image.shape[1] // output_downscale, image.shape[0] // output_downscale))

        loss, pred_dmap, count_error = test_function(Xs, Ys, network)
        # bp()
        max_val = max(np.max(pred_dmap[0, 0].reshape(-1)), np.max(Ys[0, 0].reshape(-1)))
        maps = [(np.transpose(image,(2,0,1)), {}),
                (pred_dmap[0,0], {'cmap': 'jet', 'vmin': 0., 'vmax': max_val}),
                (Ys[0,0], {'cmap': 'jet', 'vmin': 0., 'vmax': max_val})]
        # bp()
        loss_list.append(loss)
        count_error_list.append(count_error)

        # -- Plotting boxes
        if print_path:
            print_graph(maps, "Gt:{},Pred:{}".format(np.sum(Ys),np.sum(pred_dmap)), os.path.join(print_path, image_name))

    loss = np.mean(loss_list)
    mae = np.mean(count_error_list)
    mse = np.sqrt(np.mean(np.square(count_error_list)))
    return {'loss1':loss,'new_mae':mae,'mse':mse}, mae

def check_conv_gradient_change(network):
    """
        Check if conv layers are frozen
        Parameters
        ----------
        network : NRN network object
        """
    s = []
    count = 0
    for _,main_module in network.named_children():
        for name, module in main_module.named_children():
            if isinstance(module, nn.Conv2d):
                s.append(module.weight.data.cpu().detach().numpy().reshape(-1))
                count += 1
    return np.concatenate(s)

def train_network():
    """
    Main training loop for BSDR
    """
    network = BSDR_Net()
    nrn_networks = []
    nrn_snapshot_path = 'models_NRN/train2/snapshots'
    nrn_models_list = pickle.load(open(os.path.join(nrn_snapshot_path,'best_model.pkl'),'rb'))

    before_nrn_sum = []
    # bp()
    for i in range(num_density_categories):
        nrn_net = NRN()
        nrn_model_file = nrn_models_list[i]
        nrn_net = load_net(nrn_net,nrn_snapshot_path,nrn_model_file)
        nrn_net = nrn_net.cuda()
        nrn_net.eval()
        nrn_networks.append(nrn_net)
        before_nrn_sum.append(check_conv_gradient_change(nrn_net))

    # load_model_VGG16(network)
    model_save_path = os.path.join(model_save_dir, 'train2')
    if not os.path.exists(model_save_path):
        os.makedirs(model_save_path)
        os.makedirs(os.path.join(model_save_path, 'snapshots'))
        os.makedirs(os.path.join(model_save_dir,'dump'))
        os.makedirs(os.path.join(model_save_dir,'dump_test'))
    global f
    snapshot_path = os.path.join(model_save_path, 'snapshots')
    f = open(os.path.join(model_save_path, 'train0.log'), 'w')

    # -- Logging Parameters
    log(f, 'args: ' + str(args))
    log(f, 'model: ' + str(network), False)
    log(f, 'Training0...')
    log(f, 'LR: %.12f.' % (args.lr))
    log(f,'NRN folder {}'.format(nrn_snapshot_path))
    log(f,'NRN models : {}'.format(nrn_models_list))

    start_epoch = 0
    num_epochs = args.epochs
    valid_losses = {}
    test_losses = {}
    train_losses = {}
    for metric in ['loss1', 'new_mae','mse']:
        valid_losses[metric] = []
        test_losses[metric] = []

    for metric in ['loss1']:
        train_losses[metric] = []

    batch_size = args.batch_size
    num_train_images = len(dataset.data_files['train'])
    num_patches_per_image = args.patches
    assert(batch_size < (num_patches_per_image * num_train_images))
    num_batches_per_epoch = num_patches_per_image * num_train_images // batch_size
    assert(num_batches_per_epoch >= 1)

    optimizer = optim.SGD(filter(lambda p: p.requires_grad, network.parameters()),
                          lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay)
    network = load_rot_model_blocks(network, snapshot_path='models_rot_net/train2/snapshots', excluded_layers=excluded_layers)

    # -- Main Training Loop
    all_epoch_test_accs = []

    global sampled_GT

    log(f, 'Testing...')
    epoch_test_losses, mae = test_network(dataset, 'test', network, False)
    log(f, 'TEST epoch: ' + str(-1) + ' test loss1, mae:' + str(epoch_test_losses))

    for e_i, epoch in enumerate(range(start_epoch, num_epochs)):
        avg_loss = []

        global blur_sigma
        for b_i in range(num_batches_per_epoch):
            # Generate next training sample
            Xs, Ys, Ys_full_counts = dataset.train_get_data(batch_size=args.batch_size)
            category_labels = np.digitize(Ys_full_counts, count_density_threshold,right = True)
            if args.use_noisygt:

                before_noisy_gt_maps = []

                for i in range(batch_size):
                    image = Xs[i].transpose((1,2,0)).astype('uint8')#(224,224,3)
                    noisy_gt_map = create_noisy_gt(image,output_downscale,blur_sigma)
                    before_noisy_gt_maps.append(noisy_gt_map[None,...])
                before_noisy_gt_maps = np.array(before_noisy_gt_maps)
                # bp()
                assert(Ys.shape == before_noisy_gt_maps.shape)
                before_noisy_X = torch.autograd.Variable(torch.from_numpy(before_noisy_gt_maps)).cuda().float()
                before_noisy_X.requires_grad = False
                Ys_counts = Ys.reshape((Ys.shape[0],-1)).sum(axis=1).astype('int')
                assert(Ys_counts.shape == Ys_full_counts.shape)

                factor_arr = []
                upsample = nn.Upsample(scale_factor=network_output_downscale, mode='nearest')
                for i in range(batch_size):
                    factor = nrn_networks[category_labels[i]](before_noisy_X[i][None,...])[0]
                    factor_arr.append(factor.detach().cpu().numpy())

                factor_arr = np.array(factor_arr)
                factor_arr = torch.autograd.Variable(torch.from_numpy(factor_arr)).cuda().float()
                # bp()
                after_noisy_X = upsample(factor_arr) * before_noisy_X
                after_noisy_gt_maps = after_noisy_X.detach().cpu().numpy()
                assert(Ys.shape == after_noisy_gt_maps.shape == before_noisy_gt_maps.shape)
                Ys = after_noisy_gt_maps

            train_loss = train_function(Xs, Ys, network, optimizer)#sampled_GT
            avg_loss.append(train_loss)
            for i in range(num_density_categories):
                after_sum = check_conv_gradient_change(nrn_networks[i])
                assert(np.all(before_nrn_sum[i] == after_sum))
            # Logging losses after 1k iterations.
            if b_i % 10 == 0:
                log(f, 'Epoch %d [%d]: %s loss: %s.' % (epoch, b_i, [network.name], train_loss))

        avg_loss = np.mean(np.array(avg_loss))
        train_losses['loss1'].append(avg_loss)
        log(f, 'TRAIN epoch: ' + str(epoch) + ' train mean loss1:' + str(avg_loss))

        torch.cuda.empty_cache()
        log(f, 'Testing...')
        epoch_test_losses, mae = test_network(dataset, 'test', network, False)
        log(f, 'TEST epoch: ' + str(epoch) + ' test loss1, mae:' + str(epoch_test_losses))

        epoch_val_losses, valid_mae = test_network(dataset, 'test_valid', network, False)
        log(f, 'TEST valid epoch: ' + str(epoch) + ' test valid loss1, mae' + str(epoch_val_losses))
        # exit(0)

        for metric in ['loss1', 'new_mae','mse']:
            valid_losses[metric].append(epoch_val_losses[metric])
            test_losses[metric].append(epoch_test_losses[metric])

        min_valid_epoch = np.argmin(valid_losses['new_mae'])
        min_test_epoch = np.argmin(test_losses['new_mae'])

        log(f,'Best valid so far epoch: {}, valid mae: {},test (mae:{},mse:{}), min_test epoch {}, (mae:{},mse:{})'.format(
            min_valid_epoch,
            valid_losses['new_mae'][min_valid_epoch],
            test_losses['new_mae'][min_valid_epoch],
            test_losses['mse'][min_valid_epoch],
            min_test_epoch,
            test_losses['new_mae'][min_test_epoch],
            test_losses['mse'][min_test_epoch]))
        # Save networks
        save_checkpoint({
            'epoch': epoch + 1,
            'state_dict': network.state_dict(),
            'optimizer': optimizer.state_dict(),
        }, snapshot_path, get_filename(network.name, epoch + 1))

        print('saving graphs...')
        with open(os.path.join(snapshot_path, 'losses.pkl'), 'wb') as lossfile:
            pickle.dump((train_losses, valid_losses, test_losses), lossfile, protocol=2)

        for metric in train_losses.keys():
            if "maxima_split" not in metric:
                if isinstance(train_losses[metric][0], list):
                    for i in range(len(train_losses[metric][0])):
                        plt.plot([a[i] for a in train_losses[metric]])
                        plt.savefig(os.path.join(snapshot_path, 'train_%s_%d.png' % (metric, i)))
                        plt.clf()
                        plt.close()
                # print(metric, "METRIC", train_losses[metric])
                plt.plot(train_losses[metric])
                plt.savefig(os.path.join(snapshot_path, 'train_%s.png' % metric))
                plt.clf()
                plt.close()

        for metric in valid_losses.keys():
            if isinstance(valid_losses[metric][0], list):
                for i in range(len(valid_losses[metric][0])):
                    plt.plot([a[i] for a in valid_losses[metric]])
                    plt.savefig(os.path.join(snapshot_path, 'valid_%s_%d.png' % (metric, i)))
                    plt.clf()
                    plt.close()
            plt.plot(valid_losses[metric])
            plt.savefig(os.path.join(snapshot_path, 'valid_%s.png' % metric))
            plt.clf()
            plt.close()

        for metric in test_losses.keys():
            if isinstance(test_losses[metric][0], list):
                for i in range(len(test_losses[metric][0])):
                    plt.plot([a[i] for a in test_losses[metric]])
                    plt.savefig(os.path.join(snapshot_path, 'test_%s_%d.png' % (metric, i)))
                    plt.clf()
                    plt.close()
            plt.plot(test_losses[metric])
            plt.savefig(os.path.join(snapshot_path, 'test_%s.png' % metric))
            plt.clf()
            plt.close()

    min_valid_epoch = np.argmin(valid_losses['new_mae'])
    network = load_net(network,snapshot_path,get_filename(network.name, min_valid_epoch + 1))
    log(f,'Testing on best model {}'.format(min_valid_epoch))
    epoch_test_losses, mae = test_network(dataset, 'test', network, print_output=os.path.join(model_save_dir,'dump_test'))
    log(f, 'TEST epoch: ' + str(epoch) + ' test loss1, mae:' + str(epoch_test_losses))
    log(f, 'Exiting train...')
    f.close()
    return

if __name__ == '__main__':
    args = parser.parse_args()
    # -- Assign GPU
    os.environ["CUDA_VISIBLE_DEVICES"] = str(args.gpu)

    # -- Assertions
    assert (args.dataset)

    # -- Setting seeds for reproducability
    np.random.seed(11)
    random.seed(11)
    torch.manual_seed(11)
    torch.backends.cudnn.enabled = False
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.benchmark = False
    torch.cuda.manual_seed(11)
    torch.cuda.manual_seed_all(11)

    # -- Dataset paths
    if args.dataset == "parta":
        validation_set = 60
        path = '../dataset/ST_partA/'
        output_downscale = 8
        density_map_sigma = 1
        blur_sigma = 1
        image_size_min = 256
        image_crop_size = 256
        network_output_downscale = 4
    elif args.dataset == "ucfqnrf":
        validation_set = 240
        output_downscale = 8
        path = '../dataset/UCF-QNRF_ECCV18/'
        output_downscale = 8
        density_map_sigma = 1
        blur_sigma = 1
        image_size_min = 256
        image_crop_size = 256
        network_output_downscale = 4
    else:
        validation_set = 0
        output_downscale = 8
        path = '../../dataset/ST_partA_' + args.dataset.replace('parta_', '') + '/'

    model_save_dir = './models_BSDR'

    batch_size = args.batch_size
    count_density_threshold = str(args.count_thresh).split(',') if args.count_thresh !='' else []
    count_density_threshold = list(map(lambda x: float(x),count_density_threshold))

    num_density_categories = len(count_density_threshold) + 1
    dataset = CrowdDataset(path, name=args.dataset, valid_set_size=validation_set,
                           gt_downscale_factor=output_downscale,density_map_sigma=density_map_sigma,
                           image_size_multiple = output_downscale * network_output_downscale,
                           image_size_min = image_size_min , image_crop_size = image_crop_size)
    #print(dataset.data_files['test_valid'], len(dataset.data_files['test_valid']))
    print(dataset.data_files['train'], len(dataset.data_files['train']))

    # -- Train the model
    train_network()

reason: import numpy,import scipy


text:
# --------------
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split

# path- variable storing file path
df=pd.read_csv(path)
df.head()
#Code starts here
X=df.drop(['Price'], axis=1)
y=df['Price']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=6)
corr= X_train.corr()
#Code ends here

# --------------
from sklearn.linear_model import LinearRegression
from sklearn.metrics import r2_score

# Code starts here
regressor=LinearRegression()
regressor.fit(X_train, y_train)
y_pred=regressor.predict(X_test)
r2 =regressor.score (X_test,y_test)

# --------------
from sklearn.linear_model import Lasso
# Code starts here
lasso=Lasso()
lasso.fit(X_train, y_train)
lasso_pred = lasso.predict(X_test)
r2_lasso = lasso.score(X_test, y_test)

# --------------
from sklearn.linear_model import Ridge
# Code starts here
ridge=Ridge()
ridge.fit(X_train, y_train)
ridge_pred = ridge.predict(X_test)
r2_ridge= ridge.score(X_test, y_test)

# Code ends here

# --------------
from sklearn.model_selection import cross_val_score
#Code starts here
regressor= LinearRegression()
score= cross_val_score(regressor, X_train, y_train, cv=10)
mean_score=score.mean()
print(mean_score)

# --------------
from sklearn.preprocessing import PolynomialFeatures
from sklearn.pipeline import make_pipeline
#Code starts here
model = make_pipeline(PolynomialFeatures(2), LinearRegression())
model.fit(X_train, y_train)
y_pred = model.predict(X_test)
r2_poly = model.score(X_test,y_test)


text:
# -*- coding: utf-8 -*-

"""
Module with functions related to laboratory work and data acquired with CATMAN.

"""

import numpy as np
import csv
import codecs
from os.path import basename, splitext
from matplotlib import pyplot as plt
import PySS.analytic_geometry as ag

class Experiment:
    """
    Laboratory test data.

    Class laboratory experiment containing methods for loading and manipulating data recorded with CATMAN software. The
    class is designed to be used with the `from_...` alternative constructors and not by giving the data directly for
    the object to be constructed.

    """
    def __init__(self, name=None, channels=None):
        if channels is None:
            self.channels = {}
        elif not isinstance(channels, dict):
            print("Channels must be a dictionary.")
            return NotImplemented
        else:
            self.channels = channels
        self.name = name

    def plot2d(
            self,
            x_data,
            y_data,
            ax=None,
            scale=None,
            axlabels=True,
            reduced=1,
            **kargs
    ):
        """
        Plot two recorded channels against each other.

        Parameters
        ----------
        x_data : str
            Key from self.data dictionary for the x-axis.
        y_data : str
            Key from self.data dictionary for the y-axis.
        ax : axis object, optional
            The axis to be used for plotting. By default, new figure and axis are created.
        scale : list or tuple of 2 numbers
            Scaling factors for `x` and `y` axis
        axlabels : bool, optional
            Write labels for x and y axes based on the channel name. TeX is used so certain names may cause problems in
            which case the used might want to deactivate plotting axes labels. Default is True.
        reduced : float, optional
            Plot a subset of points, the size of which is given as a fraction of the full set. Default value is 1.0
            which implies all points are plotted.

        """
        #plt.rc('text', usetex=False)

        if ax is None:
            fig, ax = plt.subplots()

        if scale is None:
            scale = (1, 1)

        ax.plot(
            self.channels[x_data]["data"][0::int(1/reduced)]*scale[0],
            self.channels[y_data]["data"][0::int(1/reduced)]*scale[1],
            label=self.name,
            **kargs
        )

        if axlabels:
            plt.xlabel(x_data + ", [" + self.channels[x_data]["units"] + "]")
            plt.ylabel(y_data + ", [" + self.channels[y_data]["units"] + "]")
            plt.title(self.name)

        return ax

    def add_new_channel_zeros(self, ch_name, units):
        """"
        Initialise a new channel entry in the dictionary with zeros.

        Parameters
        ----------
        ch_name : str
        units : str

        """
        self.channels[ch_name] = {}
        self.channels[ch_name]["data"] = np.zeros([len(self.channels[next(iter(self.channels))]["data"])])
        self.channels[ch_name]["units"] = units

    def invert_sign(self, ch_name):
        """Invert the sign on the values of a channel"""
        self.channels[ch_name]["data"] = -1 * self.channels[ch_name]["data"]

    @classmethod
    def from_file(cls, fh):
        """
        Alternative constructor.

        Imports data from a CATMAN output file. Uses 'ISO-8859-1' text encoding.

        Parameters
        ----------
        fh : str
            Path of ascii data file.

        """

        # Name from filename.
        filename = splitext(basename(fh))[0]

        # Open the requested file.
        f = codecs.open(fh, 'r', 'ISO-8859-1')

        # Read the header
        header = list(csv.reader([next(f) for x in range(7)], delimiter='\t'))

        # Read the column headers.
        next(f)
        column_head = list(csv.reader([next(f) for x in range(29)], delimiter='\t'))

        # Read the tab separated values.
        next(f)
        values = list(csv.reader(f, delimiter='\t'))

        # Get the number of channels
        n_chan = len(values[0])

        # Create a dictionary.
        channels = {}

        # Build a dictionary with the data using the column header to fetch the dict keys.
        for i in range(n_chan):
            data = np.empty(len(values))
            name = column_head[0][i].partition(' ')[0]
            channels[name] = {}
            units = column_head[1][i]
            channels[name]["units"] = units
            for j, row in enumerate(values):
                data[j] = float(row[i].replace(',', '.'))

            channels[name]["data"] = data

        return cls(name=filename, channels=channels)

class CouponTest(Experiment):
    """
    Class for coupon tests.

    Inherits the basic properties of a generic experiment class and offers standard calculations for rectangular
    cross-section tensile coupon tests.

    """

    def __init__(self, name=None, channels=None, thickness=None, width=None, l_0=None):
        super().__init__(name=name, channels=channels)
        self.thickness = thickness
        self.width = width
        self.l_0 = l_0
        self.initial_stiffness = None
        self.young = None
        # If geometric data exist, calculate the initial area.
        if (
            isinstance(thickness, float) or isinstance(thickness, int)
        ) and (
            isinstance(width, float) or isinstance(width, int)
        ):
            self.A_0 = thickness * width
        else:
            self.A_0 = None

    def clean_initial(self):
        """
        Remove head and tail of the load-displacement data.

        """

        # Filter the indices of the values larger then 1 kN, as a way of excluding head and tail recorded data.
        valid = np.where(self.channels["Load"]["data"] > 1)[0]

        for i in self.channels.values():
            i["data"] = i["data"][valid]

    def calc_init_stiffness(self):
        """
        Calculate the modulus of elasticity.

        The calculation is based on the range between 20% and 30% of the max load.

        """
        # Find the 20% and 30% of the max load
        lims = np.round([np.max(self.channels["Load"]["data"]) * 0.2, np.max(self.channels["Load"]["data"]) * 0.3]).astype(int)

        indices = [np.argmax(self.channels["Load"]["data"] > lims[0]), np.argmax(self.channels["Load"]["data"] > lims[1])]
        disp_el = self.channels["Epsilon"]["data"][indices[0]:indices[1]]
        load_el = self.channels["Load"]["data"][indices[0]:indices[1]]

        # fitting
        A = np.vstack([disp_el, np.ones(len(disp_el))]).T
        m, c = np.linalg.lstsq(A, load_el)[0]

        # Save the initial stiffness in th eobject
        self.initial_stiffness = m

        # Return the tangent
        return [m, c]

    def offset_to_0(self):
        """
        Offset the stroke values to start from 0 based on a regression on the initial stiffness.
        """

        # Linear regression on the elastic part to get stiffness and intercept
        m, c = self.calc_init_stiffness()
        # calculate the stroke offset
        offset = -c/m

        # Offset original values
        self.channels["Epsilon"]["data"] = self.channels["Epsilon"]["data"] - offset

        # Add the initial 0 values in both stroke and load arrays
        self.channels["Epsilon"]["data"] = np.concatenate([[0], self.channels["Epsilon"]["data"]])
        self.channels["Load"]["data"] = np.concatenate([[0], self.channels["Load"]["data"]])

    def add_initial_data(self, thickness, width, l_0):
        """
        Add initial geometrical data of the coupon specimen.

        Assuming that the imported test data provide stroke-load readings from testing, the geometric data of the coupon
        (thickness, width, initial gauge length) are needed to calculate the stress-strain curves. The method will
        overwrite pre existing.

        Parameters
        ----------
        thickness : float
            Coupon's measured thickness.
        width : float
            Coupon's measured width.
        l_0 : float
            Initial gauge length, %l_{0}%

        """
        self.thickness = thickness
        self.width = width
        self.l_0 = l_0
        self.A_0 = thickness * width

    def calc_stress_strain(self):
        """
        Calculate stress-strain curve.

        Calculate the engineering ant the true stress-strain curves for the coupon test. Initial geometric data as well
        as test data (stroke, load) must be assigned to the object for the method to run. The method assumes that
        entries with keys 'Stroke' and 'Load' exit in the data dictionary.

        """

        self.channels['StressEng'] = {"data": 1000 * self.channels['Load']["data"] / self.A_0,
                                      "units": "MPa"}
        self.channels['StrainEng'] = {"data": self.channels['Epsilon']["data"] / self.l_0,
                                      "units": "mm/mm"}

    def calc_young(self):
        """Calculate the modulus of elasticity."""
        if self.l_0 and self.A_0:
            if not self.initial_stiffness:
                self.calc_init_stiffness()

            self.young = 1000 * self.initial_stiffness * (self.l_0 / self.A_0)
        else:
            print("Missing information. Check if l_0 and A_0 exist.")

    def calc_plastic_strain(self):
        """
        Calculate the plastic strain.

        """
        self.channels["StrainPl"] = {"data": self.channels["StrainEng"]["data"] - self.channels["StressEng"]["data"] / self.young,
                                     "units": "mm/mm"}

    def calc_fy(self):
        """
        Calculate the yield stress based on the 0.2% plastic strain criterion.

        """
        line = ag.Line2D.from_2_points([0.2, 0], [0.2 + 900 / 2100, 900])
        for i, strain in enumerate(self.channels["StrainEng"]["data"] * 100):
            if (self.channels["StressEng"]["data"][i] - line.y_for_x(strain)) < 0:
                self.f_yield = self.channels["StressEng"]["data"][i]
                return

    def plot_stressstrain_eng(self, ax=None, **kargs):
        ax = self.plot2d('StrainEng', 'StressEng', scale=[100, 1], ax=ax, **kargs)
        plt.xlabel('Strain, $\\varepsilon_{eng}$ [\%]')
        plt.ylabel('Stress, $\\sigma_{eng}$ [MPa]')
        plt.title(self.name)
        return ax

def load_coupons():
    cp = {}

    # 2 mm plate
    widths = [20.408, 20.386, 20.397, 20.366, 20.35, 20.39]
    thicknesses = [1.884, 1.891, 1.9, 1.88, 1.882, 1.878]
    l_0 = 80
    for i in range(1, 7):
        coupon = CouponTest.from_file('./data/coupons/S700_2mm/cp{}.asc'.format(i))
        coupon.clean_initial()
        coupon.calc_init_stiffness()
        coupon.offset_to_0()
        coupon.add_initial_data(thicknesses[i-1], widths[i-1], l_0=l_0)
        coupon.calc_stress_strain()
        coupon.calc_young()
        coupon.calc_plastic_strain()
        coupon.calc_fy()

        cp["cp{}_2mm".format(i)] = coupon

    # 3 mm plate
    widths = [19.68, 19.63, 19.68, 19.716, 19.681, 19.76]
    thicknesses = [3.037, 3.031, 3.038, 3.051, 3.038, 3.036]

    for i in range(1, 7):
        coupon = CouponTest.from_file('./data/coupons/S700_3mm/cp{}.asc'.format(i))
        coupon.clean_initial()
        coupon.calc_init_stiffness()
        coupon.offset_to_0()
        coupon.add_initial_data(thicknesses[i-1], widths[i-1], l_0=l_0)
        coupon.calc_stress_strain()
        coupon.calc_young()
        coupon.calc_plastic_strain()
        coupon.calc_fy()

        cp["cp{}_3mm".format(i)] = coupon

    # 4 mm plate
    widths = [19.375, 20.335, 20.345]
    thicknesses = [4.0732, 4.0744, 4.0906]

    for i in [1, 2, 3]:
        coupon = CouponTest.from_file('./data/coupons/S700_4mm/cp{}.asc'.format(i))
        coupon.clean_initial()
        coupon.calc_init_stiffness()
        coupon.offset_to_0()
        coupon.add_initial_data(thicknesses[i-1], widths[i-1], l_0=l_0)
        coupon.calc_stress_strain()
        coupon.calc_young()
        coupon.calc_plastic_strain()
        coupon.calc_fy()

        cp["cp{}_4mm".format(i)] = coupon

    # 3 mm plate
    # for i in range(1, 7):
    #     coupon = CouponTest.from_file('./data/coupons/3mm/cp{}.asc'.format(i))
    #     coupon.clean_initial()
    #     coupon.calc_init_stiffness()
    #     coupon.offset_to_0()
    #
    #     cp["cp{}_2mm".format(i)] = coupon
    #
    #
    # widths = [20.408, 20.386, 20.397, 20.366, 20.35, 20.39]
    # thicknesses = [1.884, 1.891, 1.9, 1.88, 1.882, 1.878]
    # l_0s = [80., 80., 80., 80., 80., 80.]
    #
    # for i in range(1, 7):
    #     x = cp["cp{}_2mm".format(i)]
    #     x.add_initial_data(thicknesses[i-1], widths[i-1], l_0s[i-1])
    #     x.calc_stress_strain()
    #     x.calc_young()
    #     x.calc_plastic_strain()
    #     # x.plot_stressstrain_eng()
    #
    #     ax = plt.axes()
    #     x.plot_stressstrain_eng(ax=ax)
    #
    # ax.plot([0.002, 0.002 + 900 / 210000], [0, 900])

    return cp


text:
import numpy as np
from torchvision import datasets, transforms
import torch

class MNIST_data():
    @staticmethod
    def train():
        train_loader = torch.utils.data.DataLoader(
            datasets.MNIST('../data', train=True, download=True,
                        transform=transforms.Compose([
                            transforms.ToTensor(),
                            transforms.Normalize((0.1307,), (0.3081,))
                        ])),
            batch_size=60000, shuffle=True)
        for (data, target) in train_loader:#getting whole batch one time
            return (data, target)
    @staticmethod
    def test():
        train_loader = torch.utils.data.DataLoader(
            datasets.MNIST('../data', train=False, download=True,
                        transform=transforms.Compose([
                            transforms.ToTensor(),
                            transforms.Normalize((0.1307,), (0.3081,))
                        ])),
            batch_size=10000, shuffle=True)
        for (data, target) in train_loader:#getting whole batch one time
            return (data, target)

##data is from https://github.com/sunblaze-ucb/dpml-benchmark
class Covertype_data():
    @staticmethod
    def train():
        x=np.load('../data'+'/covertype_processed_x.npy')
        y=np.load('../data'+'/covertype_processed_y.npy')
        total=x.shape[0]
        num_trn=int(total*0.8)
        trn_x=torch.tensor(x[0:num_trn], dtype=torch.float32)
        trn_y=torch.zeros(num_trn, dtype=torch.long)
        for i in range(num_trn):
            for j in range(len(y[0])):
                if(y[i][j]==1):
                    trn_y[i]=j
        return (trn_x, trn_y)

    @staticmethod
    def test():
        x=np.load('../data'+'/covertype_processed_x.npy')
        y=np.load('../data'+'/covertype_processed_y.npy')
        total=x.shape[0]
        idx=int(total*0.8)
        num_test=int(0.2*total)
        test_x=torch.tensor(x[int(total*0.8):-1], dtype=torch.float32)
        test_y=torch.zeros(num_test, dtype=torch.long)
        for i in range(num_test):
            for j in range(len(y[0])):
                if(y[idx+i][j]==1):
                    test_y[i]=j
        return (test_x, test_y)

class CIFAR_data():
    @staticmethod
    def train():
        train_loader = torch.utils.data.DataLoader(
            datasets.CIFAR10('../data', train=True, download=True,
                        transform=transforms.Compose([transforms.ToTensor(),
                        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])),
            batch_size=50000, shuffle=True)

        for (data, target) in train_loader:#getting whole batch one time
            return (data, target)
    @staticmethod
    def test():
        train_loader = torch.utils.data.DataLoader(
            datasets.CIFAR10('../data', train=False, download=True,
                        transform=transforms.Compose([transforms.ToTensor(),
                        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])),
            batch_size=10000, shuffle=True)

        for (data, target) in train_loader:#getting whole batch one time
            return (data, target)


text:
import h5py
import numpy as np
import matplotlib.pyplot as plt
import imageio
from scipy.ndimage import gaussian_filter as gf
from . import spectra_mocking as sm
from . import  minima
from . import load_data

class Paint:
    """ A class to paint stoughs on flux map """
    def __init__(self, m_dach, dim, sigma, thresh, linking_contour=-2, load_BH=False, load_halos=False, load_progen_DM=False, load_progen_Gas=False, load_protoc=True, pc=None, periodic_bound=True):

        self.m_dach = m_dach
        self.dim = dim
        self.sigma = sigma
        self.thresh = thresh
        self.linking_contour = linking_contour
        if load_BH :
            self.data_BH = load_data.load_BH()
        if load_halos :
            self.halos_coords = load_data.load_halos()
        if load_progen_DM :
            self.DM_coords = load_data.load_progen_DM()
        if load_progen_Gas:
            self.Gas_coords = load_data.load_progen_Gas()
        if load_protoc :
            if pc is None :
                m_conv = gf(self.m_dach, self.sigma, mode='wrap')
                pc = minima.find_extrema_v3(mapconv = m_conv, thresh=self.thresh, linking_contour=-self.linking_contour, periodic_bound=periodic_bound)
            pc_coords = np.empty(shape=(pc['x'].size, 3))
            # Switch x and y. As we want to overplot it on plt.imshow map.(converting position on matrix m to plt.imshow map) -- I switched back to normal
            pc_coords[:,0], pc_coords[:,1], pc_coords[:,2]  =  pc['x'], pc['y'], pc['z']
            self.pc_coords = pc_coords
            self.pc = pc
            """
            # This "if" is used for the selected peaks in true map, pass the indices to pc_ind argument
            if pc_ind.size > 0 :
                self.pc_ind = pc_ind
            # if pc_indis empty, then we overlay all the peaks
            else :
                self.pc_ind = np.arange(pc['x'].size)
            """

    def _within_linking_length(self, coords, z, deltaz=3):
        """Getting the objects within +- b parameter of FOF"""
        mask = np.ones((coords.shape[0],), dtype=bool)
        #mask *= coords[:,2] == z
        mask *= coords[:,2] >= z-deltaz
        mask *= coords[:,2] <= z+deltaz

        return coords[mask].astype(int), mask

    def circle(self, ax, x0, y0, r, c):
        """Draw a circle on axis ax"""
        box_size=self.m_dach.shape[0]
        x0, y0, r = np.array(x0), np.array(y0), np.array(r)
        for i in range(x0.size):
            x, y = np.array([]),np.array([])
            for t in np.arange(0,2*np.pi,0.1):
                x = np.append(x, (x0[i] + r[i]*np.cos(t))%(box_size) )
                y = np.append(y,  (y0[i] + r[i]*np.sin(t))%(box_size) )
            mask =  (x>0)*(x<self.m_dach.shape[0])*(y>0)*(y<self.m_dach.shape[0])
            x, y =x[mask], y[mask]
            ax.scatter(x,y,c=c, s=5, alpha=0.9)

    def _plot_Rsig_circle(self, ax, z, c, pc_ind):

        d = np.abs(self.pc_coords[:,2][pc_ind] - z)
        ind = np.where( self.pc['Rsig'][pc_ind]-d >=0 )
        r  = np.sqrt(self.pc['Rsig'][pc_ind][ind]**2 - d[ind]**2)
        self.circle(ax=ax, x0=self.pc_coords[pc_ind][ind][:,1], y0=self.pc_coords[pc_ind][ind][:,0], r=r, c=c)

    def plot_2D_flux(self, ax, z, true_map=False, contour_color='k', only_contour=False, plot_Rsig=True,levels=[-7, -6, -5, -4, -3, -2], cmap_reversed=True, color_bar=True, pc_ind=np.array([])):
        """plots the flux map at coordinate z = z on the axis passed to the function"""
        # Get the location of protoclusters and smoothed map
        #if m is None :
        #   m = np.fromfile('./spectra/maps/map_'+map_file+'.dat').reshape((dim[0], dim[1], dim[2]))
        m = gf(self.m_dach, self.sigma, mode='wrap')
        m = m/np.std(m)
        if cmap_reversed:
            cmap = plt.get_cmap('jet').reversed()
        else :
            cmap = plt.get_cmap('jet')
        m_2D = m[:,:,z]
        if true_map :
            m_2D = m_2D.T
        if not only_contour:
            im = ax.imshow(m_2D, cmap=cmap, origin='lower', extent=[0, self.dim[0], 0, self.dim[1]], interpolation='bilinear', vmin=-4, vmax=3)
        ax.contour(m_2D, levels= levels, origin='lower', colors=contour_color)
        try :
            DM_coords, _ = self._within_linking_length(self.DM_coords, z=z)
            ax.scatter(DM_coords[:,1], DM_coords[:,0], color='b', marker='*', s=10)
        except AttributeError: pass
        try :
            g_coords, _ = self._within_linking_length(self.Gas_coords, z=z)
            ax.scatter(g_coords[:,1], g_coords[:,0], color='m', marker='.', s=5)
        except AttributeError: pass
        # Overplot protoclusters
        try :
            # pc_ind is not empty only if we want to plot the closest peaks in true map to the peaks in mock map
            if pc_ind.size == 0 :
                pc_ind = np.arange(self.pc['x'].size)

            p_coords, mask = self._within_linking_length(self.pc_coords[pc_ind,:], z=z)
            # all protoclusters in a single z
            ax.scatter(p_coords[:,1], p_coords[:,0], color=contour_color, marker='*', s=50)
            if plot_Rsig:
                self._plot_Rsig_circle(ax=ax, z=z, c=contour_color, pc_ind=pc_ind)

        except AttributeError: pass
        try :
          BH_coords, _ = self._within_linking_length(self.data_BH['coords'], z=z)
          # all BHs in a single z
          ax.scatter(BH_coords[:,1], BH_coords[:,0], color='k', marker='*', s=50)
        except AttributeError: pass
        try :
          h_coords, _ = self._within_linking_length(self.halos_coords, z=z)
          ax.scatter(h_coords[:,1], h_coords[:,0], color='w',  alpha=0.5 ,s=50)
        except AttributeError: pass

        ax.set_title('z_coordonate = '+str(z)+r'$h^-1 cMpc$')
        ax.set_xlim(0,self.dim[0])
        ax.set_ylim(0,self.dim[1])
        if not only_contour :
            return ax, im
        else :
            return ax

max_stars_repo_name: mahdiqezlou/LyTomo-Watershed

max_issues_repo_name: mahdiqezlou/LyTomo-Watershed

max_forks_repo_name: mahdiqezlou/LyTomo-Watershed


text:
import numpy as np
import pycqed.analysis_v2.base_analysis as ba

class OptimizationAnalysis(ba.BaseDataAnalysis):
    def __init__(self, t_start: str=None, t_stop: str=None,
                 label: str='', data_file_path: str=None,
                 options_dict: dict=None, extract_only: bool=False,
                 do_fitting: bool=True, auto=True):
        super().__init__(t_start=t_start, t_stop=t_stop,
                         label=label,
                         data_file_path=data_file_path,
                         options_dict=options_dict,
                         extract_only=extract_only, do_fitting=do_fitting)
        # self.single_timestamp = False
        self.params_dict = {'xlabel': 'sweep_name',
                            'xunit': 'sweep_unit',
                            'xvals': 'sweep_points',
                            'measurementstring': 'measurementstring',
                            'value_names': 'value_names',
                            'value_units': 'value_units',
                            'measured_values': 'measured_values'}

        self.numeric_params = []
        if auto:
            self.run_analysis()

    def prepare_plots(self):
        # assumes that value names are unique in an experiment

        for i, val_name in enumerate(self.raw_data_dict['value_names'][0]):
            yvals = self.raw_data_dict['measured_values_ord_dict'][val_name][0]

            self.plot_dicts['{}_vs_iteration'.format(val_name)] = {
                'plotfn': self.plot_line,
                'xvals': np.arange(len(yvals)),
                'xlabel': 'Iteration',
                'xunit': '#',
                'yvals': yvals,
                'ylabel': val_name,
                'yunit': self.raw_data_dict['value_units'][0][i]}

            legend_title = 'timestamp'

        # for i, val_name in enumerate(self.raw_data_dict['value_names'][0]):

        #     self.plot_dicts[val_name] = {
        #         'plotfn': self.plot_line,
        #         'xvals': xvals,
        #         'xlabel': self.raw_data_dict['xlabel'][0],
        #         'xunit': self.raw_data_dict['xunit'][0][0],
        #         'yvals': yvals,
        #         'ylabel': val_name,
        #         'yrange': self.options_dict.get('yrange', None),
        #         'xrange': self.options_dict.get('xrange', None),
        #         'yunit': self.raw_data_dict['value_units'][0][i],
        #         'setlabel': setlabel,
        #         'legend_title': legend_title,
        #         'title': (self.raw_data_dict['timestamps'][0]+' - ' +
        #                   self.raw_data_dict['timestamps'][-1] + '\n' +
        #                   self.raw_data_dict['measurementstring'][0]),
        #         'do_legend': do_legend,
        #         'legend_pos': 'upper right'}


text:
import numpy as np
import sys
import gzip

allS = open(sys.argv[1])

allPeaks = {}
allScores = {}
scores = []

for line in allS:
    parts = line.strip().split()
    name = parts[0] + ":" + parts[1]
    score = float(parts[3])
    scores.append(score)
    allScores[name] = score
allS.close()

pcs = []
for x in range(1, 101, 1):
    pcs.append(np.percentile(scores, x))

allS = open(sys.argv[1])
for line in allS:
    parts = line.strip().split()
    name = parts[0] + ":" + parts[1]
    v = allScores[name]
    mid = int(parts[1]) + ((int(parts[2]) - int(parts[1])) / 2)
    i = 0
    while v >= pcs[i]:
        if i == len(pcs) - 1:
            break
        i += 1

    print "chr"+parts[0] + "\t" + parts[1] +"\t"+ parts[2] + "\t" + name + "\t" + parts[3] + "\t+\t" + str(mid) + "\t" + str(i)


text:
import pygame
import numpy as np

class displayer:

    def __init__(self, _map_shape, pix_size, ALPHA_CHANNEL, has_gap=False):
        """
        _map_size: tuple
        color_map: a list indicates the color to each index.
                   0 : empty block, should always white
                   1+: varies building types
        """
        pygame.init()
        clock = pygame.time.Clock()
        clock.tick(60)

        self.ALPHA_CHANNEL = ALPHA_CHANNEL
        self._map_shape = _map_shape
        self.has_gap = has_gap
        self.pix_size = pix_size
        self.screen = pygame.display.set_mode((_map_shape[1]*self.pix_size,
                                               _map_shape[0]*self.pix_size + 100))
        self.color_map = [(255,255,255),
                          (128,255,128),
                          (255,255,128),
                          (255,180, 50),
                          (255,128,128),
                          (255,128,255)]
        self.screen.fill((255,255,255))
        pygame.draw.line(self.screen, (0,0,0),
                         (0, self._map_shape[0]*self.pix_size),
                         (self._map_shape[1]*self.pix_size, self._map_shape[0]*self.pix_size))
        pygame.display.update()

    def clear(self):
        self.screen.fill((255,255,255))
        pygame.draw.line(self.screen, (0,0,0),
                         (0, self._map_shape[0]*self.pix_size),
                         (self._map_shape[1]*self.pix_size, self._map_shape[0]*self.pix_size))
        pygame.display.update()

    def draw_all(self, _map, valid_mask):
        for i in range(_map.shape[0]):
            for j in range(_map.shape[1]):
                x = j * self.pix_size + int(self.pix_size/2)
                y = i * self.pix_size + int(self.pix_size/2)
                if self.has_gap:
                    size = min(int(self.pix_size * 0.75), self.pix_size-2)
                else:
                    size = self.pix_size
                s = pygame.Surface((size,size))
                if valid_mask[i,j,0]:
                    c = self.color_map[(np.argmax(_map[i,j,:self.ALPHA_CHANNEL])+1)*(_map[i,j,self.ALPHA_CHANNEL]>0.1)]
                else:
                    c = (0,0,0)
                s.fill(c)
                self.screen.blit(s, (x-int(size/2), y-int(size/2)))
        pygame.display.update()

    def draw(self, _map, valid_mask, valid_coords):
        for i,j in valid_coords:
            x = j * self.pix_size + int(self.pix_size/2)
            y = i * self.pix_size + int(self.pix_size/2)
            if self.has_gap:
                size = min(int(self.pix_size * 0.75), self.pix_size-2)
            else:
                size = self.pix_size
            s = pygame.Surface((size,size))
            if valid_mask[i,j,0]:
                c = self.color_map[(np.argmax(_map[i,j,:self.ALPHA_CHANNEL])+1)*(_map[i,j,self.ALPHA_CHANNEL]>0.1)]
            else:
                c = (0,0,0)
            s.fill(c)
            self.screen.blit(s, (x-int(size/2), y-int(size/2)))
        pygame.display.update()


text:
""" Tests the duct component. """

import unittest
import os

import numpy as np

from openmdao.api import Problem, Group
from openmdao.utils.assert_utils import assert_near_equal, assert_check_partials

from pycycle.mp_cycle import Cycle
from pycycle.elements.duct import Duct
from pycycle.elements.flow_start import FlowStart
from pycycle import constants
from pycycle.thermo.cea import species_data

fpath = os.path.dirname(os.path.realpath(__file__))
ref_data = np.loadtxt(fpath + "/reg_data/duct.csv", delimiter=",", skiprows=1)

header = [
    'dPqP',
    'Qin',
    'Fl_I.W',
    'Fl_I.V',
    'Fl_I.MN',
    'Fl_I.s',
    'Fl_I.Pt',
    'Fl_I.Tt',
    'Fl_I.ht',
    'Fl_I.rhot',
    'Fl_I.gamt',
    'Fl_O.MN',
    'Fl_O.s',
    'Fl_O.Pt',
    'Fl_O.Tt',
    'Fl_O.ht',
    'Fl_O.rhot',
    'Fl_O.gamt',
    'Fl_O.Ps',
    'Fl_O.Ts',
    'Fl_O.hs',
    'Fl_O.rhos',
    'Fl_O.gams']
h_map = dict(((v_name, i) for i, v_name in enumerate(header)))

np.seterr(all="raise")

class DuctTestCase(unittest.TestCase):

    def test_case1(self):

        self.prob = Problem()
        cycle = self.prob.model = Cycle()
        cycle.options['thermo_method'] = 'CEA'
        cycle.options['thermo_data'] = species_data.janaf

        cycle.add_subsystem('flow_start', FlowStart(), promotes=['MN', 'P', 'T'])
        cycle.add_subsystem('duct', Duct(), promotes=['MN'])

        cycle.pyc_connect_flow('flow_start.Fl_O', 'duct.Fl_I')

        cycle.set_input_defaults('MN', 0.5)
        cycle.set_input_defaults('duct.dPqP', 0.0)
        cycle.set_input_defaults('P', 17., units='psi')
        cycle.set_input_defaults('T', 500., units='degR')
        cycle.set_input_defaults('flow_start.W', 500., units='lbm/s')

        self.prob.setup(check=False, force_alloc_complex=True)
        self.prob.set_solver_print(level=-1)

        # 6 cases to check against
        for i, data in enumerate(ref_data):

            self.prob['duct.dPqP'] = data[h_map['dPqP']]

            # input flowstation
            self.prob['P'] = data[h_map['Fl_I.Pt']]
            self.prob['T'] = data[h_map['Fl_I.Tt']]
            self.prob['MN'] = data[h_map['Fl_O.MN']]
            self.prob['flow_start.W'] = data[h_map['Fl_I.W']]
            self.prob['duct.Fl_I:stat:V'] = data[h_map['Fl_I.V']]

            # give a decent initial guess for Ps

            print(i, self.prob['P'], self.prob['T'], self.prob['MN'])

            self.prob.run_model()

            # check outputs
            pt, ht, ps, ts = data[h_map['Fl_O.Pt']], data[
                h_map['Fl_O.ht']], data[h_map['Fl_O.Ps']], data[h_map['Fl_O.Ts']]
            pt_computed = self.prob['duct.Fl_O:tot:P']
            ht_computed = self.prob['duct.Fl_O:tot:h']
            ps_computed = self.prob['duct.Fl_O:stat:P']
            ts_computed = self.prob['duct.Fl_O:stat:T']

            tol = 2.0e-2
            assert_near_equal(pt_computed, pt, tol)
            assert_near_equal(ht_computed, ht, tol)
            assert_near_equal(ps_computed, ps, tol)
            assert_near_equal(ts_computed, ts, tol)

            partial_data = self.prob.check_partials(out_stream=None, method='cs',
                                                    includes=['duct.*'], excludes=['*.base_thermo.*',])
            assert_check_partials(partial_data, atol=1e-8, rtol=1e-8)

    def test_case_with_dPqP_MN(self):

        self.prob = Problem()

        # need two cycles, because we can't mix design and off-design
        cycle_DES = self.prob.model.add_subsystem('DESIGN', Cycle())
        cycle_DES.options['thermo_method'] = 'CEA'
        cycle_DES.options['thermo_data'] = species_data.janaf

        cycle_OD = self.prob.model.add_subsystem('OFF_DESIGN', Cycle())
        cycle_OD.options['design'] = False
        cycle_OD.options['thermo_method'] = 'CEA'
        cycle_OD.options['thermo_data'] = species_data.janaf

        cycle_DES.add_subsystem('flow_start', FlowStart(), promotes=['P', 'T', 'MN', 'W'])

        cycle_OD.add_subsystem('flow_start_OD', FlowStart(), promotes=['P', 'T', 'W', 'MN'])

        expMN = 1.0
        cycle_DES.add_subsystem('duct', Duct(expMN=expMN), promotes=['MN'])
        cycle_OD.add_subsystem('duct', Duct(expMN=expMN, design=False))

        cycle_DES.pyc_connect_flow('flow_start.Fl_O', 'duct.Fl_I')
        cycle_OD.pyc_connect_flow('flow_start_OD.Fl_O', 'duct.Fl_I')

        cycle_DES.set_input_defaults('P', 17., units='psi')
        cycle_DES.set_input_defaults('T', 500., units='degR')
        cycle_DES.set_input_defaults('MN', 0.5)
        cycle_DES.set_input_defaults('duct.dPqP', 0.0)
        cycle_DES.set_input_defaults('W', 500., units='lbm/s')

        cycle_OD.set_input_defaults('P', 17., units='psi')
        cycle_OD.set_input_defaults('T', 500., units='degR')
        cycle_OD.set_input_defaults('MN', 0.25)
        cycle_OD.set_input_defaults('duct.dPqP', 0.0)
        cycle_OD.set_input_defaults('W', 500., units='lbm/s')

        self.prob.model.connect("DESIGN.duct.s_dPqP", "OFF_DESIGN.duct.s_dPqP")
        self.prob.model.connect("DESIGN.duct.Fl_O:stat:area", "OFF_DESIGN.duct.area")

        self.prob.setup(check=False, force_alloc_complex=True)
        self.prob.set_solver_print(level=-1)

        data = ref_data[0]
        self.prob['DESIGN.duct.dPqP'] = data[h_map['dPqP']]

        # input flowstation
        self.prob['DESIGN.P'] = data[h_map['Fl_I.Pt']]
        self.prob['DESIGN.T'] = data[h_map['Fl_I.Tt']]
        self.prob['DESIGN.MN'] = data[h_map['Fl_O.MN']]
        self.prob['DESIGN.W'] = data[h_map['Fl_I.W']]
        # self.prob['DESIGN.duct.Fl_I:stat:V'] = data[h_map['Fl_I.V']]

        # input flowstation
        self.prob['OFF_DESIGN.P'] = data[h_map['Fl_I.Pt']]
        self.prob['OFF_DESIGN.T'] = data[h_map['Fl_I.Tt']]
        self.prob['OFF_DESIGN.W'] = data[h_map['Fl_I.W']]
        # self.prob['OFF_DESIGN.duct.Fl_I:stat:V'] = data[h_map['Fl_I.V']]

        # give a decent initial guess for Ps

        print(self.prob['DESIGN.P'], self.prob['DESIGN.T'], self.prob['DESIGN.MN'])

        self.prob.run_model()

        # check outputs
        pt, ht, ps, ts = data[h_map['Fl_O.Pt']], data[
            h_map['Fl_O.ht']], data[h_map['Fl_O.Ps']], data[h_map['Fl_O.Ts']]
        pt_computed = self.prob['OFF_DESIGN.duct.Fl_O:tot:P']
        ht_computed = self.prob['OFF_DESIGN.duct.Fl_O:tot:h']
        ps_computed = self.prob['OFF_DESIGN.duct.Fl_O:stat:P']
        ts_computed = self.prob['OFF_DESIGN.duct.Fl_O:stat:T']

        tol = 1.0e-4
        assert_near_equal(pt_computed, 8.84073152, tol)
        assert_near_equal(ht_computed, ht, tol)
        assert_near_equal(ps_computed, 8.26348914, tol)
        assert_near_equal(ts_computed, ts, tol)

        partial_data = self.prob.check_partials(out_stream=None, method='cs',
                                                    includes=['duct.*'], excludes=['*.base_thermo.*',])
        assert_check_partials(partial_data, atol=1e-8, rtol=1e-8)

if __name__ == "__main__":
    unittest.main()

max_stars_repo_name: anilyil/pyCycle

max_issues_repo_name: anilyil/pyCycle

max_forks_repo_name: anilyil/pyCycle