text:
import random
import networkx as nx
from collections import Counter
from inspect import signature

from .types import NodeSpec, EdgeSpec
from .generate_graph import StationProperties, LineProperties

from typing import List, Dict

import logging
logger = logging.getLogger(__name__)

# --------------------------------------------------------------------------
# Executable syntax tree to represent and calculate answers
# --------------------------------------------------------------------------

class FunctionalOperator(object):
	def __init__(self, *args):
		self.args = args

	def __call__(self, graph):
		"""Execute this whole program to get an answer"""

		def ex(item):
			if isinstance(item, FunctionalOperator):
				return item(graph)
			else:
				return item

		vals = [ex(i) for i in self.args]
		try:
			return self.op(graph, *vals)
		except Exception as ex:
			logger.debug("Failed to execute operation {}({}) {}".format(type(self).__name__, vals, ex))
			raise ex

	def op(self, *args):
		"""
		Perform this individual operation

		Operations should raise ValueError if it is not possible to generate
		a valid answer, but no error has occured. This exception will be silently
		swallowed.

		"""
		raise NotImplementedError()

	def stripped(self):
		"""Represent this program for export"""

		def ex(item):
			try:
				return item.stripped()
			except AttributeError:

				# YAML export will freak out if it hits a lambda, so symbolically replace it
				if callable(item):
					sig = signature(item)
					args = [LambdaArg(i) for i in sig.parameters]
					return Lambda(item(*args)).stripped()
				else:
					return item

		k = [ex(i) for i in self.args]

		r = {}
		r[type(self).__name__] = k
		return r

def macro(f):
	return f

# --------------------------------------------------------------------------
#  Noun operations
# --------------------------------------------------------------------------

class Station(FunctionalOperator):
	@classmethod
	def get(self, graph):
		return Station(random.choice(list(graph.nodes.values())))

class FakeStationName(FunctionalOperator):
	@classmethod
	def get(self, graph):
		# This needs generalised later
		actual_station_names = {str(j.name()) for j in graph.nodes.values()}
		max_stn = len(graph.nodes) * 2
		nonexistent_stations = [i for i in range(max_stn) if str(i) not in actual_station_names]
		return FakeStationName(random.choice(nonexistent_stations))

class StationPropertyName(FunctionalOperator):
	@classmethod
	def get(self, graph):
		return StationPropertyName(random.choice(StationProperties.keys()))

class StationProperty(FunctionalOperator):
	@classmethod
	def get(self, graph):
		key = random.choice(list(StationProperties.keys()))
		return StationProperty(key, StationProperties[key])

class Line(FunctionalOperator):
	@classmethod
	def get(self, graph):
		return Line(random.choice(list(graph.lines.values())))

class Architecture(FunctionalOperator):
	@classmethod
	def get(self, graph):
		return Architecture(random.choice(StationProperties["architecture"]))

class Size(FunctionalOperator):
	@classmethod
	def get(self, graph):
		return Size(random.choice(StationProperties["size"]))

class Music(FunctionalOperator):
	@classmethod
	def get(self, graph):
		return Music(random.choice(StationProperties["music"]))

class Cleanliness(FunctionalOperator):
	@classmethod
	def get(self, graph):
		return Cleanliness(random.choice(StationProperties["cleanliness"]))

class Boolean(FunctionalOperator):
	@classmethod
	def get(self, graph):
		return Boolean(random.choice([True, False]))

# --------------------------------------------------------------------------
# General operations
# --------------------------------------------------------------------------

class Const(FunctionalOperator):
	def op(self, graph, a):
		return a

class Lambda(FunctionalOperator):
	def op(self, graph, a):
		return a

class LambdaArg(FunctionalOperator):
	def op(self, graph, a):
		return a

class Pluck(FunctionalOperator):
	def op(self, graph, a, b):
		return [i[b] for i in a]

class Pick(FunctionalOperator):
	def op(self, graph, a, b):
		return a[b]

class Equal(FunctionalOperator):
	def op(self, graph, a, b):
		return a == b

# --------------------------------------------------------------------------
# Graph operations
# --------------------------------------------------------------------------

class AllEdges(FunctionalOperator):
	def op(self, graph):
		return graph.edges

class AllNodes(FunctionalOperator):
	def op(self, graph):
		return graph.nodes.values()

class Edges(FunctionalOperator):
	def op(self, graph, a):
		if isinstance(a, NodeSpec):
			return [edge[2]['attr_dict'] for edge in graph.gnx.edges([a["id"]], data=True)]
		else:
			return [
				edge[2]['attr_dict']
				for node in a
				for edge in graph.gnx.edges([node["id"]], data=True)
			]

class Nodes(FunctionalOperator):
	def op(self, graph, edges:EdgeSpec):
		n = []
		for i in edges:
			n.append(graph.nodes[i["station1"]])
			n.append(graph.nodes[i["station2"]])
		return list(set(n))

def ids_to_nodes(graph, ids):
	return [graph.nodes[i] for i in ids]

class ShortestPath(FunctionalOperator):
	def op(self, graph, a:NodeSpec, b:NodeSpec, fallback):
		try:
			return ids_to_nodes(graph, nx.shortest_path(graph.gnx, a["id"], b["id"]))
		except nx.exception.NetworkXNoPath:
			return fallback

class ShortestPathOnlyUsing(FunctionalOperator):
	def op(self, graph, a:NodeSpec, b:NodeSpec, only_using_nodes:List[NodeSpec], fallback):
		try:
			induced_subgraph = nx.induced_subgraph(graph.gnx, [i["id"] for i in only_using_nodes + [a,b]])
			return ids_to_nodes(graph, nx.shortest_path(induced_subgraph, a["id"], b["id"]))
		except nx.exception.NetworkXNoPath:
			return fallback

class Paths(FunctionalOperator):
	def op(self, graph, a:NodeSpec, b:NodeSpec):
		return [ids_to_nodes(graph, i) for i in nx.all_simple_paths(graph.gnx, a["id"], b["id"])]

class HasCycle(FunctionalOperator):
	def op(self, graph, a:NodeSpec):

		# Would all_simple_paths also solve this for us?

		def canonical_edge(e):
			return (frozenset(e[:2]), e[2]["attr_dict"]["line_id"])

		def dfs_unidirected_cycle(head_id, path_nodes=frozenset(), path_edges=frozenset(), indent=""):
			for e in graph.gnx.edges([head_id], data=True):
				assert e[0] == head_id
				assert head_id in path_nodes

				from_id = head_id
				to_id = e[1]

				# Nothing new
				if canonical_edge(e) in path_edges:
					continue

				# If we've returned home
				if to_id == a["id"]:
					return True

				# Nothing new
				if to_id in path_nodes:
					continue

				ir = dfs_unidirected_cycle(
					to_id,
					path_nodes | set([to_id]),
					path_edges | set([canonical_edge(e)]),
					indent=indent+"  ",
				)

				if ir:
					return True

			return False
		return dfs_unidirected_cycle(a["id"], frozenset([a["id"]]))

class FilterAdjacent(FunctionalOperator):
	def op(self, graph, a:List, b:List):
		r = []
		for i in a:
			for j in b:
				ns = graph.gnx.neighbors(i["id"])
				if j["id"] in ns:
					r.append([i,j])
		return r

class Neighbors(FunctionalOperator):
	def op(self, graph, station:NodeSpec):
		return ids_to_nodes(graph, graph.gnx.neighbors(station["id"]))

class WithinHops(FunctionalOperator):
	def op(self, graph, station:NodeSpec, hops:int):
		rs = set()
		tips = set([station])
		for i in range(hops):
			next_tips = set()
			for j in tips:
				next_tips |= set(ids_to_nodes(graph, graph.gnx.neighbors(j["id"])))

			rs |= tips
			tips = next_tips - rs

		rs |= next_tips
		rs.remove(station)
		return list(rs)

class FilterHasPathTo(FunctionalOperator):
	def op(self, graph, a:List, b:NodeSpec):
		return [i for i in a if nx.has_path(graph.gnx, i["id"], b["id"])]

# --------------------------------------------------------------------------
# List operators
# --------------------------------------------------------------------------

class NotEmpty(FunctionalOperator):
	def op(self, graph, l):
		return len(l) > 0

class Count(FunctionalOperator):
	def op(self, graph, l):
		return len(l)

class CountIfEqual(FunctionalOperator):
	def op(self, graph, l, t):
		return len([i for i in l if i == t])

class Mode(FunctionalOperator):
	def op(self, graph, l):

		if len(l) == 0:
			raise ValueError("Cannot find mode of empty sequence")

		c = Counter(l)
		most = c.most_common(2)

		# Only one unique value in l
		if len(most) == 1:
			return most[0][0]

		# If the most common occurs more than any other
		if most[0][1] > most[1][1]:
			return most[0][0]

		raise ValueError("No unique mode")

class Unique(FunctionalOperator):
	def op(self, graph, l):
		return list(set(l))

class SlidingPairs(FunctionalOperator):
	def op(self, graph, l):
		return [(l[i], l[i+1]) for i in range(len(l)-1)]

@macro
def GetLines(a):
	return Unique(Pluck(Edges(a), "line_name"))

@macro
def Adjacent(a, b):
	return Equal(Count(ShortestPath(a, b, [])), 2)

@macro
def CountNodesBetween(a):
	return Subtract(Count(a), 2)

class HasIntersection(FunctionalOperator):
	def op(self, graph, a, b):
		for i in a:
			if i in b:
				return True
		return False

class Intersection(FunctionalOperator):
	def op(self, graph, a, b):
		return list(set(a) & set(b))

class Filter(FunctionalOperator):
	def op(self, graph, a:List, b, c):
		return [i for i in a if i[b] == c]

class Without(FunctionalOperator):
	def op(self, graph, a:List, b, c):
		return [i for i in a if i[b] != c]

class UnpackUnitList(FunctionalOperator):
	"""This operator will raise if the given list is not length 1 - this is used as a guard against generating ambiguous questions"""
	def op(self, graph, l:List):
		if len(l) == 1:
			return l[0]
		else:
			raise ValueError(f"List is length {len(l)}, expected 1")

class Sample(FunctionalOperator):
	def op(self, graph, l:List, n:int):
		if len(l) < n:
			raise ValueError(f"Cannot sample {n} items from list of length {len(l)}")
		else:
			return random.choices(l, k=n)

class First(FunctionalOperator):
	def op(self, graph, l:List):
		return l[0]

class MinBy(FunctionalOperator):
	def op(self, graph, a, b):
		if len(a) == 0:
			raise ValueError("Cannot perform MinBy on empty list")
		return min(a, key=lambda i: b(i)(graph))

# --------------------------------------------------------------------------
# Numerical operations
# --------------------------------------------------------------------------

class Subtract(FunctionalOperator):
	def op(self, graph, a, b):
		return a - b

class Round(FunctionalOperator):
	def op(self, graph, a):
		try:
			return [round(float(i)) for i in a]
		except TypeError:
			return round(float(a))

max_stars_repo_name: sdliuyuzhi/clevr-graph

max_issues_repo_name: sdliuyuzhi/clevr-graph

max_forks_repo_name: sdliuyuzhi/clevr-graph

reason: import networkx


text:
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#
# Copyright (c) 2019, Eurecat / UPF
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#     * Neither the name of the <organization> nor the
#       names of its contributors may be used to endorse or promote products
#       derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#
#   @file   test_script_sh.py
#   @author Andrés Pérez-López
#   @date   24/07/2019
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #

import numpy as np
from masp import shoebox_room_sim as srs
import time
import librosa
import sys
import scipy.signal
pp = "/Users/andres.perez/source/parametric_spatial_audio_processing"
sys.path.append(pp)
import parametric_spatial_audio_processing as psa
import matplotlib.pyplot as plt
# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
# SETUP

# Room definition
room = np.array([10, 10, 10])

# Desired RT per octave band, and time to truncate the responses
rt60 = np.array([0.5])
nBands = len(rt60)

# Generate octave bands
band_centerfreqs = np.empty(nBands)
band_centerfreqs[0] = 1000

# Absorption for approximately achieving the RT60 above - row per band
abs_wall = srs.find_abs_coeffs_from_rt(room, rt60)[0]

# Critical distance for the room
_, d_critical, _ = srs.room_stats(room, abs_wall)

# Receiver position
rec = np.array([ [5, 5, 5] ])
nRec = rec.shape[0]

# Source positions
src = np.array([ [5, 5, 6] ])
nSrc = src.shape[0]

# SH orders for receivers
rec_orders = np.array([1]) # rec1: first order(4ch), rec2: 3rd order (16ch)

# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
# RUN SIMULATOR

# Echogram
tic = time.time()

maxlim = 1.5 # just stop if the echogram goes beyond that time ( or just set it to max(rt60) )
limits = np.minimum(rt60, maxlim)

# Compute echograms
# abs_echograms, rec_echograms, echograms = srs.compute_echograms_sh(room, src, rec, abs_wall, limits, rec_orders)
abs_echograms = srs.compute_echograms_sh(room, src, rec, abs_wall, limits, rec_orders)

# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
# RENDERING

# In this case all the information (e.g. SH directivities) are already
# encoded in the echograms, hence they are rendered directly to discrete RIRs
fs = 48000
sh_rirs = srs.render_rirs_sh(abs_echograms, band_centerfreqs, fs).squeeze()
sh_rirs = sh_rirs * np.sqrt(4*np.pi) * [1, 1./np.sqrt(3), 1./np.sqrt(3), 1./np.sqrt(3)]  # SN3D norm
plt.figure()
plt.plot(sh_rirs)
plt.show()

signal_len_samples = int(np.floor(1. * fs))
signal = np.random.randn(signal_len_samples)

reverberant_signal = np.zeros((signal_len_samples, 4))
for i in range(4):
    reverberant_signal[:,i] = scipy.signal.fftconvolve(signal, sh_rirs[:,i].squeeze())[:signal_len_samples]
x = psa.Signal(reverberant_signal.T, fs, 'acn', 'sn3d')
psa.plot_signal(x,title='waveform')

analysis_window_size = 512
window_overlap = analysis_window_size // 2
fft_size = analysis_window_size
stft = psa.Stft.fromSignal(x,
                           window_size=analysis_window_size,
                           window_overlap=window_overlap,
                           nfft=fft_size)
psa.plot_magnitude_spectrogram(stft,title='magnitude spectrogram')
doa = psa.compute_DOA(stft)
psa.plot_doa(doa,title='doa')

plt.show()

i = psa.compute_intensity_vector(stft)

psa.plot_magnitude_spectrogram(i)
plt.show()

ext: py

max_stars_repo_name: andresperezlopez/masp

max_issues_repo_name: andresperezlopez/masp

max_forks_repo_name: andresperezlopez/masp

reason: import numpy,import scipy


text:
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from numba import cfunc

# A helper function to write a python function as LLVM bitcode
def write_pyfunc_as_bitcode(pyfunc, sig, filename_suffix=""):
  with open(pyfunc.__name__ + filename_suffix + ".bc", "wb") as fout:
    f = cfunc(sig)(pyfunc)
    fout.write(f._library._final_module.as_bitcode())

def pyAdd(a, b):
  return a + b;

def pySum(x):
  (s, ) = x.shape
  sum = 0.0
  for i in range(s):
    sum += x[i]
  return sum

write_pyfunc_as_bitcode(pyAdd, "float32(float32, float32)", "-float32")
write_pyfunc_as_bitcode(pySum, "float32(float32[:])", "-float32")

max_stars_repo_name: maropu/lljvm-translator

max_issues_repo_name: maropu/lljvm-as

max_forks_repo_name: maropu/lljvm-as

reason: from numba


text:
import unittest
import sys
import os
import numpy as np
sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))

from matlabparser import *
from matlabparser import parser as mparser

# --------------------------------------------------------------------------------}
# --- Convert matlab files to python files
# --------------------------------------------------------------------------------{

class options:
    def __init__(self,output=None):
        self.numbers=False
        self.no_comments=False
        self.no_resolve=False
        self.smop=False
        self.output=output

def convert(mfile):
    opts = options(mfile.replace('.m','.py'))
    #print('>>>>>>')
    #print('>>>>>> Converting:',mfile)
    #     opts.smop=True
    mparser.matlab2python(mfile,opts)

def compare_to_ref(mfile):
    opts = options(mfile.replace('.m','.py'))
    ref_output =mfile.replace('.m','_ref.py')
    #print('>>>>>> Converting:',mfile)
    mparser.matlab2python(mfile,opts)
    #print('>>>>>>')

class TestMatlab2Python(unittest.TestCase):

    def test_spectrum(self):
        convert('_tests/files/run_all.m')
        compare_to_ref('_tests/files/fSpectrum.m')

# --------------------------------------------------------------------------------}
# --- Try to run the converted files
# --------------------------------------------------------------------------------{

class TestZAllConverted(unittest.TestCase):
    def test_Zspectrum(self):
        import _tests.files
        from _tests.files.fSpectrum import fSpectrum
        dt = 0.1
        t = np.arange(0,1+dt,dt)
        y = np.sin(t)
        #S,f = fSpectrum(y,len(y),1 / dt)
        S_ref = np.array([0.2285364,0.0258482,0.0066528,0.0033719,0.0023203,0.0019575])
        f_ref = np.array([0.0,0.90909,1.81818,2.72727,3.63636,4.54545])
        #print(S-S_ref)
        #print(f-f_ref)

if __name__ == '__main__':
    unittest.main()

reason: import numpy


text:
import time
import pandas as pd
import numpy as np

CITY_DATA = { 'chicago': 'chicago.csv',
              'new york city': 'new_york_city.csv',
              'washington': 'washington.csv' }

MONTH_DATA = ['all', 'january', 'february', 'march', 'april', 'may', 'june']

DAY_DATA = ['all', 'monday', 'tuesday', 'wednesday', 'friday', 'saturday', 'sunday']

def get_filters():
    """
    Asks user to specify a city, month, and day to analyze.

    Returns:
        (str) city - name of the city to analyze
        (str) month - name of the month to filter by, or "all" to apply no month filter
        (str) day - name of the day of week to filter by, or "all" to apply no day filter
    """
    print('Hello! Let\'s explore some US bikeshare data!')
    # TO DO: get user input for city (chicago, new york city, washington). HINT: Use a while loop to handle invalid inputs
    city_name = ''
    while city_name.lower() not in CITY_DATA:
        city_name = input("\nWhat is the name of the city to analyze data? (E.g. Input either chicago, new york city, washington)\n")
        if city_name.lower() in CITY_DATA:
            #We were able to get the name of the city to analyze data.
            city = CITY_DATA[city_name.lower()]
        else:
            #We were not able to get the name of the city to analyze data so we continue the loop.
            print("Sorry we were not able to get the name of the city to analyze data, Please input either chicago, new york city or washington.\n")

    # TO DO: get user input for month (all, january, february, ... , june)
    month_name = ''
    while month_name.lower() not in MONTH_DATA:
        month_name = input("\nWhat is the name of the month to filter data? (E.g. Input either 'all' to apply no month filter or january, february, ... , june)\n")
        if month_name.lower() in MONTH_DATA:
            #We were able to get the name of the month to analyze data.
            month = month_name.lower()
        else:
            #We were not able to get the name of the month to analyze data so we continue the loop.
            print("Sorry we were not able to get the name of the month to filter data, Please input either 'all' to apply no month filter or january, february, ... , june.\n")

    # TO DO: get user input for day of week (all, monday, tuesday, ... sunday)
    day_name = ''
    while day_name.lower() not in DAY_DATA:
        day_name = input("\nWhat is the name of the day to filter data? (E.g. Input either 'all' to apply no day filter or monday, tuesday, ... sunday)\n")
        if day_name.lower() in DAY_DATA:
            #We were able to get the name of the month to analyze data.
            day = day_name.lower()
        else:
            #We were not able to get the name of the month to analyze data so we continue the loop.
            print("Sorry we were not able to get the name of the day to filter data, Please input either 'all' to apply no day filter or monday, tuesday, ... sunday.\n")

    print('-'*40)
    return city, month, day

def load_data(city, month, day):
    """
    Loads data for the specified city and filters by month and day if applicable.

    Args:
        (str) city - name of the city to analyze
        (str) month - name of the month to filter by, or "all" to apply no month filter
        (str) day - name of the day of week to filter by, or "all" to apply no day filter
    Returns:
        df - Pandas DataFrame containing city data filtered by month and day
    """
    # load data file into a dataframe
    df = pd.read_csv(city)

    # convert the Start Time column to datetime
    df['Start Time'] = pd.to_datetime(df['Start Time'])

    # extract month and day of week from Start Time to create new columns
    df['month'] = df['Start Time'].dt.month
    df['day_of_week'] = df['Start Time'].dt.weekday_name
    df['hour'] = df['Start Time'].dt.hour

    # filter by month if applicable
    if month != 'all':
        # use the index of the months list to get the corresponding int
        month = MONTH_DATA.index(month)

        # filter by month to create the new dataframe
        df = df.loc[df['month'] == month]

    # filter by day of week if applicable
    if day != 'all':
        # filter by day of week to create the new dataframe
        df = df.loc[df['day_of_week'] == day.title()]

    return df

def time_stats(df):
    """Displays statistics on the most frequent times of travel.

    Args:
        (DataFrame) df - Pandas DataFrame containing city data filtered by month and day
    """

    print('\nCalculating The Most Frequent Times of Travel...\n')
    start_time = time.time()

    # TO DO: display the most common month
    common_month = df['month'].mode()[0]
    print("The most common month from the given fitered data is: " + MONTH_DATA[common_month].title())

    # TO DO: display the most common day of week
    common_day_of_week = df['day_of_week'].mode()[0]
    print("The most common day of week from the given fitered data is: " + common_day_of_week)

    # TO DO: display the most common start hour
    common_start_hour = df['hour'].mode()[0]
    print("The most common start hour from the given fitered data is: " + str(common_start_hour))

    print("\nThis took %s seconds." % (time.time() - start_time))
    print('-'*40)

def station_stats(df):
    """Displays statistics on the most popular stations and trip.

    Args:
        (DataFrame) df - Pandas DataFrame containing city data filtered by month and day
    """

    print('\nCalculating The Most Popular Stations and Trip...\n')
    start_time = time.time()

    # TO DO: display most commonly used start station
    common_start_station = df['Start Station'].mode()[0]
    print("The most commonly used start station from the given fitered data is: " + common_start_station)

    # TO DO: display most commonly used end station
    common_end_station = df['End Station'].mode()[0]
    print("The most commonly used end station from the given fitered data is: " + common_end_station)

    # TO DO: display most frequent combination of start station and end station trip
    frequent_combination = (df['Start Station'] + "||" + df['End Station']).mode()[0]
    print("The most frequent combination of start station and end station trip is : " + str(frequent_combination.split("||")))

    print("\nThis took %s seconds." % (time.time() - start_time))
    print('-'*40)

def trip_duration_stats(df):
    """Displays statistics on the total and average trip duration.

    Args:
        (DataFrame) df - Pandas DataFrame containing city data filtered by month and day
    """

    print('\nCalculating Trip Duration...\n')
    start_time = time.time()

    # TO DO: display total travel time
    total_travel_time = df['Trip Duration'].sum()
    print("The total travel time from the given fitered data is: " + str(total_travel_time))

    # TO DO: display mean travel time
    mean_travel_time = df['Trip Duration'].mean()
    print("The mean travel time from the given fitered data is: " + str(mean_travel_time))

    print("\nThis took %s seconds." % (time.time() - start_time))
    print('-'*40)

def user_stats(df, city):
    """Displays statistics on bikeshare users.

    Args:
        (DataFrame) df - Pandas DataFrame containing city data filtered by month and day
    """

    print('\nCalculating User Stats...\n')
    start_time = time.time()

    # TO DO: Display counts of user types
    user_types = df['User Type'].value_counts()
    print("The count of user types from the given fitered data is: \n" + str(user_types))

    if city == 'chicago.csv' or city == 'new_york_city.csv':
        # TO DO: Display counts of gender
        gender = df['Gender'].value_counts()
        print("The count of user gender from the given fitered data is: \n" + str(gender))

        # TO DO: Display earliest, most recent, and most common year of birth
        earliest_birth = df['Birth Year'].min()
        most_recent_birth = df['Birth Year'].max()
        most_common_birth = df['Birth Year'].mode()[0]
        print('Earliest birth from the given fitered data is: {}\n'.format(earliest_birth))
        print('Most recent birth from the given fitered data is: {}\n'.format(most_recent_birth))
        print('Most common birth from the given fitered data is: {}\n'.format(most_common_birth) )

    print("\nThis took %s seconds." % (time.time() - start_time))
    print('-'*40)

def display_raw_data(df):
    """Displays raw data on user request.

    Args:
        (DataFrame) df - Pandas DataFrame containing city data filtered by month and day
    """
    print(df.head())
    next = 0
    while True:
        view_raw_data = input('\nWould you like to view next five row of raw data? Enter yes or no.\n')
        if view_raw_data.lower() != 'yes':
            return
        next = next + 5
        print(df.iloc[next:next+5])

def main():
    while True:
        city, month, day = get_filters()
        df = load_data(city, month, day)

        time_stats(df)
        station_stats(df)
        trip_duration_stats(df)
        user_stats(df, city)
        while True:
            view_raw_data = input('\nWould you like to view first five row of raw data? Enter yes or no.\n')
            if view_raw_data.lower() != 'yes':
                break
            display_raw_data(df)
            break

        restart = input('\nWould you like to restart? Enter yes or no.\n')
        if restart.lower() != 'yes':
            break

if __name__ == "__main__":
    main()


text:
import itertools
import multiprocessing
import os
import random
import shutil
import unittest

import exemcl
import numpy as np
import pandas
from sklearn.datasets import make_blobs

DIM = 10
N = 500

def L(V, S):
    accu = 0
    for e in V:
        accu += min([pow(np.linalg.norm(e - v), 2.0) for v in S])
    return accu / len(V)

def F(V, S):
    zero = np.zeros(DIM)
    zeroVecValue = L(V, [zero])
    return zeroVecValue - L(V, np.append(S, [zero], axis=0))

def F_gain(V, S, e):
    return F(V, np.append(S, [e], axis=0)) - F(V, S)

class TestCase(unittest.TestCase):
    @classmethod
    def setUpClass(cls) -> None:
        # Clean test file dir.
        shutil.rmtree(os.path.join(os.path.dirname(os.path.realpath(__file__)), "testfiles"), ignore_errors=True)

        # Define tolerancies.
        cls.rtol = 1e-02
        cls.atol = 1e-05

    def test_ExemplarBasedClustering(self):
        # Generate ground set.
        V, _ = make_blobs(n_samples=N, centers=3, n_features=DIM, random_state=0)
        V_frame = pandas.DataFrame(V)

        # Generate subsets.
        subsets_df = pandas.DataFrame()
        S = list()
        for i in range(50):
            subset_df = V_frame.iloc[np.random.choice(range(N), random.randint(int(0.1 * N), int(0.5 * N)))]
            S.append(subset_df.values)
            subset_df["subset_idx"] = i
            subsets_df = subsets_df.append(subset_df)

        # Calculate f(s) for every s \in S
        pool = multiprocessing.Pool()
        f_values = pool.starmap(F, [(V, s) for s in S])
        pool.close()
        pool.join()

        # Calculate marginal gains.
        pool = multiprocessing.Pool()
        e = V[len(V) - 1]
        marginal_values = pool.starmap(F_gain, [(V, s, e) for s in S])
        pool.close()
        pool.join()

        # Compare to output.
        for precision, device in itertools.product(["fp16", "fp32", "fp64"], ["cpu", "gpu"]):
            # Skip FP16/CPU combination.
            if precision == "fp16" and device == "cpu":
                continue

            # Make comparison.
            exem = exemcl.ExemplarClustering(ground_set=V, precision=precision, device=device, worker_count=-1)
            f_values_exemcl = exem(S)
            marginal_values_exemcl = exem(S, e)
            self.assertTrue(np.allclose(np.asarray(f_values), np.asarray(f_values_exemcl), atol=self.atol, rtol=self.rtol))
            self.assertTrue(np.allclose(np.asarray(marginal_values), np.asarray(marginal_values_exemcl), atol=self.atol, rtol=self.rtol))

        # Write output files.
        out_dir = os.path.dirname(os.path.realpath(__file__))
        os.makedirs(os.path.join(out_dir, "testfiles", "exem"))
        V_frame.to_csv(os.path.join(out_dir, "testfiles", "exem", "ground_set.csv"), index_label="ObjectID")
        subsets_df.reset_index(drop=True).to_csv(os.path.join(out_dir, "testfiles", "exem", "subsets.csv"), index_label="ObjectID")
        pandas.DataFrame(f_values).to_csv(os.path.join(out_dir, "testfiles", "exem", "f_values.csv"), index_label="ObjectID")
        pandas.DataFrame(marginal_values).to_csv(os.path.join(out_dir, "testfiles", "exem", "marginal_values.csv"), index_label="ObjectID")

if __name__ == '__main__':
    unittest.main()

max_stars_repo_name: philippjh/exemcl

max_issues_repo_name: philippjh/exemcl

max_forks_repo_name: philippjh/exemcl


text:
import argparse
import sys
import jsonlines
from tqdm import tqdm
import logging
import json
import torch
import torch.nn.functional as F
import jsonlines
import random
import os
import numpy as np
from scipy.special import softmax
# os.environ["NCCL_SHM_DISABLE"] = "1"
from tqdm import tqdm
from typing import List
from sklearn.metrics import f1_score, precision_score, recall_score
from datasets import Dataset
from torch.utils.data import Dataset, DataLoader
from transformers import AutoConfig, AutoTokenizer, AutoModelForSequenceClassification, get_cosine_schedule_with_warmup
from transformers import AutoModelForSequenceClassification, TrainingArguments, Trainer, EvalPrediction, default_data_collator, set_seed
from transformers import InputExample, PreTrainedTokenizer, InputFeatures
import os
os.environ["TOKENIZERS_PARALLELISM"] = "false"

LABELS = ["SUPPORTS", "REFUTES", "NOT ENOUGH INFO"]

def get_json_lines(inp_file):
    lines = []
    with jsonlines.open(inp_file) as reader:
        for obj in reader:
            lines.append(obj)

    return lines

def write_json_lines(output_file_name, list_data, output_folder):
    with jsonlines.open(output_folder+ output_file_name, mode='w') as writer:
        for dataline in list_data:
            writer.write(dataline)

class ClassificationModel():
    def __init__(self, num_labels=2, max_length=256, model_name_or_path='albert-large-v2', config_name=None, tokenizer_name=None):
        NUM_LABELS = num_labels
        self.max_seq_length = 256
        self.model_name_or_path = model_name_or_path
        self.config_name = config_name
        self.tokenizer_name = tokenizer_name
        self.max_length = max_length

        config = AutoConfig.from_pretrained(
            self.config_name if self.config_name else self.model_name_or_path,
            num_labels=NUM_LABELS,
            # cache_dir='.cache/',
        )
        add_prefix_space = False
        if 'roberta' in self.model_name_or_path:
            add_prefix_space = True
        self.tokenizer = AutoTokenizer.from_pretrained(
            self.tokenizer_name if self.tokenizer_name else self.model_name_or_path,
            # cache_dir=model_args.cache_dir,
            add_prefix_space=True,
            # use_fast=True,
        )
        self.model = AutoModelForSequenceClassification.from_pretrained(
            self.model_name_or_path,
            from_tf=bool(".ckpt" in self.model_name_or_path),
            config=config,
            # cache_dir=args.cache_dir,
        )

    def get_string_text(self, tokens_a, tokens_b):
        max_num_tokens = self.max_seq_length - 3
        total_length = len(tokens_a) + len(tokens_b)
        if total_length > max_num_tokens:
            len_b = len(tokens_b)
            a_begin = max_num_tokens - len_b
            tokens_a = tokens_a[-a_begin:]
        try:
            assert len(tokens_a) + len(tokens_b) <= max_num_tokens
            assert len(tokens_a) >= 1
        except:
            import pdb;
            pdb.set_trace()
            print('some problem with preproc')
        # assert len(tokens_b) >= 1

        tokens = []
        segment_ids = []
        tokens.append(self.tokenizer.cls_token)
        segment_ids.append(0)
        for token in tokens_a:
            tokens.append(token)
            segment_ids.append(0)

        tokens.append(self.tokenizer.sep_token)
        segment_ids.append(0)

        for token in tokens_b:
            tokens.append(token)
            segment_ids.append(1)
        tokens.append(self.tokenizer.sep_token)
        segment_ids.append(1)

        return tokens, segment_ids

    def tokenize_function_test(self, examples):
        # Remove empty lines
        #         examples["text"] = [line for line in examples["text"] if len(line) > 0 and not line.isspace()]
        #         examples = [line for line in examples if len(line) > 0 and not line.isspace()]
        all_texts = []
        all_segment_ids = []
        all_labels = []
        #     import pdb;pdb.set_trace()
        processed = []
        items = []
        # keys = list(examples.keys())
        # for i in range(len(examples[keys[0]])):
        #     ex = {}
        #     for k in keys:
        #         ex[k] = examples[k][i]
        #     items.append(ex)
        #     import pdb;pdb.set_trace()
        items = examples
        max_seq_length = 216
        for example in items:
            first_tokens = self.tokenizer.tokenize(example['actual'])
            for sent2 in example['prediction']:
                sec_tokens = self.tokenizer.tokenize(sent2)
                tokens = ["[CLS]"] + first_tokens + ["[SEP]"] + sec_tokens
                if len(sec_tokens) + len(first_tokens) > max_seq_length - 1:
                    tokens = tokens[:(max_seq_length - 1)]
                tokens = tokens + ["[SEP]"]

                segment_ids = [0] * (len(first_tokens) + 2)
                segment_ids += [1] * (len(sec_tokens) + 1)
                all_texts.append(tokens)
                all_segment_ids.append(segment_ids)

        tokenized = self.tokenizer.batch_encode_plus(
            all_texts,
            padding='max_length',
            truncation=True,
            max_length=max_seq_length,
            is_split_into_words=True,
            return_special_tokens_mask=True,
            add_special_tokens=False,
        )

        # print(len(tokenized['input_ids']))
        padded_length = len(tokenized['input_ids'][0])
        all_segment_ids = [x + [0] * (padded_length - len(x)) for x in all_segment_ids]
        tokenized['token_type_ids'] = all_segment_ids
        # tokenized['label'] = all_labels

        return tokenized

    def tokenize_function(self, examples, sent2_type='evidence_touse', sent1_type='prediction'):
        all_texts = []
        all_segment_ids = []
        all_labels = []
        processed = []
        items = []
        max_seq_length = 216
        for example in examples:
            evidence_data = example[sent2_type]
            sent2 = evidence_data
            for p, sent1 in enumerate(example[sent1_type]):
                if type(evidence_data) is list:
                    sent2 = example[sent2_type][p]
                items.append([sent2, sent1])
        # import pdb;pdb.set_trace()
        try:
            batch_encoding = self.tokenizer(
                [(example[0], example[1])
                 for example in items],
                max_length=self.max_length,
                padding="max_length",
                truncation=True,
            )
        except:
            import pdb;pdb.set_trace()
        # import pdb;pdb.set_trace()

        features = []
        input1 = list(batch_encoding.keys())[0]
        num_inputs = len(batch_encoding[input1])
        for i in range(num_inputs):
            inputs = {k: batch_encoding[k][i] for k in batch_encoding}
            feature = InputFeatures(**inputs)
            features.append(feature)

        return features

    def tokenize_function_data(self, examples, sent2_type='evidence_touse', sent1_type='response'):
        all_texts = []
        all_segment_ids = []
        all_labels = []
        processed = []
        items = []
        max_seq_length = 256
        for example in examples:
            evidence_data = example[sent2_type]
            sent2 = evidence_data
            sent1 = example[sent1_type]
            sent1 = '[CONTEXT]: ' + ' [EOT] '.join(example['context'][-2:]) + ' [RESPONSE]: ' + sent1
            items.append([sent2, sent1])
        # import pdb;pdb.set_trace()
        try:
            batch_encoding = self.tokenizer(
                [(ex[0], ex[1])
                 for ex in items],
                max_length=self.max_length,
                padding="max_length",
                truncation=True,
            )
        except:
            import pdb;pdb.set_trace()
        # import pdb;pdb.set_trace()

        features = []
        input1 = list(batch_encoding.keys())[0]
        num_inputs = len(batch_encoding[input1])
        for i in range(num_inputs):
            inputs = {k: batch_encoding[k][i] for k in batch_encoding}
            feature = InputFeatures(**inputs)
            features.append(feature)

        return features

def create_data_loader(tokenized_eval_dataset, batch_size):

    return DataLoader(
        tokenized_eval_dataset,
        batch_size=batch_size,
        num_workers=4,
        collate_fn=default_data_collator

    )

def score_testdata(args, classification_model_dnli, testdata):
    tokenized_eval_dataset = classification_model_dnli.tokenize_function_data(testdata, sent1_type=args.response_tag)
#     import pdb;pdb.set_trace()
#     tdataset = Dataset.from_dict(tokenized_eval_dataset)
#     test_data_loader = create_data_loader(tdataset, args.batch_size)
    test_data_loader = create_data_loader(tokenized_eval_dataset, args.batch_size)
    all_scores = []
    parsed = 0
    for idx, d in enumerate(tqdm(test_data_loader)):
        input_ids = d["input_ids"].to(device)
        attention_mask = d["attention_mask"].to(device)
        token_type_ids = d["token_type_ids"].to(device)
        outputs = classification_model_dnli.model(
                  input_ids=input_ids,
                  attention_mask=attention_mask,
                token_type_ids=token_type_ids
            )

        outputs = softmax(outputs['logits'].tolist(),axis=1)
        for oidx, out in enumerate(outputs):
            softmax_l1 = out.tolist()
            # dnli_score = [x[0] for x in softmax_l1]
    #         print(softmax_l1)s
            # all_scores+=softmax_l1
            testdata[parsed][args.typeprefix+'fever_score'] = softmax_l1
            parsed+=1

def score_data(args, classification_model_dnli, max_evidences=5):
    testdata = get_json_lines(args.input_file)
    for i, datapoint in enumerate(tqdm(testdata)):
#         lines = datapoint[args.response_tag]
        if 'evidence_list' in datapoint:
            all_evidences = datapoint['evidence_list'][:max_evidences]
            # for e, evilist in enumerate(datapoint['evidence_list'][:max_evidences]):
            #     all_evidences = evilist#datapoint['evidence_list']
            #     print(all_evidences)
            #     print(['title: ' + x[0] + ' content: ' + x[2] for x in all_evidences])
            all_evidence_texts = ['title: ' + x[0] + ' content: ' + x[2] for x in all_evidences]
            # evidence_text = ' ### '.join(all_evidence_texts)
            evidence_text = ' '.join(all_evidence_texts)
            datapoint['evidence_touse'] = evidence_text

        if args.claim_only:
            datapoint['evidence_touse'] = ''
        # import pdb;pdb.set_trace()
        if len(datapoint[args.response_tag])==0:
            continue

    score_testdata(args, classification_model_dnli, testdata)
        # scores = lm_scores(lines, model, tokenizer, device)
#         datapoint['dnli_score'] = scores

    write_json_lines(args.preds_file, testdata, args.output_folder)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-c', '--cuda_device', type=int, help='id of GPU to use', default=0)
    parser.add_argument('-m', '--model', type=str, help='model name to use', default='colloquial_bert_large/')
    parser.add_argument('-i', '--input_file', type=str, help='path to the file containing the evaluation data', required=True)
    parser.add_argument('-o', '--preds_file', type=str, help='output file to save the results')
    parser.add_argument('--output_folder', type=str, help='output file to save the results', default='colloquialfeverscores/')
    parser.add_argument('--response_tag', type=str, help='tag', default='response')
    parser.add_argument('--batch_size', type=int, help='batch size', default=20)
    parser.add_argument('--claim_only', action='store_true', default=False, help='Disables evidence')
    parser.add_argument('--max_seq_length', type=int, help='batch size', default=256)
    parser.add_argument('--knowledgeformat', type=str, help='tag', default='') # wikijoin
    parser.add_argument('--typeprefix', type=str, help='tag', default='')
    parser.add_argument('--outputprefix', type=str, help='tag', default='')

#     parser.add_argument('-append', action='store_true', help='allow append to previous run', default=False)

    args = parser.parse_args()
    if args.preds_file is None:
        args.preds_file = args.input_file.split('/')[-1]

    args.preds_file = args.outputprefix + args.preds_file
#     assert(not os.path.exists(args.preds_file))
    if args.cuda_device>=0:
        device = 'cuda:'+str(args.cuda_device)
    else:
        device = 'cpu'

    args.device = device
    classification_model_dnli = ClassificationModel(num_labels=3,model_name_or_path=args.model)
    classification_model_dnli.model = classification_model_dnli.model.to(device)
    print('model loaded')
    classification_model_dnli.model.eval()
    score_data(args, classification_model_dnli)

# python fever_scoring.py -i ../post_generation/contextagg_maskfill_mix1_wow_test_tsc_200_t1.5.jsonl --output_folder vitamincscores/ -m tals/albert-xlarge-vitaminc
# python fever_scoring.py -i ../post_generation/contextagg_maskfill_mix1_wow_test_tsc_200_t1.5.jsonl --knowledgeformat wikijoin --typeprefix colloq_ --output_folder colloquialfeverscores/ -m colloquial_bert_large

max_stars_repo_name: salesforce/DialFact

max_issues_repo_name: salesforce/DialFact

max_forks_repo_name: salesforce/DialFact

reason: import numpy,from scipy


text:
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

"""Module to compute Trotter errors in the plane-wave dual basis."""
from __future__ import absolute_import
from future.utils import iteritems, itervalues

import numpy
import openfermion.hamiltonians

from openfermion.ops import FermionOperator
from openfermion.utils import count_qubits, normal_ordered
from openfermion.utils._commutators import (
    double_commutator,
    trivially_double_commutes_dual_basis,
    trivially_double_commutes_dual_basis_using_term_info)

def low_depth_second_order_trotter_error_operator(
        terms, indices=None, is_hopping_operator=None, jellium_only=False,
        verbose=False):
    """Determine the difference between the exact generator of unitary
    evolution and the approximate generator given by the second-order
    Trotter-Suzuki expansion.

    Args:
        terms: a list of FermionOperators in the Hamiltonian in the
               order in which they will be simulated.
        indices: a set of indices the terms act on in the same order as terms.
        is_hopping_operator: a list of whether each term is a hopping operator.
        jellium_only: Whether the terms are from the jellium Hamiltonian only,
                      rather than the full dual basis Hamiltonian (i.e. whether
                      c_i = c for all number operators i^ i, or whether they
                      depend on i as is possible in the general case).
        verbose: Whether to print percentage progress.

    Returns:
        The difference between the true and effective generators of time
            evolution for a single Trotter step.

    Notes: follows Equation 9 of Poulin et al.'s work in "The Trotter Step
        Size Required for Accurate Quantum Simulation of Quantum Chemistry",
        applied to the "stagger"-based Trotter step for detailed in
        Kivlichan et al., "Quantum Simulation of Electronic Structure with
        Linear Depth and Connectivity", arxiv:1711.04789.
    """
    more_info = bool(indices)
    n_terms = len(terms)

    if verbose:
        import time
        start = time.time()

    error_operator = FermionOperator.zero()
    for beta in range(n_terms):
        if verbose and beta % (n_terms // 30) == 0:
            print('%4.3f percent done in' % (
                (float(beta) / n_terms) ** 3 * 100), time.time() - start)

        for alpha in range(beta + 1):
            for alpha_prime in range(beta):
                # If we have pre-computed info on indices, use it to determine
                # trivial double commutation.
                if more_info:
                    if (not
                        trivially_double_commutes_dual_basis_using_term_info(
                            indices[alpha], indices[beta],
                            indices[alpha_prime], is_hopping_operator[alpha],
                            is_hopping_operator[beta],
                            is_hopping_operator[alpha_prime], jellium_only)):
                        # Determine the result of the double commutator.
                        double_com = double_commutator(
                            terms[alpha], terms[beta], terms[alpha_prime],
                            indices[beta], indices[alpha_prime],
                            is_hopping_operator[beta],
                            is_hopping_operator[alpha_prime])
                        if alpha == beta:
                            double_com /= 2.0

                        error_operator += double_com

                # If we don't have more info, check for trivial double
                # commutation using the terms directly.
                elif not trivially_double_commutes_dual_basis(
                        terms[alpha], terms[beta], terms[alpha_prime]):
                    double_com = double_commutator(
                        terms[alpha], terms[beta], terms[alpha_prime])

                    if alpha == beta:
                        double_com /= 2.0

                    error_operator += double_com

    error_operator /= 12.0
    return error_operator

def low_depth_second_order_trotter_error_bound(
        terms, indices=None, is_hopping_operator=None,
        jellium_only=False, verbose=False):
    """Numerically upper bound the error in the ground state energy
    for the second-order Trotter-Suzuki expansion.

    Args:
        terms: a list of single-term FermionOperators in the Hamiltonian
            to be simulated.
        indices: a set of indices the terms act on in the same order as terms.
        is_hopping_operator: a list of whether each term is a hopping operator.
        jellium_only: Whether the terms are from the jellium Hamiltonian only,
                      rather than the full dual basis Hamiltonian (i.e. whether
                      c_i = c for all number operators i^ i, or whether they
                      depend on i as is possible in the general case).
        verbose: Whether to print percentage progress.

    Returns:
        A float upper bound on norm of error in the ground state energy.

    Notes:
        Follows Equation 9 of Poulin et al.'s work in "The Trotter Step
        Size Required for Accurate Quantum Simulation of Quantum
        Chemistry" to calculate the error operator, for the "stagger"-based
        Trotter step for detailed in Kivlichan et al., "Quantum Simulation
        of Electronic Structure with Linear Depth and Connectivity",
        arxiv:1711.04789.
    """
    # Return the 1-norm of the error operator (upper bound on error).
    return numpy.sum(numpy.absolute(list(
        low_depth_second_order_trotter_error_operator(
            terms, indices, is_hopping_operator,
            jellium_only, verbose).terms.values())))

def simulation_ordered_grouped_low_depth_terms_with_info(
        hamiltonian, input_ordering=None):
    """Give terms from the dual basis Hamiltonian in simulated order.

    Uses the simulation ordering, grouping terms into hopping
    (i^ j + j^ i) and number (i^j^ i j + c_i i^ i + c_j j^ j) operators.
    Pre-computes term information (indices each operator acts on, as
    well as whether each operator is a hopping operator.

    Args:
        hamiltonian (FermionOperator): The Hamiltonian.
        input_ordering (list): The initial Jordan-Wigner canonical order.

    Returns:
        A 3-tuple of terms from the Hamiltonian in order of simulation,
        the indices they act on, and whether they are hopping operators
        (both also in the same order).

    Notes:
        Follows the "stagger"-based simulation order discussed in Kivlichan
        et al., "Quantum Simulation of Electronic Structure with Linear
        Depth and Connectivity", arxiv:1711.04789; as such, the only
        permitted types of terms are hopping (i^ j + j^ i) and potential
        terms which are products of at most two number operators.
    """
    n_qubits = count_qubits(hamiltonian)
    hamiltonian = normal_ordered(hamiltonian)

    ordered_terms = []
    ordered_indices = []
    ordered_is_hopping_operator = []

    # If no input mode ordering is specified, default to range(n_qubits).
    try:
        input_ordering = list(input_ordering)
    except TypeError:
        input_ordering = list(range(n_qubits))

    # Half a second-order Trotter step reverses the input ordering: this tells
    # us how much we need to include in the ordered list of terms.
    final_ordering = list(reversed(input_ordering))

    # Follow odd-even transposition sort. In alternating steps, swap each even
    # qubits with the odd qubit to its right, and in the next step swap each
    # the odd qubits with the even qubit to its right. Do this until the input
    # ordering has been reversed.
    parity = 0
    while input_ordering != final_ordering:
        results = stagger_with_info(
            hamiltonian, input_ordering, parity)
        terms_in_step, indices_in_step, is_hopping_operator_in_step = results

        ordered_terms.extend(terms_in_step)
        ordered_indices.extend(indices_in_step)
        ordered_is_hopping_operator.extend(is_hopping_operator_in_step)

        # Alternate even and odd steps of the reversal procedure.
        parity = 1 - parity

    return (ordered_terms, ordered_indices, ordered_is_hopping_operator)

def stagger_with_info(hamiltonian, input_ordering, parity):
    """Give terms simulated in a single stagger of a Trotter step.

    Groups terms into hopping (i^ j + j^ i) and number
    (i^j^ i j + c_i i^ i + c_j j^ j) operators.
    Pre-computes term information (indices each operator acts on, as
    well as whether each operator is a hopping operator).

    Args:
        hamiltonian (FermionOperator): The Hamiltonian.
        input_ordering (list): The initial Jordan-Wigner canonical order.
        parity (boolean): Whether to determine the terms from the next even
            (False = 0) or odd (True = 1) stagger.

    Returns:
        A 3-tuple of terms from the Hamiltonian that are simulated in the
        stagger, the indices they act on, and whether they are hopping
        operators (all in the same order).

    Notes:
        The "staggers" used here are the left (parity=False) and right
        (parity=True) staggers detailed in Kivlichan et al., "Quantum
        Simulation of Electronic Structure with Linear Depth and
        Connectivity", arxiv:1711.04789. As such, the Hamiltonian must be
        in the form discussed in that paper. This constrains it to have
        only hopping terms (i^ j + j^ i) and potential terms which are
        products of at most two number operators (n_i or n_i n_j).
    """
    terms_in_step = []
    indices_in_step = []
    is_hopping_operator_in_step = []

    zero = FermionOperator.zero()
    n_qubits = count_qubits(hamiltonian)

    # A single round of odd-even transposition sort.
    for i in range(parity, n_qubits - 1, 2):
        # Always keep the max on the left to avoid having to normal order.
        left = max(input_ordering[i], input_ordering[i + 1])
        right = min(input_ordering[i], input_ordering[i + 1])

        # Calculate the hopping operators in the Hamiltonian.
        left_hopping_operator = FermionOperator(
            ((left, 1), (right, 0)), hamiltonian.terms.get(
                ((left, 1), (right, 0)), 0.0))
        right_hopping_operator = FermionOperator(
            ((right, 1), (left, 0)), hamiltonian.terms.get(
                ((right, 1), (left, 0)), 0.0))

        # Calculate the two-number operator l^ r^ l r in the Hamiltonian.
        two_number_operator = FermionOperator(
            ((left, 1), (right, 1), (left, 0), (right, 0)),
            hamiltonian.terms.get(
                ((left, 1), (right, 1), (left, 0), (right, 0)), 0.0))

        # Calculate the left number operator, left^ left.
        left_number_operator = FermionOperator(
            ((left, 1), (left, 0)), hamiltonian.terms.get(
                ((left, 1), (left, 0)), 0.0))

        # Calculate the right number operator, right^ right.
        right_number_operator = FermionOperator(
            ((right, 1), (right, 0)), hamiltonian.terms.get(
                ((right, 1), (right, 0)), 0.0))

        # Divide single-number terms by n_qubits-1 to avoid over-counting.
        # Each qubit is swapped n_qubits-1 times total.
        left_number_operator /= (n_qubits - 1)
        right_number_operator /= (n_qubits - 1)

        # If the overall hopping operator isn't close to zero, append it.
        # Include the indices it acts on and that it's a hopping operator.
        if not (left_hopping_operator +
                right_hopping_operator) == zero:
            terms_in_step.append(left_hopping_operator +
                                 right_hopping_operator)
            indices_in_step.append(set((left, right)))
            is_hopping_operator_in_step.append(True)

        # If the overall number operator isn't close to zero, append it.
        # Include the indices it acts on and that it's a number operator.
        if not (two_number_operator + left_number_operator +
                right_number_operator) == zero:
            terms_in_step.append(two_number_operator +
                                 left_number_operator +
                                 right_number_operator)
            indices_in_step.append(set((left, right)))
            is_hopping_operator_in_step.append(False)

        # Modify the current Jordan-Wigner canonical ordering in-place.
        input_ordering[i], input_ordering[i + 1] = (input_ordering[i + 1],
                                                    input_ordering[i])

    return terms_in_step, indices_in_step, is_hopping_operator_in_step

def ordered_low_depth_terms_no_info(hamiltonian):
    """Give terms from Hamiltonian in dictionary output order.

    Args:
        hamiltonian (FermionOperator): The Hamiltonian.

    Returns:
        A list of terms from the Hamiltonian in simulated order.

    Notes:
        Assumes the Hamiltonian is in the form discussed in Kivlichan
        et al., "Quantum Simulation of Electronic Structure with Linear
        Depth and Connectivity", arxiv:1711.04789. This constrains the
        Hamiltonian to have only hopping terms (i^ j + j^ i) and potential
        terms which are products of at most two number operators (n_i or
        n_i n_j).
    """
    n_qubits = count_qubits(hamiltonian)
    hamiltonian = normal_ordered(hamiltonian)
    terms = []

    for operators, coefficient in iteritems(hamiltonian.terms):
        terms += [FermionOperator(operators, coefficient)]

    return terms


text:
# !/usr/bin/python
# -*- coding: UTF-8 -*-
"""
Created on JUL 5, 2019

@author: woshihaozhaojun@sina.com
"""
import numpy as np

def mean_absolute_percent_error(y_true, y_pred, sample_weight=None):
    """计算平均百分比绝对误差.

    要保证真实值中没有0!

    Args:
         y_true(array-like) :-  of shape (n_samples), 真实值的列表或者序列
         y_pred(array-like) :-  of shape (n_samples), 预测值的列表或者序列
         sample_weight(array-like) :-  of shape (n_samples), 样例权重的列表或者序列
    Returns:
        float, 预测值相对真实值的平均百分比绝对误差
    """
    y_true = np.array(y_true)
    y_pred = np.array(y_pred)
    y_true.reshape(-1,)
    y_pred.reshape(-1,)

    try:
        assert all(y_true != 0)
    except AssertionError:
        raise ValueError("y_true中不能有0")

    try:
        assert y_true.shape == y_pred.shape
    except AssertionError:
        raise ValueError("y_true和y_pred的维度必需一致")

    if not sample_weight:
        sample_weight = np.array([1 for _ in range(y_true.shape[0])])

    sample_weight = np.array(sample_weight)
    return np.mean(abs((y_pred - y_true) / y_true * sample_weight))

max_stars_repo_name: HAOzj/practices-in-Python

max_issues_repo_name: HAOzj/practices-in-Python

max_forks_repo_name: HAOzj/practices-in-Python


text:
import sys
from os.path import join
import setuptools
import versioneer
from setuptools.extension import Extension
from Cython.Build import cythonize
import numpy

COMMON_INCLUDE_DIRS = [
    './geoist/magmod',
    './geoist/magmod/include',
    join(sys.prefix, 'include'),
]
COMMON_INCLUDE_DIRS.append(numpy.get_include())

with open("README.md", "r") as fh:
    long_description = fh.read()

extensions = [
    Extension("geoist.pfm._prism",
              ["geoist/pfm/_prism.pyx"],
              include_dirs=[numpy.get_include()]
    ),
    Extension("geoist.inversion._ttime2d",
              ["geoist/inversion/_ttime2d.pyx"],
              include_dirs=[numpy.get_include()]
    ),
    Extension("geoist.inversion._wavefd",
              ["geoist/inversion/_wavefd.pyx"],
              include_dirs=[numpy.get_include()]
    ),
    Extension(
            'geoist.magmod._pymm',
            sources=[
                'geoist/magmod/pymm.c',
            ],
            libraries=[],
            library_dirs=[],
            include_dirs=COMMON_INCLUDE_DIRS,
        ),
    Extension(
            'geoist.magmod._pysunpos',
            sources=[
                'geoist/magmod/pysunpos.c',
            ],
            libraries=[],
            library_dirs=[],
            include_dirs=COMMON_INCLUDE_DIRS,
        ),
    Extension(
            'geoist.magmod._pytimeconv',
            sources=[
                'geoist/magmod/pytimeconv.c',
            ],
            libraries=[],
            library_dirs=[],
            include_dirs=COMMON_INCLUDE_DIRS,
        ),
]

exts = cythonize(extensions)

install_requires = ["numpy","matplotlib","scipy","h5py","numba","pandas",
                    "pytest","future","Cython",'statsmodels>=0.9.0',"PyWavelets",
                    "seaborn","patsy", "appdirs"]
setuptools.setup(
    name="geoist",
    version=versioneer.get_version(),
    cmdclass=versioneer.get_cmdclass(),
    author="IGP-GRAVITY",
    author_email="gravity_igp@sina.com",
    description="An Open-Source Geophysical Python Library for Geoscience Prototype Research",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/igp-gravity/geoist",
    packages=setuptools.find_packages(),
    ext_modules=exts,
    include_package_data=True,
    install_requires=install_requires,
    classifiers=(
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent",
    ),
)

max_stars_repo_name: irxat/geoist

max_issues_repo_name: irxat/geoist

max_forks_repo_name: irxat/geoist


text:
"""
grid world mdp class
"""
import numpy as np

class GridMDP:
    """
    Grid world MDP
    """

    def __init__(self):
        self.x_dim = 10
        self.y_dim = 10
        self.num_states = self.x_dim * self.y_dim
        self.start_state = np.asarray([0, 0])
        self._state = self.start_state
        self.action_space = [0, 1, 2, 3]
        self.target_state = [np.asarray([9, 9])]
        self.time_limit = 15
        self.t = 0
        self._done = False
        self._rewards = np.zeros(shape=(self.x_dim, self.y_dim), dtype=np.float)
        for t in self.target_state:
            self._rewards[t[0], t[1]] = 1.
        self.dims = 2
        self.min_state = np.asarray([0, 0])
        self.max_state = np.asarray([9, 9])

    def target_distribution(self, weight=3.) -> np.ndarray:
        """
        Return the target distribution
        :return:
        """
        # dist = np.exp(self._rewards * weight)
        dist = self._rewards * weight
        dist /= np.sum(dist)
        return dist

    def reset(self) -> np.ndarray:
        """
        reset state to 0
        :return:
        """
        self._state = np.copy(self.start_state)
        self.t = 0
        self.done()
        return self._state

    def step(self, act):
        """
        take action and return next state
        :param act:
        :return:
        """
        act = int(act)
        if self.t >= self.time_limit:
            print("Time limit exceeded. Reset for new episode")
            raise Exception()

        if self._done:
            print("Episode ended. Reset for new episode")
            raise Exception()

        assert act in self.action_space

        if np.random.random() < 0.:
            return self._state, self.reward(), self._done

        if act < 2:
            self._state[0] += act * 2 - 1
            self._state[0] = np.clip(self._state[0], 0, self.x_dim - 1)
        else:
            act = (act - 2) * 2 - 1
            self._state[1] += act
            self._state[1] = np.clip(self._state[1], 0, self.y_dim - 1)
        self.t += 1
        self.done()
        return self._state, self.reward(), self._done, {}

    def done(self):
        """
        check if episode over
        :return:
        """
        if self.t >= self.time_limit:
            self._done = True
        elif any(np.array_equal(self._state, x) for x in self.target_state):
            self._done = True
        else:
            self._done = False
        return

    def reward(self):
        """
        Reward function
        :return:
        """
        return self._rewards[self._state[0], self._state[1]]
        # if self._state == self.target_state:
        #     return 1.
        # return 0.

class MazeWorld(GridMDP):
    """
    Maze World is the same grid world as above
    But it has multiple goals, one of which is unreachable
    """

    def __init__(self):
        super(MazeWorld, self).__init__()
        self.start_state = np.asarray([1, 1])
        self._state = self.start_state
        self.target_state = [np.asarray([1, 4])]  # , np.asarray([8, 1]), np.asarray([6, 6]), np.asarray([4, 9])]
        self.time_limit = 100
        self._rewards = np.zeros(shape=(self.x_dim, self.y_dim), dtype=np.float)
        for c, t in enumerate(self.target_state):
            self._rewards[t[0], t[1]] = 1. / (c + 1.)

    def sample_transitions(self, num=50):
        """
        sample valid transitions from the mdp
        """
        transitions = []
        temp_state = self._state
        temp_time = self.t
        temp_done = self._done
        self._done = False
        self.t = 0
        while len(transitions) < num:
            state = np.random.randint(self.min_state, self.max_state)
            act = np.random.choice(self.action_space)
            self._state = np.copy(state)
            next_state, _, _, _ = self.step(act)
            if not np.equal(state, next_state).all():
                transitions.append([state, next_state])
            self.t = 0
            self._done = False
        self._state = temp_state
        self._done = temp_done
        self.t = temp_time
        states, next_states = (np.asarray(x) for x in zip(*transitions))
        return states, next_states

    def step(self, act):
        """
        take action and return next state
        Maze is as follows:
        -  -  -  -  -  -  -  -  -  -

        -  -  -  -  -  -  -  -  -  -
                  |
        -  -  -  -| -  -  -  -  -  -
                  |
        -  -  -  -| -  -  -  -  -  -
                  |
        -  -  -  -| -  -  -  -  -  -
                  |
        -  -  -  -| -  -  -  -  -  -
                  |
        -  1  -  -| -  -  -  -  -  -
        __________|
        -  -  -  -  -  -  -  -  -  -

        -  A  -  -  -  -  -  -  -  -

        -  -  -  -  -  -  -  -  -  -

        :param act:
        :return:
        """
        act = int(act)
        if self.t >= self.time_limit:
            print("Time limit exceeded. Reset for new episode")
            raise Exception()

        if self._done:
            print("Episode ended. Reset for new episode")
            raise Exception()

        assert act in self.action_space

        if np.random.random() < 0.:
            return self._state, self.reward(), self._done

        if act == 0:
            if not (self._state[0] == 4 and (2 < self._state[1] < 8)):
                self._state[0] -= 1
            self._state[0] = np.clip(self._state[0], 0, self.x_dim - 1)
        elif act == 1:
            if not (self._state[0] == 3 and (2 < self._state[1] < 8)):
                self._state[0] += 1
            self._state[0] = np.clip(self._state[0], 0, self.x_dim - 1)
        elif act == 2:
            if not (self._state[1] == 3 and self._state[0] < 4):
                self._state[1] -= 1
            self._state[1] = np.clip(self._state[1], 0, self.y_dim - 1)
        else:
            if not (self._state[1] == 2 and self._state[0] < 4):
                self._state[1] += 1
            self._state[1] = np.clip(self._state[1], 0, self.y_dim - 1)
        self.t += 1
        self.done()
        return self._state, self.reward(), self._done, np.copy(self.target_state[0])

class ToroidWorld(GridMDP):
    """
    Toroidal grid world is same as grid world
    Except the grid bends in on itself
    so walking off one side causes transitions to the other side
    """

    def __init__(self):
        super(ToroidWorld, self).__init__()
        self.start_state = np.asarray([2, 2])
        self._state = self.start_state
        self.target_state = [np.asarray(
            [self.x_dim - 3, self.y_dim - 3])]  # , np.asarray([8, 1]), np.asarray([6, 6]), np.asarray([4, 9])]
        self.time_limit = 40
        self._rewards = np.zeros(shape=(self.x_dim, self.y_dim), dtype=np.float)
        for c, t in enumerate(self.target_state):
            self._rewards[t[0], t[1]] = 1. / (c + 1.)

    def step(self, act):
        """
        take action and return next state
        :param act:
        :return:
        """
        act = int(act)
        if self.t >= self.time_limit:
            print("Time limit exceeded. Reset for new episode")
            raise Exception()

        if self._done:
            print("Episode ended. Reset for new episode")
            raise Exception()

        assert act in self.action_space

        if np.random.random() < 0.:
            return self._state, self.reward(), self._done

        if act < 2:
            self._state[0] += act * 2 - 1
            self._state[0] %= self.x_dim
        else:
            act = (act - 2) * 2 - 1
            self._state[1] += act
            self._state[1] %= self.y_dim
        self.t += 1
        self.done()
        return self._state, self.reward(), self._done, {}

class WindyMazeWorld(MazeWorld):
    """
    Windy maze world is same as Maze world
    Except in part of the world there is a wind
    that causes stochastic transitions for certain actions
    """

    def __init__(self):
        super(WindyMazeWorld, self).__init__()

    def step(self, act):
        """
        take action and return next state
        Maze is as follows:
        -  -  -  -  -  -  -  -  -  -

        -  -  -  -  -  -  -  -  -  -
                  |
        -  -  -  -| -  -  -  -  -  -
                  |
        -  -  -  -| -  -  1  -  -  -
                  |
        -  -  -  -| -  -  -  -  -  -
                  |
        -  -  -  -| -  -  -  -  -  -
                  |
        -  1  -  -| -  -  -  -  -  -
        __________|
        -  -  -  -  -  -  -  -  -  -

        -  A  -  -  -  -  -  -  1  -

        -  -  -  -  -  -  -  -  -  -

        :param act:
        :return:
        """
        act = int(act)
        if self.t >= self.time_limit:
            print("Time limit exceeded. Reset for new episode")
            raise Exception()

        if self._done:
            print("Episode ended. Reset for new episode")
            raise Exception()

        assert act in self.action_space

        if np.random.random() < 0.:
            return self._state, self.reward(), self._done

        if act == 0:
            if not (self._state[0] == 4 and (2 < self._state[1] < 8)):
                self._state[0] -= 1
            # Move down due to wind
            if 4 <= self._state[0] and np.random.rand() < 0.4:
                self._state[1] = np.minimum(self._state[1] - 1, 0)
            self._state[0] = np.clip(self._state[0], 0, self.x_dim - 1)
        elif act == 1:
            if not (self._state[0] == 3 and (2 < self._state[1] < 8)):
                self._state[0] += 1
            # Move down due to wind
            if 4 <= self._state[0] < self.x_dim and np.random.rand() < 0.4:
                self._state[1] = np.minimum(self._state[1] - 1, 0)
            self._state[0] = np.clip(self._state[0], 0, self.x_dim - 1)
        elif act == 2:
            if not (self._state[1] == 3 and self._state[0] < 4):
                self._state[1] -= 1
            self._state[1] = np.clip(self._state[1], 0, self.y_dim - 1)
        else:
            if not (self._state[1] == 2 and self._state[0] < 4):
                self._state[1] += 1
            # Move down due to wind
            if 4 <= self._state[0] < self.x_dim and np.random.rand() < 0.4:
                self._state[1] -= 1
            self._state[1] = np.clip(self._state[1], 0, self.y_dim - 1)
        self.t += 1
        self.done()
        return self._state, self.reward(), self._done, np.copy(self.target_state[0])


text:
import os.path, os, sys
import argparse
sys.path.append('.')
sys.path.append('..')
sys.path.append('../../')
sys.path.append('../../../')
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Configuration of setup and training process")
    parser.add_argument('-AUs', '--AUs_dir', type=str, required=True,
                        help='Path with the AUs')
    parser.add_argument('-out', '--out_dir', type=str,
                        help='Path to save the plots of the AUs',
                        default='./')
    args = parser.parse_args()

    plt.rcParams['figure.figsize'] = (10, 5)
    emot_dict = {1:"Neutral",
                 2:"Calm",
                 3:"happy",
                 4:"sad",
                 5:"angry",
                 6:"fear",
                 7:"disgust",
                 8:"surprise"}
    plot_presence = True

    for au_file in os.listdir(args.AUs_dir):
        splitted_video_name = au_file.split("-")
        emotion = int(splitted_video_name[2]) #- 1  # See dict
        actor = splitted_video_name[-1].split(".")[0]
        emotional_intensity = int(splitted_video_name[3])  # 01 = normal, 02 = strong
        path_file = os.path.join(args.AUs_dir, au_file)
        df_AUs = pd.read_csv(path_file, sep=";", header=0)
        presenceAU_df = df_AUs[df_AUs.columns[17::]]
        intensity_AU_df = df_AUs[df_AUs.columns[0:17]]

        #Where are the most intense regions -> clear AUs
        x = range(0, len(intensity_AU_df))
        fig, ax = plt.subplots()  # figsize=(13,10)
        if(plot_presence):
            extra_name = "_presence"
            presenceAU_df.plot(colormap='tab20', grid=True)
            plt.title(au_file+" "+emot_dict[emotion]+" - "+str(emotional_intensity))
            plt.yticks(np.arange(0, 1.5, 0.5))
        else:
            extra_name = "_intensity"
            intensity_AU_df.plot(colormap='tab20',  grid=True)
            plt.title(au_file + " " + emot_dict[emotion] + " - " + str(emotional_intensity))
            plt.yticks(np.arange(0, 5.5, 0.5))
        #plt.show()

        os.makedirs(os.path.join(args.out_dir, "EMOTIONS",emot_dict[emotion]), exist_ok=True)
        os.makedirs(os.path.join(args.out_dir,"ACTOR", actor), exist_ok=True)

        plt.savefig(os.path.join(args.out_dir, "EMOTIONS",emot_dict[emotion], au_file.replace(".csv", extra_name+".png")))
        plt.savefig(os.path.join(args.out_dir,"ACTOR", actor,au_file.replace(".csv", extra_name+".png")))

        plt.close('all')


text:
import sys
import os
import time
import glob
import numpy as np
from model_updater import has_best, lock, unlock
from train_controller import get_train_data_files

from lib.lib import VertexCoverLib

if __name__ == '__main__':
    api = VertexCoverLib(sys.argv)

    opt = {}
    for i in range(1, len(sys.argv), 2):
        opt[sys.argv[i][1:]] = sys.argv[i + 1]

    save_dir = opt['save_dir']
    if has_best(opt):
        while lock(opt, 'best'):
            pass
        api.LoadModel('best')
        unlock(opt, 'best')
    else:
        print('[learner] best model not found', flush=True)

    for _ in range(int(opt['save_interval'])):
        api.ClearTrainData()

        train_data_files = []
        while 1:
            while lock(opt, 'data_all'):
                pass
            train_data_files = get_train_data_files(opt)
            if len(train_data_files) < int(opt['batch_num']):
                unlock(opt, 'data_all')
                print('[learner] training data not found', flush=True)
                time.sleep(np.random.randint(30, 120))
            else:
                break

        batch_num = int(opt['batch_num'])
        train_data_files = np.random.choice(train_data_files, batch_num)

        for train_data_file in train_data_files:
            api.AddTrainData(train_data_file)
        unlock(opt, 'data_all')

        api.Train()

    filename = 'model{}'.format(time.time())
    api.SaveModel(filename)
    os.system('touch {}/timestamp_{}'.format(save_dir, filename))
    print('[learner] generated new model', flush=True)

max_stars_repo_name: hmtbgc/combopt-zero

max_issues_repo_name: hmtbgc/combopt-zero

max_forks_repo_name: hmtbgc/combopt-zero


text:
# Author: Yin Cheng Ng

import numpy as np, tensorflow as tf
from scipy import sparse
import GPflow
from GPflow.param import DataHolder, Param
from GPflow import transforms
from ggp.utils import sparse_to_tuple
from ggp.param import SparseDataHolder

float_type = GPflow.settings.dtypes.float_type
np_float_type = np.float32 if float_type == tf.float32 else np.float64

class SparseGraphPolynomial(GPflow.kernels.Kern):
    def __init__(self, denseAdjMat, denseFeatureMat, X_tr, degree=3.0, variance=1.0, offset=1.0):
        GPflow.kernels.Kern.__init__(self, 1)
        self.degree = degree
        self.offset = Param(offset, transform=transforms.positive)
        self.variance = Param(variance, transforms.positive)
        denseAdjMat[np.diag_indices(len(denseAdjMat))] = 1.
        self.tr_features, self.tr_masks, self.tr_masks_counts = self._diag_tr_helper(denseFeatureMat, denseAdjMat, X_tr)
        self.sparse_P = SparseDataHolder(sparse_to_tuple(sparse.csr_matrix(
                        denseAdjMat/np.sum(denseAdjMat, 1, keepdims=True))))
        self.sparseFeatureMat = SparseDataHolder(sparse_to_tuple(sparse.csr_matrix(denseFeatureMat)))
        self.denseFeatureMat = DataHolder(denseFeatureMat)

    def K(self, X, X2=None):
        X = tf.reshape(tf.cast(X, tf.int32), [-1])
        X2 = tf.reshape(tf.cast(X2, tf.int32), [-1]) if X2 is not None else X
        base_K_mat = (self.variance * tf.matmul(self.denseFeatureMat, self.denseFeatureMat, transpose_b = True) + self.offset) ** self.degree
        t1 = tf.sparse_tensor_dense_matmul(self.sparse_P, base_K_mat)
        t2 = tf.sparse_tensor_dense_matmul(self.sparse_P, t1, adjoint_b=True)
        return tf.gather(tf.gather(t2, X), X2, axis=1)

    def Kdiag(self, X):
        return tf.diag_part(self.K(X))

    def Kzx(self, Z, X):
        X_reshaped = tf.reshape(tf.cast(X, tf.int32), [-1])
        Kmat = (self.variance * tf.sparse_tensor_dense_matmul(self.sparseFeatureMat, Z, adjoint_b=True) + self.offset)**self.degree
        t1 = tf.sparse_tensor_dense_matmul(self.sparse_P, Kmat)
        return tf.gather(tf.transpose(t1), X_reshaped, axis=1)

    def Kzz(self, Z):
        return (self.variance * (tf.matmul(Z, Z, transpose_b=True)) + self.offset) ** self.degree

    @GPflow.param.AutoFlow((float_type, [None, None]), (float_type, [None, None]))
    def compute_Kzx(self, Z, X):
        return self.Kzx(Z)

    def Kdiag_tr(self):
        base_K_mat = self.tr_masks * (self.variance * tf.matmul(self.tr_features, self.tr_features,
                                                                 transpose_b=True) + self.offset)**self.degree
        return tf.reduce_sum(base_K_mat, [1, 2])/self.tr_masks_counts

    def _diag_tr_helper(self, node_features, adj_mat, x_tr):
        z = np.asarray([node_features[a == 1.] for a in adj_mat[x_tr.flatten()]])
        max_n = np.max([t.shape[0] for t in z])
        out = np.zeros((len(z), max_n, node_features.shape[1]))
        masks = np.zeros((len(z), max_n, max_n), dtype=np_float_type)
        for i in range(len(z)):
            out[i,:len(z[i]),:] = z[i]
            masks[i, :len(z[i]), :len(z[i])] = 1
        return DataHolder(out), DataHolder(masks), DataHolder(np.sum(np.sum(masks, 2), 1))

max_stars_repo_name: yincheng/GGP

max_issues_repo_name: yincheng/GGP

max_forks_repo_name: yincheng/GGP


text:
import argparse
import numpy as np

from meanpress import *

# Using argparse is a good thing :)
ap = argparse.ArgumentParser(description='Compress and decompress images using meanpress')
ap.add_argument("-c",action='store_true',help='Compress mode')
ap.add_argument("-x",action='store_true',help='Decompress mode')
ap.add_argument("-d",action='store_true',help='Save matrix displays instead')
ap.add_argument("input",help='Input file')
ap.add_argument("output",help='Output file')
args = vars(ap.parse_args())

prediction = False

if not (args['c'] or args['x']):
    ap.error('No action requested, add -c or -x')
if not args['c'] and args['d']:
    ap.error('Cannot display matrixes unless -c')

image = load_image(args['input'])
target_shape = image.shape

# Simulate compression
if prediction:
    imagei32 = np.array(image,dtype=np.int32)
    prediction = (imagei32[:-1,:-1]+imagei32[1:,:-1]+imagei32[:-1,1:]+1)//3
    error = np.abs(imagei32[1:,1:]-prediction)
    byteseq = compress_image(error,verbose=2,display=args['d'],
        display_base_fname=args['output'])
else:
    byteseq = compress_image(image,verbose=2,display=args['d'],
        display_base_fname=args['output'])

print("Decompressing to assert that the reconstruction is right.")
image2 = decompress_image(byteseq)
assert(np.all(image==image2))
# Compute radio
rad = 4*byteseq.size/(target_shape[0]*target_shape[1]*target_shape[2])

print("data : %d"%(4*byteseq.size))
print("radio: %f"%rad)

max_stars_repo_name: autopawn/meanpress

max_issues_repo_name: autopawn/meanpress

max_forks_repo_name: autopawn/meanpress


text:
"""
Manage the folder for architecture search
"""
import operator
from functools import reduce
import os
import subprocess
import json
import pickle
import numpy as np
from data_providers.utils import get_data_provider_by_name
from expdir_monitor.expdir_monitor import ExpdirMonitor
from expdir_monitor import distributed

class NetPool:
	def __init__(self, path):
		self.path = os.path.realpath(path)
		os.makedirs(self.path, exist_ok=True)

		self.net_str2id = {}
		self.net_id2val = {}
		self.net_id2params = {}
		self.running_set = {'stone': 0}

		self.on_load()

	@property
	def str2id_path(self):
		return '%s/net.str2id' % self.path

	@property
	def id2val_path(self):
		return '%s/net.id2val' % self.path

	@property
	def id2params_path(self):
		return '%s/net.id2params' % self.path

	def on_load(self):
		if os.path.isfile(self.str2id_path):
			self.net_str2id = json.load(open(self.str2id_path, 'r'))
		if os.path.isfile(self.id2val_path):
			net_id2val = json.load(open(self.id2val_path, 'r'))
			for key in net_id2val:
				self.net_id2val[int(key)] = net_id2val[key]
		to_rename = []
		for folder in os.listdir(self.path):
			if folder.startswith('#'):
				out_file = '%s/%s/output' % (self.path, folder)
				if not os.path.isfile(out_file):
					subprocess.run(['rm', '-rf', os.path.join(self.path, folder)])
				else:
					net_str = json.load(open('%s/%s/net.str' % (self.path, folder), 'r'))['net_str']
					if self.net_str2id.get(net_str) is None:
						record = json.load(open(out_file, 'r'))
						net_val = float(record['valid_acc'])
						net_id = self.add_net(net_str, net_val)
						folder_path = self.get_net_folder(net_id)
					else:
						net_id = self.net_str2id[net_str]
						folder_path = self.get_net_folder(net_id)
					if folder_path != folder:
						to_rename.append([folder, folder_path])
		for src_folder, dst_folder in to_rename:
			src_folder = os.path.join(self.path, src_folder)
			dst_folder = os.path.join(self.path, dst_folder)
			os.rename(src_folder, dst_folder)

	def add_net(self, net_str, net_val):
		assert self.net_str2id.get(net_str) is None, '%s exists' % net_str
		net_id = net_str.__hash__()
		while net_id in self.net_id2val:
			net_id += 1
		self.net_str2id[net_str] = net_id
		self.net_id2val[net_id] = net_val
		return net_id

	def get_net_val(self, net_str):
		net_id = self.net_str2id.get(net_str)
		if net_id is None:
			if net_str in self.running_set:
				running_id = self.running_set[net_str]
			else:
				running_id = net_str.__hash__()
				while running_id in self.running_set.values():
					running_id += 1
				self.running_set[net_str] = running_id
			net_folder = '%s/#Running_%s' % (self.path, running_id)
			return None, net_folder
		else:
			net_val = self.net_id2val[net_id]
			net_folder = '%s/%s' % (self.path, self.get_net_folder(net_id))
			return net_val, net_folder

	def on_running_finished(self, net_str, net_folder, net_val):
		net_id = self.add_net(net_str, net_val)
		# folder_path = self.get_net_folder(net_id)
		# self.running_set.pop(net_str)
		# os.rename(net_folder, os.path.join(self.path, folder_path))

	def update_params(self, net_str_list, params_list):
		for net_str, params in zip(net_str_list, params_list):
			self.net_id2params[self.net_str2id[net_str]] = params

	def save(self):
		json.dump(self.net_str2id, open(self.str2id_path, 'w'), indent=4)
		json.dump(self.net_id2val, open(self.id2val_path, 'w'), indent=4)
		json.dump(self.net_id2params, open(self.id2params_path, 'w'), indent=4)

	@staticmethod
	def get_net_folder(net_id):
		return '#%s' % net_id

class ArchManager:
	def __init__(self, start_net_path, arch_path, net_pool_path):
		self.start_net_monitor = ExpdirMonitor(start_net_path)
		self.start_net_config, self.data_provider = None, None

		self.net_pool = NetPool(net_pool_path)

		self.arch_path = os.path.realpath(arch_path)
		os.makedirs(self.arch_path, exist_ok=True)

		self.episode = 0
		self.net_val_wrt_episode = []
		self.net_params_wrt_episode = []

		self.val_log_writer = open(self.val_logs_path, 'a')
		self.params_logs_writer = open(self.params_logs_path, 'a')
		self.net_log_writer = open(self.net_logs_path, 'a')
		self.on_load()

	@property
	def meta_controller_path(self):
		return '%s/controller' % self.arch_path

	@property
	def val_logs_path(self):
		return '%s/val.log' % self.arch_path

	@property
	def params_logs_path(self):
		return '%s/params.log' % self.arch_path

	@property
	def net_logs_path(self):
		return '%s/net.log' % self.arch_path

	def on_load(self):
		if os.path.isfile(self.val_logs_path):
			with open(self.val_logs_path, 'r') as fin:
				for line in fin.readlines():
					line = line[:-1]
					self.episode += 1
					net_val_list = line.split('\t')[4:]
					net_val_list = [float(net_val) for net_val in net_val_list]
					self.net_val_wrt_episode.append(net_val_list)
		if os.path.isfile(self.params_logs_path):
			with open(self.params_logs_path, 'r') as fin:
				for line in fin.readlines():
					line = line[:-1]
					params_list = line.split('\t')[2:]
					params_list = [float(params) for params in params_list]
					self.net_params_wrt_episode.append(params_list)

	def update_start_net(self, net_str):
		net_id = self.net_pool.net_str2id[net_str]
		net_folder1 = '%s/#Running_%s' % (self.net_pool.path, net_id)
		net_folder2 = '%s/#%s' % (self.net_pool.path, net_id)
		if os.path.isdir(net_folder1):
			self.start_net_monitor = ExpdirMonitor(net_folder1)
		elif os.path.isdir(net_folder2):
			self.start_net_monitor = ExpdirMonitor(net_folder2)
		self.start_net_config = None
		self.get_start_net()

	def get_start_net(self, copy=False):
		if self.start_net_config is None:
			# prepare start net
			print('Load start net from %s' % self.start_net_monitor.expdir)
			init = self.start_net_monitor.load_init()
			dataset = 'C10+' if init is None else init.get('dataset', 'C10+')
			run_config = self.start_net_monitor.load_run_config(print_info=True, dataset=dataset)
			run_config.renew_logs = False

			net_config, model_name = self.start_net_monitor.load_net_config(init, print_info=True)
			self.data_provider = get_data_provider_by_name(run_config.dataset, run_config.get_config())
			self.start_net_config = [net_config, run_config, model_name]
		if copy:
			net_config, run_config, model_name = self.start_net_config[:3]
			return [
				net_config.copy(), run_config.copy(), model_name
			]
		else:
			return self.start_net_config

	@staticmethod
	def prepare_folder_for_valid(net_str, net_config, run_config, exp_dir):
		os.makedirs(exp_dir, exist_ok=True)
		monitor = ExpdirMonitor(exp_dir)
		json.dump(net_config.get_config(), open(monitor.net_config_path, 'w'), indent=4)
		json.dump(run_config.get_config(), open(monitor.run_config_path, 'w'), indent=4)
		pickle.dump(net_config.renew_init(None), open(monitor.init, 'wb'))
		json.dump({'net_str': net_str}, open(os.path.join(monitor.expdir, 'net.str'), 'w'), indent=4)

	def get_net_vals(self, net_str_list, net_configs, run_configs):
		net_val_list = [-1] * len(net_str_list)

		to_run = {}
		for _i, net_str in enumerate(net_str_list):
			net_val, net_folder = self.net_pool.get_net_val(net_str)
			if net_val is None:
				if net_folder in to_run: to_run[net_folder] += [_i]
				else:
					to_run[net_folder] = [_i]
					self.prepare_folder_for_valid(net_str, net_configs[_i], run_configs[_i], net_folder)
			else:
				net_val_list[_i] = net_val

		task_list = [[net_folder, to_run[net_folder]] for net_folder in to_run]
		distributed.run(task_list)
		episode_total_running_time = 0
		for net_folder, idx, net_val in task_list:
			net_str = net_str_list[idx[0]]
			net_val, running_time = net_val
			episode_total_running_time += running_time
			self.net_pool.on_running_finished(net_str, net_folder, net_val)
			for _id in idx:
				net_val_list[_id] = net_val
		params_list = self.get_net_params(net_str_list)
		self.net_pool.update_params(net_str_list, params_list)
		self.log_nets(net_str_list, episode_total_running_time)
		self.net_pool.save()
		return net_val_list

	def get_net_params(self, net_str_list):
		param_list = []
		for net_str in net_str_list:
			net_id = self.net_pool.net_str2id.get(net_str)
			net_folder1 = '%s/#Running_%s' % (self.net_pool.path, net_id)
			net_folder2 = '%s/#%s' % (self.net_pool.path, net_id)
			if os.path.isdir(net_folder1):
				em = ExpdirMonitor(net_folder1)
			elif os.path.isdir(net_folder2):
				em = ExpdirMonitor(net_folder2)
			init = em.load_init()
			total_params = 0
			for layer in init['layer_cascade']['layers']:
				if layer is None:
					continue
				for k, v in layer.items():
					if 'moving_' not in k:
						total_params += reduce(operator.mul, v.shape)
			param_list.append(total_params)
		return param_list

	def val2reward(self, net_val_list, func=None, params_list=None):
		rewards = []
		for i, net_val in enumerate(net_val_list):
			if params_list is not None:
				rewards.append(np.tan(net_val * np.pi / 2) - params_list[i]/5e4)
			elif func is None:
				rewards.append(net_val)
			elif func == 'tan':
				reward = np.tan(net_val * np.pi / 2)
				rewards.append(reward)
			else:
				raise NotImplementedError
		return rewards

	def reward(self, net_val_list, net_str_list, reward_config):
		use_params = reward_config.get('use_params', False)
		if use_params:
			net_id_list = [self.net_pool.net_str2id[net_str] for net_str in net_str_list]
			params_list = [self.net_pool.net_id2params[net_id] for net_id in net_id_list]
		else:
			params_list = None
		rewards = self.val2reward(net_val_list, reward_config.get('func'), params_list=params_list)
		rewards = np.array(rewards)
		# baseline function
		decay = reward_config['decay']
		if 'exp_moving_avg' not in self.__dict__:
			self.exp_moving_avg = 0
			for old_net_val_list, old_params_list in zip(self.net_val_wrt_episode[:-1], self.net_params_wrt_episode[:-1]):
				params_list = old_params_list if use_params else None
				old_rewards = self.val2reward(old_net_val_list, reward_config.get('func'), params_list=old_params_list)
				self.exp_moving_avg += decay * (np.mean(old_rewards) - self.exp_moving_avg)
		self.exp_moving_avg += decay * (np.mean(rewards) - self.exp_moving_avg)
		return rewards - self.exp_moving_avg

	def log_nets(self, net_str_list, running_time, print_info=True):
		net_id_list = [self.net_pool.net_str2id[net_str] for net_str in net_str_list]
		nets_num = len(net_id_list)
		new_nets_num = len(set(net_id_list))

		net_val_list = [self.net_pool.net_id2val[net_id] for net_id in net_id_list]
		mean_val, max_val = np.mean(net_val_list), np.max(net_val_list)
		self.net_log_writer.write('%d.\t nets=%d (total=%d)\t%s\n' % (self.episode, new_nets_num, nets_num,
														'\t'.join([str(net_id) for net_id in net_id_list])))

		log_str = '%d.\t nets=%d (total=%d)\t mean_val=%s (max_val=%s)\t using %s(min)\t%s' % \
				  (self.episode + 1, new_nets_num, nets_num, mean_val, max_val, running_time,
				   '\t'.join([str(net_val) for net_val in net_val_list]))
		if print_info:
			print(log_str)
		self.val_log_writer.write(log_str + '\n')

		params_list = [self.net_pool.net_id2params[net_id] for net_id in net_id_list]
		self.params_logs_writer.write('%d.\t nets=%d (total=%d)\t%s\n' % (self.episode, new_nets_num, nets_num,
														'\t'.join([str(params) for params in params_list])))

		self.val_log_writer.flush()
		self.net_log_writer.flush()
		self.params_logs_writer.flush()
		self.net_val_wrt_episode.append(net_val_list)
		self.net_params_wrt_episode.append(params_list)
		self.episode += 1

max_stars_repo_name: tchittesh/EAS-

max_issues_repo_name: tchittesh/EAS-

max_forks_repo_name: tchittesh/EAS-