import json
import os
import numpy as np
import tensorflow as tf

from tf_agents.trajectories import policy_step
from tf_agents.trajectories import time_step as ts
from tf_agents.trajectories import trajectory as tf_trajectory

from google.protobuf import text_format

import common.generate_protos  # pylint: disable=unused-import

import action_pb2
import brain_pb2
import episode_pb2
import observation_pb2
import data_store_pb2

from learner.brains import tfa_specs

PROJECT_ID = 'project1'
BRAIN_ID = 'brain1'
SESSION_ID = 'session1'
EPISODE_ID = 'episode1'
ASSIGNMENT_ID = (
    # Choose settings that maximize speed and reduce non-determinism.
    '{"batch_size": 500, "training_examples": 500, "save_interval_batches": 5,'
    ' "synchronous_export": true, "use_xla_jit": false, '
    ' "num_batches_to_sample": 5}')

def populate_data_store(data_store, assignment_id=None, episode_ids=None,
                        steps_per_episode_chunk=None):
  """Populate a datastore with an assignment.

  Args:
    data_store: data_store.DataStore instance to populate.
    assignment_id: Assignment ID to create or ASSIGNMENT_ID if this is None.
    episode_ids: IDs of episodes to create or a single episode with
      EPISODE_ID if this is None.
    steps_per_episode_chunk: List of number of steps for each episode chunk.
      For example, [2, 4] would create two chunks with 2 and 4 steps
      respectively. If this is None, this argument defaults to [2, 2].

  Returns:
    (assignment_resource_id, episode_chunk_resource_ids) where
    assignment_resource_id is resource ID of the written assignment and
    episode_chunk_resource_ids is a list of episode chunk resource IDs written
    to the data store.
  """
  assignment_proto = assignment(assignment_id=assignment_id)
  assignment_resource_id = data_store.to_resource_id(assignment_proto)
  protos = [
      project(),
      data_store_brain(),
      session(),
      assignment_proto,
  ]
  if not steps_per_episode_chunk:
    steps_per_episode_chunk = [2, 2]
  if episode_ids is None:
    episode_ids = [EPISODE_ID]
  episode_chunk_resource_ids = []
  if episode_ids:
    for episode_id in episode_ids:
      for chunk in episode_chunks(steps_per_episode_chunk, episode_id):
        episode_chunk_resource_ids.append(data_store.to_resource_id(chunk))
        protos.append(chunk)
  for proto in protos:
    data_store.write(proto)
  return assignment_resource_id, episode_chunk_resource_ids

def project():
  """Return a test data_store_pb2.Project."""
  return data_store_pb2.Project(project_id=PROJECT_ID)

def data_store_brain():
  """Return a test data_store_pb2.Brain."""
  return data_store_pb2.Brain(
      project_id=PROJECT_ID,
      brain_id=BRAIN_ID,
      brain_spec=brain_spec())

def session():
  """Return a test data_store_pb2.Session."""
  return data_store_pb2.Session(
      project_id=PROJECT_ID,
      brain_id=BRAIN_ID,
      session_id=SESSION_ID)

def create_assignment_id(override_dict):
  """Return the default test assignment id with hparam overrides."""
  assignment_dict = json.loads(ASSIGNMENT_ID)
  assignment_dict.update(override_dict)
  return json.dumps(assignment_dict)

def assignment(assignment_id=None):
  """Returns a test data_store_pb2.Assignment.

  Args:
    assignment_id: Assignment ID string to use or None to use
      ASSIGNMENT_ID.

  Returns:
    data_store_pb2.Assignment instance.
  """
  return data_store_pb2.Assignment(
            project_id=PROJECT_ID,
            brain_id=BRAIN_ID,
            session_id=SESSION_ID,
            assignment_id=(assignment_id if assignment_id else ASSIGNMENT_ID))

def episode_chunks(steps_per_episode_chunk, episode_id):
  """Get example episode chunks.

  Args:
    steps_per_episode_chunk: List of number of steps for each episode chunk.
      For example, [2, 4] would create two chunks with 2 and 4 steps
      respectively.
    episode_id: ID of the episode all chunks belong to.

  Returns:
    List of episode_pb2.EpisodeChunk protos.
  """
  chunk = episode_pb2.EpisodeChunk()
  chunks = []
  step_timestamp = 0
  number_of_chunks = len(steps_per_episode_chunk)
  for chunk_index, chunk_size in enumerate(steps_per_episode_chunk):
    chunk = data_store_pb2.EpisodeChunk(
        project_id=PROJECT_ID,
        brain_id=BRAIN_ID,
        session_id=SESSION_ID,
        episode_id=episode_id,
        chunk_id=chunk_index)
    for _ in range(chunk_size):
      step = episode_pb2.Step()
      step.timestamp_millis = step_timestamp
      step_timestamp += 1000
      step.observation.CopyFrom(observation_data(50, [1, 2, 3]))
      step.action.CopyFrom(action_data(1, 0.3))

      chunk.data.episode_id = episode_id
      chunk.data.chunk_id = chunk_index
      chunk.data.steps.append(step)
      if chunk_index < number_of_chunks - 1:
        chunk.data.episode_state = episode_pb2.IN_PROGRESS
      else:
        chunk.data.episode_state = episode_pb2.SUCCESS
    chunks.append(chunk)

  return chunks

def _read_text_file(filename):
  """Read the contents of a text file relative into a string.

  Args:
    filename: Path to a file relative to this module.

  Returns:
    String contents of the file.
  """
  with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), filename),
            mode='rt') as file_obj:
    return file_obj.read()

def observation_spec():
  """Builds an example ObservationSpec."""
  spec = observation_pb2.ObservationSpec()
  text_format.Parse(_read_text_file('example_model/observation_spec.pbtxt'),
                    spec)
  return spec

def observation_data(health, position):
  """Builds an example ObservationData object."""
  data = observation_pb2.ObservationData()
  text_format.Parse(
      _read_text_file(
          'example_model/observation_data.pbtxt_template').format(
              position0=position[0],
              position1=position[1],
              position2=position[2],
              health=health), data)
  return data

def action_spec():
  """Builds an example ActionSpec."""
  spec = action_pb2.ActionSpec()
  text_format.Parse(_read_text_file('example_model/action_spec.pbtxt'), spec)
  return spec

def action_data(switch_weapon, throttle,
                joy1=None, joy2=None, joy3=None):
  """Builds an example ActionData."""
  joy1 = None or [1.0, 0.0]
  joy2 = None or [0.0, -1.0]
  joy3 = None or [-1.0, 0.5]
  data = action_pb2.ActionData()
  text_format.Parse(
      _read_text_file('example_model/action_data.pbtxt_template').format(
          switch_weapon=switch_weapon,
          throttle=throttle,
          joy1_x_axis=joy1[0],
          joy1_y_axis=joy1[1],
          joy2_x_axis=joy2[0],
          joy2_y_axis=joy2[1],
          joy3_x_axis=joy3[0],
          joy3_y_axis=joy3[1]), data)
  return data

def brain_spec():
  return brain_pb2.BrainSpec(
      observation_spec=observation_spec(),
      action_spec=action_spec())

def brain():
  return data_store_pb2.Brain(
      project_id=PROJECT_ID,
      brain_id=BRAIN_ID,
      name='test_brain',
      brain_spec=brain_spec())

def trajectory(health=100, position=(0, 0, 0), switch_weapon=0, throttle=1.0,
               add_batch_dimension=False):
  """Return a single frame example trajectory.

  Args:
    health: Health to add to the observation. (0.0...100.0)
    position: Position to add to the player entity.
      (3 component tuple of floats)
    switch_weapon: Switch weapon action. (0...4)
    throttle: Throttle action. (-1.0...1.0)
    add_batch_dimension: Whether to add a batch (outer most) dimension of size
      1 around each tensor in the trajectory.

  Returns:
    TF Agents Trajectory instance.
  """
  obs_spec = tfa_specs.ObservationSpec(observation_spec())
  act_spec = tfa_specs.ActionSpec(action_spec())
  obs_data = obs_spec.tfa_value(observation_data(health, position))
  act_data = act_spec.tfa_value(action_data(switch_weapon, throttle))

  act_step = policy_step.PolicyStep(action=act_data)
  reward_array = np.array([0.0], dtype=np.float32)
  discount_array = np.array([0.99], dtype=np.float32)

  time_step = ts.transition(
      obs_data, reward=reward_array, discount=discount_array)
  traj = tf_trajectory.from_transition(
      time_step, act_step, time_step)
  if add_batch_dimension:
    traj = tf.nest.map_structure(lambda tensor: tf.expand_dims(tensor, axis=0),
                                 traj)
  return traj


from tqdm import tqdm
import torch
from torch.utils.data import Dataset
import numpy as np
import time

# Training Data
class TSPDataset(Dataset):
    def __init__(self, dataset_fname=None, train=False, size=50, num_samples=100000, random_seed=1111):
        super(TSPDataset, self).__init__()
        torch.manual_seed(random_seed)
        self.data_set = []

        # randomly sample points uniformly from [0, 1]
        for l in tqdm(range(num_samples)):
            x = torch.FloatTensor(2, size).uniform_(0, 1)
            # x = torch.cat([start, x], 1)
            self.data_set.append(x)

        self.size = len(self.data_set)

    def __len__(self):
        return self.size

    def __getitem__(self, idx):
        return self.data_set[idx]

# define problem dimension
input_dim = 2
# define tsp size
size = 50
# define training size
train_size = 1000

dataset = TSPDataset(train=True, size=size,
     num_samples=train_size)

# save the dataset
torch.save(dataset, './TSP50_1000.pt')
# load the dataset
dataset = torch.load('./TSP50_1000.pt')

# Run TSP nearest neighbor search
start = time.time()
avg_dist = 0

for j in range(train_size):
    graph = dataset.__getitem__(j).numpy()
    a = graph[:, 0].reshape(2, 1)
    route = [0]
    points_left = [x for x in range(1, size)]
    total_dist = 0
    for i in range(size-1):
        # Calculate distance from one point to every other point
        dist = np.sqrt(np.square(graph[0, :] - a[0]) + (np.square(graph[1, :] - a[1])))
        dist = np.vstack((np.linspace(0, size-1, num=size).reshape(1, size), dist))
        filtered = dist[:, np.array(points_left)]
        closest_idx = np.argmin(filtered[1, :])
        real_idx = int(filtered[0, closest_idx])
        total_dist += dist[1, real_idx]
        a = graph[:, real_idx].reshape(2, 1)
        route.append(real_idx)
        points_left.remove(real_idx)
    total_dist += dist[1, 0]

    avg_dist += 1 / train_size * total_dist
end = time.time()
print(str(train_size) + " graph used time: ", end - start)
print(str(train_size) + " graph avg dist: ", avg_dist)


import argparse
import itertools
import multiprocessing
import os
import pprint
import subprocess
from os import listdir

def main(args):
    #working_dir = os.environ['BERT_PREP_WORKING_DIR']
    working_dir = args.working_dir

    print('Working Directory:', working_dir)
    print('Action:', args.action)
    print('Dataset Name:', args.dataset)

    if args.input_files:
        args.input_files = args.input_files.split(',')

    directory_structure = {
        'hdf5': working_dir + '/hdf5',
        'phase1': working_dir + '/phase1'
    }

    print('\nDirectory Structure:')
    pp = pprint.PrettyPrinter(indent=2)
    pp.pprint(directory_structure)
    print('')

    if args.action == 'create_hdf5_files':
        last_process = None

        if not os.path.exists(directory_structure['hdf5'] + "/" + args.dataset):
            os.makedirs(directory_structure['hdf5'] + "/" + args.dataset)

        def create_record_worker(filename_prefix, shard_id, input_file, output_format='hdf5'):
            bert_preprocessing_command = 'python' + ' ' + working_dir + '/Bug-Report-Summarization/src' + '/create_pretraining_data_h.py'
            bert_preprocessing_command += ' --input_file=' + input_file
            bert_preprocessing_command += ' --output_file=' + directory_structure['hdf5'] + '/' + args.dataset + '/' + filename_prefix + '_' + str(shard_id) + '.' + output_format
            bert_preprocessing_command += ' --vocab_file=' + args.vocab_file
            bert_preprocessing_command += ' --do_lower_case' if args.do_lower_case else ''
            bert_preprocessing_command += ' --max_seq_length=' + str(args.max_seq_length)
            bert_preprocessing_command += ' --max_predictions_per_seq=' + str(args.max_predictions_per_seq)
            bert_preprocessing_command += ' --masked_lm_prob=' + str(args.masked_lm_prob)
            bert_preprocessing_command += ' --random_seed=' + str(args.random_seed)
            bert_preprocessing_command += ' --dupe_factor=' + str(args.dupe_factor)
            bert_preprocessing_process = subprocess.Popen(bert_preprocessing_command, shell=True)

            last_process = bert_preprocessing_process

            # This could be better optimized (fine if all take equal time)
            if shard_id % args.n_processes == 0 and shard_id > 0:
                bert_preprocessing_process.wait()
            return last_process

        output_file_prefix = args.dataset

        files = [directory_structure['phase1'] + '/' + file for file in listdir(directory_structure['phase1'])]
        for i in range(args.n_training_shards):
            last_process = create_record_worker(output_file_prefix + '_phase1_', i, files[i])

        last_process.wait()
        '''
        if args.n_test_shards:
            for i in range(args.n_test_shards):
                last_process = create_record_worker(output_file_prefix + '_phase2_', i)

            last_process.wait()
        '''

if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description='Preprocessing Application for Everything BERT-related'
    )

    parser.add_argument(
        '--working_dir',
        type=str,
        help='Specify the current working directory',
    )

    parser.add_argument(
        '--action',
        type=str,
        help='Specify the action you want the app to take. e.g., generate vocab, segment, create tfrecords',
        choices={
            'create_hdf5_files' # Turn each shard into a HDF5 file with masking and next sentence prediction info
        }
    )

    parser.add_argument(
        '--dataset',
        type=str,
        help='Specify the dataset to perform --action on',
        choices={
            'regular',
            'heuristics'
        }
    )

    parser.add_argument(
        '--input_files',
        type=str,
        help='Specify the input files in a comma-separated list (no spaces)',
        required=False
    )

    parser.add_argument(
        '--n_training_shards',
        type=int,
        help='Specify the number of training shards to generate',
        default=256
    )

    parser.add_argument(
        '--n_test_shards',
        type=int,
        help='Specify the number of test shards to generate',
        default=256,
        required=False
    )

    parser.add_argument(
        '--fraction_test_set',
        type=float,
        help='Specify the fraction (0..1) of the data to withhold for the test data split (based on number of sequences)',
        default=0.1,
        required=False
    )

    parser.add_argument(
        '--n_processes',
        type=int,
        help='Specify the max number of processes to allow at one time',
        default=10
    )

    parser.add_argument(
        '--random_seed',
        type=int,
        help='Specify the base seed to use for any random number generation',
        default=12345
    )

    parser.add_argument(
        '--dupe_factor',
        type=int,
        help='Specify the duplication factor',
        default=10
    )

    parser.add_argument(
        '--masked_lm_prob',
        type=float,
        help='Specify the probability for masked lm',
        default=0.15
    )

    parser.add_argument(
        '--max_seq_length',
        type=int,
        help='Specify the maximum sequence length',
        default=128
    )

    parser.add_argument(
        '--max_predictions_per_seq',
        type=int,
        help='Specify the maximum number of masked words per sequence',
        default=20
    )

    parser.add_argument(
        '--do_lower_case',
        type=int,
        help='Specify whether it is cased (0) or uncased (1) (any number greater than 0 will be treated as uncased)',
        default=0
    )

    parser.add_argument(
        '--vocab_file',
        type=str,
        help='Specify absolute path to vocab file to use)'
    )

    parser.add_argument(
        '--interactive_json_config_generator',
        type=str,
        help='Specify the action you want the app to take. e.g., generate vocab, segment, create tfrecords'
    )

    args = parser.parse_args()
    main(args)


import logging
import socket
import ssl
from queue import Queue
from typing import Callable

class DatadogHandler(logging.handlers.SocketHandler):
    """
    DataDogHandler is a subclass of SocketHandler that sends logs to DD's
    Log intake endpoint.

    To understand the exact nature of what this Handler does, reference:
    https://github.com/python/cpython/blob/master/Lib/logging/handlers.py
    """

    def __init__(self, *, print_debug: bool = False):
        """
        Initialize the parent SocketHandler with datadog's info. Hardcoding
        this is fine, as, should this change we will likely need to fix this
        code anyway. This also gives less chance for users to mess things up
        as well
        """
        # Init parent with Datadog log intake endpoint
        #
        # See: https://docs.datadoghq.com/logs/log_collection/?tab=host#supported-endpoints  # noqa
        super().__init__("intake.logs.datadoghq.com", 10516)

        # print_debug flag
        self.print_debug = print_debug

    def close(self):
        """closes the parent handler"""
        if self.print_debug:
            print("datadoglog: close")
        super().close()

    def makePickle(self, record: logging.LogRecord) -> bytes:
        """prepares record for writing over the wire"""
        return f"{record.message}\n".encode()

    def makeSocket(self, timeout: int = 1) -> ssl.SSLSocket:
        """
        A factory method which allows subclasses to define the precise type of
        they (DD) want. Since our logging data is sensitive we want to send it
        over SSL so we subclass from an ssl and act accordingly.
        """
        context = ssl.create_default_context()

        # self.address should contain the (host, port) tuple that was
        # initialized by the SocketHandler.__init__. Here we rely on inner
        # workings of other peoples code ( people who encourage us to do so
        # while having no idea what parts of their code we are depending on ).
        #
        # NOTE: written and tested for python 3.6
        conn = socket.create_connection(self.address, timeout=timeout)
        sock = context.wrap_socket(conn, server_hostname=self.host)
        if self.print_debug:
            print(
                f"datadoglog: makeSocket connected {sock} to {self.address} "
                f"with {sock.version()}"
            )
        return sock

    def send(self, s):
        """sends serialized s over the wire"""
        if self.print_debug:
            print(f"datadoglog: send {s}")
        super().send(s)

def start_datadog_logger(
    queue: Queue, *, print_debug: bool = False
) -> Callable[[], None]:
    """
    This starts the QueueListener in separate thread, which will consume and
    forward all the messages to Datadog, so the logging should never block the
    client and can be used by any high performance apps. It returns a function,
    when called will properly shut down the listener thread and close the
    underlying TCP socket.
    """
    if print_debug:
        print(f"datadoglog: starting logger for queue {queue}")

    # intialize our handler and underlying threaded queue listener
    handler = DatadogHandler(print_debug=print_debug)
    listener = logging.handlers.QueueListener(queue, handler)

    # set the log format
    formatter = logging.Formatter("%(message)s")
    handler.setFormatter(formatter)

    # start the listener and return our cancel/stop function to the caller
    listener.start()

    def closer():
        # wait for anything in the queue to process
        listener.stop()
        # close the DatadogHandler
        handler.close()

    return closer


from typing import Optional, Sequence

from ..abstracts import ServiceAbstract
from ..exceptions import PortalAPIResponse, PortalException
from ..util.graphql import GraphQLRequest

class Permission:
    def __init__(self, id: str, slug: str, service: str):
        self.id: str = id
        self.slug: str = slug
        self.service: str = service

class ServicePermissions:
    def __init__(self, graphql_response: Optional[dict]):
        self._permissions: Sequence[Permission] = []
        self._index_services: dict = {}
        self._index_permissions_by_id: dict = {}
        self._index_permissions_by_slug: dict = {}

        if graphql_response:
            self._handle_graphql_response(graphql_response)

    def _handle_graphql_response(self, res: dict) -> None:
        self._permissions = []  # reset
        self._index_services = {}

        idx = 0  # permissions are indexed in dicts for membership testing
        try:
            for entry in res['accessControl']['servicePermissions']:
                service = entry['service']['code']

                for p in entry['permissions']:
                    self._permissions.append(Permission(id=p['id'], slug=p['slug'], service=service))
                    self._index_services.setdefault(service, [])

                    # store the index to the permission
                    self._index_services[service].append(idx)
                    self._index_permissions_by_id[p['id']] = idx
                    self._index_permissions_by_slug[p['slug']] = idx
                    idx += 1
        except KeyError as exc:
            raise PortalAPIResponse(f"failed handling service permission ({exc})")

    def __iter__(self) -> Permission:
        for perm in self._permissions:
            yield perm

    def have(self, p) -> bool:
        """Returns True whether permission is available

        The p argument can be either the ID (a UUID) or the Slug of the permission.

        Note that the list of permissions being checked might be less than the
        permissions the user actually has. This is because only a subset can
        be queried for.

        :param p: ID (UUID) or Slug of the permission to check
        :return: True if permission is available
        """
        return p in self._index_permissions_by_slug or p in self._index_permissions_by_id

    def slugs(self) -> Sequence[str]:
        """Returns sequence of permissions' ID and Slug

        :return: Sequence of IDs and Slugs of all permissions for this service.
        """

    def as_slug_dict(self) -> dict:
        """Returns permission slugs for all services as dict

        :return: Dictionary with key the service code, and value sequence of permission slugs.
        """
        result = {}
        for p in self._permissions:
            try:
                result[p.service].append(p.slug)
            except KeyError:
                result[p.service] = [p.slug]

        return result

class RBACMixin(ServiceAbstract):
    _api = None  # mixed in

    def user_service_permissions(self, user_id: Optional[str] = None,
                                 services: Optional[Sequence[str]] = None) -> ServicePermissions:
        """Retrieves permissions available to user with ID `user_id`. The result is an instance
        of `ServicePermissions` which holds the user's permission for all or for selected services.

        Raises `PortalAPIError` When the GraphQL API endpoint returned an error.
        When there was an issue with the request itself, or decoding JSON failed,
        the `PortalAPIRequest` exception is raised.
        """
        variables = {
            "id": user_id,
            "services": services
        }

        request = GraphQLRequest(api_url=self._api.api_url, token=self._api.token,
                                 query=_GRAPHQL_QUERY_USER_SERVICE_PERMISSIONS, variables=variables)

        try:
            response = request.execute_dict()
        except PortalException:
            raise

        try:
            return ServicePermissions(graphql_response=response['data']['user'])
        except PortalAPIResponse:
            raise

_GRAPHQL_QUERY_USER_SERVICE_PERMISSIONS = """
query ($id: ID, $services: [String!]) {
  user: auth_user(id: $id) {
    accessControl {
      servicePermissions(filter: {serviceCode: $services}) {
        service { code }
        permissions { id slug }
      }
    }
  }
}
"""


#!/usr/bin/env python
# Plastic Metal
# (c) 2011-2012 Ian Preston
import pygame
import subprocess
import shlex
import ConfigParser
import os.path
import optparse # :(

# Guitar low E is 29 half steps below middle A
GTR_E = -29

STRINGS = {
    'E': GTR_E,
    'A': GTR_E+5,
    'D': GTR_E+5+5,
    'G': GTR_E+5+5+5,
    'B': GTR_E+5+5+5+4,
    'e': GTR_E+5+5+5+4+5
}

# Mapping of joystick input codes to color buttons
BTN_GREEN, BTN_RED, BTN_YELLOW, BTN_BLUE, BTN_ORANGE, BTN_XBOX = 0, 1, 3, 2, 4, 7

# Named constants for fretstates
FS_GREEN, FS_RED, FS_YELLOW, FS_BLUE, FS_ORANGE = 0, 1, 2, 3, 4

class ImproperlyFormattedChordSpecError(Exception): pass

class SynthChord(object):
    def __init__(self, notes, distort=True):
        self.notes = notes
        self.processes = []
        self.distort = distort

    def _play_note(self, note):
        distortion = ('overdrive 25 100' if self.distort else '')
        cmd = shlex.split('play -n synth 10 pluck %{0} {1}'.format(note, distortion))
        self.processes.append(subprocess.Popen(cmd,
                                               stdin=open('/dev/null'),
                                               stdout=open('/dev/null', 'w'),
                                               stderr=open('/dev/null', 'w')))

    def play(self):
        for n in self.notes:
            self._play_note(n)
    def stop(self):
        for proc in self.processes: proc.kill()

class PlasticMetal(object):
    def __init__(self, chord_map_filename):
        self.guitar = pygame.joystick.Joystick(0)
        self.guitar.init()

        # Parse the fretstate-to-powerchord map file
        self.chord_map = ConfigParser.RawConfigParser()
        self.chord_map.read(chord_map_filename)

        # The current state of which frets are pressed down
        self.fret_state      = [False, False, False, False, False]

        # The state of the frets last frame
        self.last_fret_state = [False, False, False, False, False]

        # Is the whammy being pressed down?
        self.is_whammy_down = False

        # Is distortion enabled? (toggle with Xbox button)
        self.is_distortion_enabled = True

        # The PowerChord that is currently playing
        self.current_chord = SynthChord([])

    def get_notes_from_states(self):
        """
        Takes the current fretstate and returns a list of notes that should
        be played, using the map file. The returned notes are integers relative
        to middle A.
        """

        # Make a string representation of the currently-pressed frets in the same format as the
        # keys in the map file. For example, if the green and red frets are held,
        # chord_map_key would be "11000"
        chord_map_key = ('1' if self.fret_state[FS_GREEN] else '0') + \
                        ('1' if self.fret_state[FS_RED] else '0') + \
                        ('1' if self.fret_state[FS_YELLOW] else '0') + \
                        ('1' if self.fret_state[FS_BLUE] else '0') + \
                        ('1' if self.fret_state[FS_ORANGE] else '0')
        try:
            notes = []

            # Get the list of note specs from the file using the key we generated
            # above
            chord_map_note_specs = self.chord_map.get('Chords', chord_map_key).split(',')
            for spec in chord_map_note_specs:
                try:
                    # Each note to play from the map file is in the format <string><fret>. I.e.
                    # to play the 3rd fret on the e string, the spec would be e3
                    spec_string = spec[0]
                    spec_fret = int(spec[1:])
                    note = STRINGS[spec_string] + spec_fret
                    notes.append(note)
                except KeyError:
                    # The first character of the note specification was not in
                    # STRINGS, so it's not a valid string.
                    raise ImproperlyFormattedChordSpecError('Not a valid guitar string: {0}'.format(spec_string))
                except ValueError:
                    # The next characters of the note specification could not
                    # be converted to an int, so they are not a fret number
                    raise ImproperlyFormattedChordSpecError('Not a valid fret: {0}'.format(spec_fret))
                except IndexError:
                    # Either [0] or [1:] was out of range, so the note spec is not
                    # valid or doesn't exist
                    raise ImproperlyFormattedChordSpecError('Invalid format: '.format(spec))
        except ConfigParser.NoOptionError:
            notes = []

        return notes

    def run(self):
        while True:
            for event in pygame.event.get():
                if event.type == pygame.JOYBUTTONDOWN:
                    if event.button == BTN_GREEN   : self.fret_state[FS_GREEN] = True
                    elif event.button == BTN_RED   : self.fret_state[FS_RED] = True
                    elif event.button == BTN_YELLOW: self.fret_state[FS_YELLOW] = True
                    elif event.button == BTN_BLUE  : self.fret_state[FS_BLUE] = True
                    elif event.button == BTN_ORANGE: self.fret_state[FS_ORANGE] = True

                elif event.type == pygame.JOYBUTTONUP:
                    if event.button == BTN_GREEN   : self.fret_state[FS_GREEN] = False
                    elif event.button == BTN_RED   : self.fret_state[FS_RED] = False
                    elif event.button == BTN_YELLOW: self.fret_state[FS_YELLOW] = False
                    elif event.button == BTN_BLUE  : self.fret_state[FS_BLUE] = False
                    elif event.button == BTN_ORANGE: self.fret_state[FS_ORANGE] = False
                    elif event.button == BTN_XBOX  : self.is_distortion_enabled = not self.is_distortion_enabled

                elif event.type == pygame.JOYHATMOTION and event.hat == 0:
                    # When the strum bar is hit, stop the current chord and start
                    # playing the chord that is pressed down.
                    if event.value[1] != 0:
                        self.current_chord.stop()
                        try:
                            self.current_chord = SynthChord(self.get_notes_from_states(), self.is_distortion_enabled)
                            self.current_chord.play()
                        except ImproperlyFormattedChordSpecError:
                            pass

                elif event.type == pygame.JOYAXISMOTION and event.axis == 3:
                    # Map whammy bar to a keystate (more than half down = pressed)
                    if event.value >= 0: self.is_whammy_down = True
                    else               : self.is_whammy_down = False

            # If the state of the frets has changed since last frame, stop the current
            # chord.
            if self.fret_state != self.last_fret_state:
                self.current_chord.stop()

            self.last_fret_state = self.fret_state[:]

if __name__ == '__main__':
    parser = optparse.OptionParser()
    parser.add_option('-m', '--mapfile', dest='mapfile', metavar='FILE', help='Specify the map file')

    options, args = parser.parse_args()
    chord_map_filename = options.mapfile or os.path.join(os.path.dirname(__file__), 'default.map')

    pygame.init()
    pygame.joystick.init()
    PlasticMetal(chord_map_filename).run()


from __future__ import annotations
import math
from typing import Dict, Optional
from returnn.tf.util.basic import get_shape
import tensorflow as tf

class _ReturnnAudioFeatureExtractor:
    def __init__(self):
        self.opts = None  # type: Optional[Params]
        self.tf_session = None  # type: Optional[tf.compat.v1.Session]
        self.tf_audio_placeholder = None
        self.tf_log_mel = None

    def __repr__(self):
        return f"{self.__class__.__name__}{self.opts.items() if self.opts else 'uninitialized'}"

    def setup(self, opts: Params):
        """
        Setup
        """
        if self.opts:
            self.opts.assert_same(opts)
            return
        self.opts = opts

        with tf.compat.v1.Graph().as_default() as g:
            self.tf_audio_placeholder = tf.compat.v1.placeholder(name="audio", dtype=tf.float32, shape=(None,))
            log_mel = extract_log_mel_features_from_audio(self.tf_audio_placeholder, **opts.items())
            log_mel = tf.squeeze(log_mel, axis=0)
            log_mel = tf.squeeze(log_mel, axis=-1)
            self.tf_log_mel = log_mel
            self.tf_session = tf.compat.v1.Session(graph=g)

    def extract(self, *, audio, num_feature_filters, sample_rate, **_other):
        """
        Extract
        """
        assert sample_rate == self.opts.sample_rate
        assert num_feature_filters == self.opts.num_bins
        return self.tf_session.run(self.tf_log_mel, feed_dict={self.tf_audio_placeholder: audio})

_global_returnn_audio_feature_extractor = _ReturnnAudioFeatureExtractor()

def make_returnn_audio_features_func(cached=_global_returnn_audio_feature_extractor, **opts):
    """
    This can be used for ExtractAudioFeatures in RETURNN,
    e.g. in OggZipDataset or LibriSpeechCorpus or others.

    This is cached by default, but you can disable this by setting cached=None.
    """
    if cached:
        extractor = cached
    else:
        extractor = _ReturnnAudioFeatureExtractor()
    p = default_asr_frontend_params()
    p.update(opts)
    extractor.setup(p)
    return extractor.extract

def default_asr_frontend_params(num_bins=80, sample_rate=16000):
    """
    Make params for default ASR frontent
    """
    p = MelAsrFrontend.make_params()
    p.sample_rate = float(sample_rate)
    p.frame_size_ms = 25.0
    p.frame_step_ms = 10.0
    p.num_bins = num_bins
    p.lower_edge_hertz = 125.0
    p.upper_edge_hertz = 7600.0
    p.preemph = 0.97
    p.noise_scale = 0.0
    p.pad_end = False
    return p

def extract_log_mel_features_from_audio(audio, **opts):
    """Create Log-Mel Filterbank Features from audio samples.
    Args:
      audio: Tensor representing audio samples (normalized in [-1,1)).
        It is currently assumed that the wav file is encoded at 16KHz.
        Shape [num_frames] or [num_frames,1].
    Returns:
      A Tensor representing three stacked log-Mel filterbank energies, sub-sampled
      every three frames.
    """
    assert isinstance(audio, tf.Tensor)
    audio *= 32768  # that's what it expects...
    if audio.shape.ndims >= 2:
        # Remove channel dimension, since we have a single channel.
        audio = tf.squeeze(audio, axis=1)
    audio = tf.expand_dims(audio, axis=0)  # [B,num_frames]
    p = default_asr_frontend_params()
    p.update(opts)
    mel_frontend = MelAsrFrontend(**p.items())
    log_mel, _ = mel_frontend.fprop(audio)
    return log_mel

def extract_log_mel_features_from_wav(wav_bytes_t, **opts):
    """Create Log-Mel Filterbank Features from raw bytes.
    Args:
      wav_bytes_t: Tensor representing raw wav file as a string of bytes. It is
        currently assumed that the wav file is encoded at 16KHz (see DecodeWav,
        below).
    Returns:
      A Tensor representing three stacked log-Mel filterbank energies, sub-sampled
      every three frames.
    """
    result = tf.audio.decode_wav(wav_bytes_t)
    sample_rate, audio = result.sample_rate, result.audio
    audio *= 32768
    # Remove channel dimension, since we have a single channel.
    audio = tf.squeeze(audio, axis=1)
    # TODO(drpng): make batches.
    audio = tf.expand_dims(audio, axis=0)
    p = default_asr_frontend_params()
    p.update(opts)
    mel_frontend = MelAsrFrontend(**p.items())
    with tf.control_dependencies([tf.assert_equal(sample_rate, int(p.sample_rate))]):
        log_mel, _ = mel_frontend.fprop(audio)
    return log_mel

def _next_power_of_two(i):
    return math.pow(2, math.ceil(math.log(i, 2)))

class MelAsrFrontend:
    """An AsrFrontend that implements mel feature extraction from PCM frames.
    This is expressed in pure TensorFlow and without reference to external
    resources.
    The frontend implements the following stages:
        `Framer -> Window -> FFT -> FilterBank -> MeanStdDev -> SubSample`
    The FProp input to this layer can either have rank 3 or rank 4 shape:
        [batch_size, timestep, packet_size, channel_count]
        [batch_size, timestep * packet_size, channel_count]
    For compatibility with existing code, 2D [batch_size, timestep] mono shapes
    are also supported.
    In the common case, the packet_size is 1. The 4D variant is accepted for
    glueless interface to input generators that frame their input samples in
    some way. The external framing choice does not influence the operation of
    this instance, but it is accepted.
    TODO(laurenzo): Refactor call sites to uniformly use the 4D variant and
    eliminate fallback logic in this class.
    Only 1 channel is currently supported.
    TODO(laurenzo): Refactor this class to operate on multi-channel inputs.
    """

    @classmethod
    def make_params(cls):
        """
        Make params
        """
        p = Params()
        p.name = "frontend"
        p.define("random_seed", 0)
        p.define("sample_rate", 16000.0, "Sample rate in Hz")
        p.define("channel_count", 1, "Number of channels.")
        p.define("frame_size_ms", 25.0, "Amount of data grabbed for each frame during analysis")
        p.define("frame_step_ms", 10.0, "Number of ms to jump between frames")
        p.define("num_bins", 80, "Number of bins in the mel-spectrogram output")
        p.define("lower_edge_hertz", 125.0, "The lowest frequency of the mel-spectrogram analsis")
        p.define("upper_edge_hertz", 7600.0, "The highest frequency of the mel-spectrogram analsis")
        p.define(
            "preemph",
            0.97,
            "The first-order filter coefficient used for preemphasis. When it " "is 0.0, preemphasis is turned off.",
        )
        p.define("noise_scale", 8.0, "The amount of noise (in 16-bit LSB units) to add")
        p.define("window_fn", "HANNING", 'Window function to apply (valid values are "HANNING", and None)')
        p.define(
            "pad_end",
            False,
            "Whether to pad the end of `signals` with zeros when the provided "
            "frame length and step produces a frame that lies partially past "
            "its end.",
        )
        p.define(
            "per_bin_mean", None, "Per-bin (num_bins) means for normalizing the spectrograms. " "Defaults to zeros."
        )
        p.define("per_bin_stddev", None, "Per-bin (num_bins) standard deviations. Defaults to ones.")
        p.define("stack_left_context", 0, "Number of left context frames to stack.")
        p.define("stack_right_context", 0, "Number of right context frames to stack.")
        p.define("frame_stride", 1, "The frame stride for sub-sampling.")

        p.define("fft_overdrive", True, "Whether to use twice the minimum fft resolution.")
        p.define("output_floor", 1.0, "Minimum output of filterbank output prior to taking logarithm.")
        p.define(
            "compute_energy",
            False,
            "Whether to compute filterbank output on the energy of spectrum " "rather than just the magnitude.",
        )
        p.define("use_divide_stream", False, "Whether use a divide stream to the input signal.")
        return p

    def __init__(self, **kwargs):
        """
        For allowed kwargs, see self._make_params.
        """
        self.params = self.make_params()
        self.params.update(kwargs)
        p = self.params
        if p.frame_stride < 1:
            raise ValueError("frame_stride must be positive.")

        assert p.channel_count == 1, "Only 1 channel currently supported."
        # Make sure key params are in floating point.
        p.sample_rate = float(p.sample_rate)
        p.frame_step_ms = float(p.frame_step_ms)
        p.frame_size_ms = float(p.frame_size_ms)
        p.lower_edge_hertz = float(p.lower_edge_hertz)
        p.upper_edge_hertz = float(p.upper_edge_hertz)

        self._frame_step = int(round(p.sample_rate * p.frame_step_ms / 1000.0))
        self._frame_size = int(round(p.sample_rate * p.frame_size_ms / 1000.0)) + 1  # +1 for the preemph

        self._fft_size = int(max(512, _next_power_of_two(self._frame_size)))
        if p.fft_overdrive:
            self._fft_size *= 2

        self._create_window_function()

        # Mean/stddev.
        if p.per_bin_mean is None:
            p.per_bin_mean = [0.0] * p.num_bins
        if p.per_bin_stddev is None:
            p.per_bin_stddev = [1.0] * p.num_bins
        assert len(p.per_bin_mean) == p.num_bins
        assert len(p.per_bin_stddev) == p.num_bins

    def _create_window_function(self):
        p = self.params
        if p.window_fn is None:
            self._window_fn = None
        elif p.window_fn == "HANNING":

            def _hanning_window(frame_size, dtype):
                return tf.signal.hann_window(frame_size, dtype=dtype)

            self._window_fn = _hanning_window
        else:
            raise ValueError("Illegal value %r for window_fn param" % p.window_fn)

    @property
    def window_frame_size(self):
        """
        Window frame size
        """
        return self._frame_size

    @property
    def window_frame_step(self):
        """
        Window frame step
        """
        return self._frame_step

    @staticmethod
    def _remove_channel_dim(pcm_audio_data):
        if pcm_audio_data.shape.rank == 3:
            pcm_audio_data = tf.squeeze(pcm_audio_data, 2)
            assert pcm_audio_data.shape.rank == 2, "MelAsrFrontend only supports one channel"
        return pcm_audio_data

    @staticmethod
    def _reshape_to_mono2d(pcm_audio_data, paddings):
        """Reshapes a 3D or 4D input to 2D.
        Since the input to FProp can be 3D or 4D (see class comments), this will
        collapse it back to a 2D, mono shape for internal processing.
        Args:
          pcm_audio_data: 2D, 3D or 4D audio input. See class comments. Must have a
            rank.
          paddings: Original paddings shaped to the first two dims of
            pcm_audio_data.
        Returns:
          Tuple of 2D [batch_size, timestep] mono audio data, new paddings.
        """
        shape = get_shape(pcm_audio_data)
        rank = len(shape)
        if rank == 2:
            return pcm_audio_data, paddings
        elif rank == 3:
            # [batch, time, channel]
            with tf.control_dependencies([tf.assert_equal(shape[2], 1)]):
                return tf.squeeze(pcm_audio_data, axis=2), paddings
        elif rank == 4:
            # [batch, time, packet, channel]
            batch_size, orig_time, orig_packet_size, channel = shape
            time = orig_time * orig_packet_size
            with tf.control_dependencies([tf.assert_equal(channel, 1)]):
                pcm_audio_data = tf.reshape(pcm_audio_data, (batch_size, time))
                # Transform paddings into the new time base with a padding per time
                # step vs per packet by duplicating each packet.
                paddings = tf.reshape(
                    tf.tile(tf.expand_dims(paddings, axis=2), [1, 1, orig_packet_size]), (batch_size, time)
                )
                return pcm_audio_data, paddings
        else:
            raise ValueError("Illegal pcm_audio_data shape")

    def fprop(self, audio, paddings=None):
        """Perform signal processing on a sequence of PCM data.
        NOTE: This implementation does not currently support paddings, and they
        are accepted for compatibility with the super-class.
        TODO(laurenzo): Rework this to support paddings.
        Args:
          audio: int16 or float32 tensor of PCM audio data, scaled to
            [-32768..32768] (versus [-1..1)!). See class comments for supported
            input shapes.
          paddings: per frame 0/1 paddings. Shaped: [batch, frame].
        Returns:
          encoder inputs which can be passed directly to a
          compatible encoder and contains:
          - 'src_inputs': inputs to the encoder, minimally of shape
          [batch, time, ...].
          - 'paddings': a 0/1 tensor of shape [batch, time].
        """
        if paddings is None:
            paddings = tf.zeros_like(audio)

        pcm_audio_data, pcm_audio_paddings = self._reshape_to_mono2d(audio, paddings)

        mel_spectrogram, mel_spectrogram_paddings = self._fprop_chunk(pcm_audio_data, pcm_audio_paddings)

        mel_spectrogram, mel_spectrogram_paddings = self._pad_and_reshape_spec(
            mel_spectrogram, mel_spectrogram_paddings
        )

        return mel_spectrogram, mel_spectrogram_paddings

    @staticmethod
    def _stack_signal(signal, stack_size, stride):
        signal = tf.signal.frame(
            signal=signal,
            frame_length=stack_size,
            frame_step=stride,
            pad_end=False,
            axis=1,
        )
        signal = tf.reshape(signal, get_shape(signal)[:2] + [-1])
        return signal

    def _pad_and_reshape_spec(self, mel_spectrogram, mel_spectrogram_paddings):
        p = self.params
        # Stack and sub-sample.
        stack_size = 1
        if p.stack_left_context > 0:
            # Since left context is leading, pad the left by duplicating the first
            # frame.
            stack_size += p.stack_left_context
            mel_spectrogram = tf.concat([mel_spectrogram[:, 0:1, :]] * p.stack_left_context + [mel_spectrogram], axis=1)
            mel_spectrogram_paddings = tf.concat(
                [mel_spectrogram_paddings[:, 0:1]] * p.stack_left_context + [mel_spectrogram_paddings], axis=1
            )

        if p.stack_right_context > 0:
            stack_size += p.stack_right_context
            mel_spectrogram = tf.concat(
                [mel_spectrogram] + [mel_spectrogram[:, -1:, :]] * p.stack_right_context, axis=1
            )
            mel_spectrogram_paddings = tf.concat(
                [mel_spectrogram_paddings] + [mel_spectrogram_paddings[:, -1:]] * p.stack_right_context, axis=1
            )

        if p.stack_left_context or p.stack_right_context:
            mel_spectrogram = self._stack_signal(mel_spectrogram, stack_size, p.frame_stride)
            mel_spectrogram_paddings = self._stack_signal(mel_spectrogram_paddings, stack_size, p.frame_stride)
            # After stacking paddings, pad if any source frame was padded.
            # Stacks into [batch_size, stacked_frame_dim, stack_size] like the
            # spectrogram stacking above, and then reduces the stack_size dim
            # to the max (effectively, making padding = 1.0 if any of the pre-stacked
            # frames were 1.0). Final shape is [batch_size, stacked_frame_dim].
            mel_spectrogram_paddings = tf.reduce_max(mel_spectrogram_paddings, axis=2)

        # Add feature dim. Shape = [batch, time, features, 1]
        mel_spectrogram = tf.expand_dims(mel_spectrogram, -1)
        return mel_spectrogram, mel_spectrogram_paddings

    def _apply_preemphasis(self, framed_signal):
        p = self.params
        preemphasized = framed_signal[:, :, 1:] - p.preemph * framed_signal[:, :, 0:-1]
        return preemphasized

    def _get_mel_padding(self, pcm_audio_paddings):
        p = self.params
        # shape: [batch, time, _frame_size]
        framed_paddings = tf.signal.frame(pcm_audio_paddings, self._frame_size, self._frame_step, p.pad_end)
        # Pad spectrograms that have any padded frames.
        mel_spectrogram_paddings = tf.reduce_max(framed_paddings, axis=2)
        return mel_spectrogram_paddings

    def _fprop_chunk(self, pcm_audio_chunk, pcm_audio_paddings):
        p = self.params
        pcm_audio_chunk = tf.cast(pcm_audio_chunk, tf.float32)
        if p.use_divide_stream:
            pcm_audio_chunk = pcm_audio_chunk / 32768.0

        # shape: [batch, time, _frame_size]
        framed_signal = tf.signal.frame(pcm_audio_chunk, self._frame_size, self._frame_step, p.pad_end)

        # Pre-emphasis.
        if p.preemph != 0.0:
            preemphasized = self._apply_preemphasis(framed_signal)
        else:
            preemphasized = framed_signal[..., :-1]

        # Noise.
        if p.noise_scale > 0.0:
            noise_signal = tf.random.normal(tf.shape(preemphasized), stddev=p.noise_scale, mean=0.0, seed=p.random_seed)
        else:
            noise_signal = 0.0

        # Apply window fn.
        windowed_signal = preemphasized + noise_signal
        if self._window_fn is not None:
            window = self._window_fn(self._frame_size - 1, framed_signal.dtype)
            windowed_signal *= window

        mel_spectrogram = self._mel_spectrogram(windowed_signal)

        mel_spectrogram_log = tf.math.log(tf.maximum(float(p.output_floor), mel_spectrogram))

        # Mean and stddev.
        mel_spectrogram_norm = (mel_spectrogram_log - tf.convert_to_tensor(p.per_bin_mean)) / tf.convert_to_tensor(
            p.per_bin_stddev
        )
        return mel_spectrogram_norm, self._get_mel_padding(pcm_audio_paddings)

    def _mel_spectrogram(self, signal):
        """Computes the mel spectrogram from a waveform signal.
        Args:
          signal: f32 Tensor, shaped [batch_size, num_samples]
        Returns:
          f32 features Tensor, shaped [batch_size, num_frames, mel_channels]
        """
        p = self.params
        # FFT.
        real_frequency_spectrogram = tf.signal.rfft(signal, [self._fft_size])
        magnitude_spectrogram = tf.abs(real_frequency_spectrogram)
        if p.compute_energy:
            magnitude_spectrogram = tf.square(magnitude_spectrogram)

        # Shape of magnitude_spectrogram is num_frames x (fft_size/2+1)
        # Mel_weight is [num_spectrogram_bins, num_mel_bins]
        mel_weight_matrix = tf.signal.linear_to_mel_weight_matrix(
            num_mel_bins=p.num_bins,
            num_spectrogram_bins=self._fft_size // 2 + 1,
            sample_rate=p.sample_rate,
            lower_edge_hertz=p.lower_edge_hertz,
            upper_edge_hertz=p.upper_edge_hertz,
            dtype=tf.float32,
        )
        # Weight matrix implemented in the magnitude domain.
        batch_size, num_frames, fft_channels = get_shape(magnitude_spectrogram)
        mel_spectrogram = tf.matmul(
            tf.reshape(magnitude_spectrogram, [batch_size * num_frames, fft_channels]), mel_weight_matrix
        )
        mel_spectrogram = tf.reshape(mel_spectrogram, [batch_size, num_frames, p.num_bins])

        return mel_spectrogram

class Params:
    """
    Dummy class to emulate Lingvo `Params` as simple as possible.
    """

    def __init__(self):
        self._params = {"name": None}

    def define(self, key, default_value, _comment=None):
        """
        Define param
        """
        self._params[key] = default_value

    def update(self, d: Dict[str]):
        """
        Update params
        """
        for key, value in d.items():
            assert key in self._params
            self._params[key] = value

    def items(self):
        """
        Param items
        """
        return self._params

    def __repr__(self):
        return "Params{%r}" % (self._params,)

    def __setattr__(self, key, value):
        if key == "_params":
            super(Params, self).__setattr__(key, value)
            return
        assert key in self._params
        self._params[key] = value

    def __getattr__(self, item):
        if item == "_params":
            raise AttributeError("__init__ not yet called")
        if item not in self._params:
            raise AttributeError("Params: key %r unknown" % item)
        return self._params[item]

    def assert_same(self, other: Params):
        """
        Assert same
        """
        assert sorted(self._params.keys()) == sorted(other._params.keys()), f"{self} vs {other}"
        for key in self._params.keys():
            assert self._params[key] == other._params[key], f"param {key}: {self._params[key]} vs {other._params[key]}"


import sys
from PySide2.QtWidgets import QApplication, QMainWindow
#from PySide2.QtUiTools import QUiLoader
from ui_mainwindow import Ui_MainWindow
import findspark
findspark.init('C:/spark')
from pyspark.sql import SparkSession,Row
from pyspark.ml.recommendation import ALS
from pyspark.sql.functions import lit
from PySide2.QtCore import Slot

class MainWindow(QMainWindow):

    def __init__(self):

        super(MainWindow,self).__init__()

        self.ui = Ui_MainWindow()
        self.ui.setupUi(self)
        self.combobox = self.ui.comboBox
        self.combobox_2 = self.ui.comboBox_2
        self.textedit = self.ui.textEdit
        self.pushbutton = self.ui.pushButton
        self.pushbutton_2 = self.ui.pushButton_2
        self.textedit_2 = self.ui.textEdit_2

        spark = SparkSession.builder.appName('movierecommed').getOrCreate()

        def parsemoviedata(line):

            fields = line.split()
            return Row(userid = int(fields[0]),movieid = int(fields[1]),rating = float(fields[2]))

        def submit():

            self.textedit.append(self.combobox.currentText())
            self.textedit_2.append(self.combobox_2.currentText())

        def recommend():

            movie_set = self.textedit.toPlainText().split('\n')
            rating_set = self.textedit_2.toPlainText().split('\n')
            movie_set = set(movie_set)
            rating_set = set(rating_set)

            with open('ml-100k/u.useritem','w') as f:
                 for movieid,rating in zip(movie_set,rating_set):
                     f.write('{0}\t{1}\t{2}\t{3}\n'.format(0,self.movie_title_id[movieid],rating,881250949))
                 with open('ml-100k/u.data') as s:
                     f.write(s.read())

            moviedata = spark.sparkContext.textFile('ml-100k/u.useritem')
            movieparsed = moviedata.map(parsemoviedata)
            df = spark.createDataFrame(movieparsed)

            als = ALS(maxIter=5,regParam=0.01,userCol='userid',itemCol='movieid',ratingCol='rating')
            model = als.fit(df)

            moviecount = df.groupBy('movieid').count().filter('count>100')
            popularmovies = moviecount.select('movieid').withColumn('userid',lit(0))

            recommendation = model.transform(popularmovies)

            spark.conf.set('spark.sql.crossJoin.enabled','true')
            top10 = recommendation.sort(recommendation.prediction.desc()).take(10)
            self.top_recommendation(top10)

        # submit button
        self.pushbutton_2.clicked.connect(submit)
        # recommend button
        self.pushbutton.clicked.connect(recommend)

    def select_movies(self,moviedict):
        self.movie_title_id = {value:key for key,value in moviedict.items()}
        self.moviedict = moviedict
        for moviename in moviedict.items():
            self.combobox.addItem(moviename[1])

    def select_ratings(self,ratings):

        for items in ratings:
            self.combobox_2.addItem(items)

    def top_recommendation(self,top10):
        self.textedit.clear()
        self.textedit_2.clear()
        print(top10)
        for recommend in top10:
            #print(moviedict[recommend['movieid']])
            self.textedit.append(self.moviedict[recommend['movieid']])
            self.textedit_2.append('{:.1f}'.format(recommend['prediction']))

if __name__ == '__main__':

    moviedict = {}
    with open('ml-100k/u.item',encoding='ISO-8859-1') as f:
        for line in f:
            fields = line.split('|')
            #print(fields[1])
            moviedict[int(fields[0])] = fields[1]

    items = []
    sum = 1.0
    for i in range(41):
        items.append('{:0.2}'.format(sum))
        sum += float(0.1)

    items = items[::-1]

    app = QApplication(sys.argv)

    window = MainWindow()

    window.select_movies(moviedict)
    window.select_ratings(items)

    window.show()
    sys.exit(app.exec_())


#!/usr/bin/python3
# -*- coding: utf-8 -*-

import sys
from PyQt5.QtWidgets import *
from PyQt5.QtGui import *

from PyQt5.QtCore import QCoreApplication

class Window_NewUser(QWidget):
    def __init__(self, parent=None):
        super(Help, self).__init__(parent)
        self.setupUi(self)

class Window(QWidget):

    def __init__(self):
        super().__init__()

        self.initUI()

    def Open_NewUser(self):
        widget = Window_NewUser()
        widget.exec_()

    def Window_NewUser(self):

        QToolTip.setFont(QFont('SansSerif', 10))

        self.setToolTip('Add new User')

        self.btn2 = QPushButton('Dialog', self)
        self.btn2.move(20, 20)
        self.btn2.clicked.connect(self.showDialog)

        self.le = QLineEdit(self)
        self.le.move(130, 20)

        btn = QPushButton('Quit', self)
        btn.setToolTip('This is a QPushButton widget')
        btn.clicked.connect(QCoreApplication.instance().quit)
        btn.resize(btn.sizeHint())
        btn.move(20, 50)

        self.sex_btn = QPushButton('Hombre', self)
        self.sex_btn.setCheckable(True)
        self.sex_btn.move(20, 80)

        self.sex_btn.clicked[bool].connect(self.check_sex)

        self.setGeometry(300, 300, 300, 200)
        self.setWindowTitle('Tooltips')
        self.show()

    def initUI(self):

        QToolTip.setFont(QFont('SansSerif', 10))

        self.setToolTip('This is a QWidget widget')

        self.btn2 = QPushButton('Dialog', self)
        self.btn2.move(20, 20)
        self.btn2.clicked.connect(self.showDialog)

        self.le = QLineEdit(self)
        self.le.move(130, 20)

        btn = QPushButton('Quit', self)
        btn.setToolTip('This is a QPushButton widget')
        btn.clicked.connect(QCoreApplication.instance().quit)
        btn.resize(btn.sizeHint())
        btn.move(20, 50)

        self.sex_btn = QPushButton('Hombre', self)
        self.sex_btn.setCheckable(True)
        self.sex_btn.move(20, 80)

        self.sex_btn.clicked[bool].connect(self.check_sex)

        self.setGeometry(300, 300, 300, 200)
        self.setWindowTitle('Tooltips')
        self.show()

    def closeEvent(self, event):

        reply = QMessageBox.question(self, 'Message',
            "Are you sure to quit?", QMessageBox.Yes |
            QMessageBox.No, QMessageBox.No)

        if reply == QMessageBox.Yes:
            event.accept()
        else:
            event.ignore()

    def check_sex(self, pressed):
        print("STATE: "+str(pressed))

        if pressed:
            self.sex_btn.setText("Mujer")
        else:
            self.sex_btn.setText("Hombre")

    def showDialog(self):

        text, ok = QInputDialog.getText(self, 'Input Dialog',
            'Enter your name:')

        if ok:
            self.le.setText(str(text))


# !/usr/bin/env python3
from setuptools import setup

with open('README.md', 'r') as fh:
    long_description = fh.read()

setup(name='hyranote',
      version='0.2.0',
      description='Transform mindnode file to weekly notes',
      long_description=long_description,
      long_description_content_type='text/markdown',
      url='http://github.com/huuhoa/hyranote',
      author='Huu Hoa NGUYEN',
      author_email='huuhoa@gmail.com',
      license='MIT License',
      install_requires=[
          'beautifulsoup4==4.9.1',
      ],
      classifiers=[
          'Programming Language :: Python :: 3',
          'License :: OSI Approved :: MIT License',
          'Operating System :: OS Independent',
      ],
      entry_points={
          'console_scripts': [
              'hyranote = hyranote.cli:main'
          ]
      })


# -*- coding: utf-8 -*-

import time
import traceback
import signal
from multiprocessing.managers import SyncManager
import pickle
import datetime

class NS:

    def __init__(self):
        mgr = SyncManager()
        mgr.start(signal.signal, (signal.SIGINT, signal.SIG_IGN))
        self.ns_default = mgr.Namespace()
        self.ns_default.error = None
        self.ns_stats = mgr.Namespace()
        self.input_queue = mgr.Queue(maxsize=100)
        self.error_occurred = mgr.Event()
        self.error_processed = mgr.Event()
        self.batch_done = mgr.Event()
        self.mgr = mgr
        self.stats_lock = mgr.Lock()
        self.main_lock = mgr.Lock()

    def set_shared_value(self, name, value, no_lock=False):
        if no_lock:
            setattr(self.ns_default, name, value)
            return
        with self.main_lock:
            setattr(self.ns_default, name, value)

    def get_shared_value(self, name, no_lock=False):
        if no_lock:
            return getattr(self.ns_default, name)
        with self.main_lock:
            return getattr(self.ns_default, name)

ns_container = NS()

class Stats:

    stats = ns_container.ns_stats
    start_time = datetime.datetime.utcnow()
    _lock = ns_container.stats_lock
    _locked = False

    @classmethod
    def time_spent(cls):
        return datetime.datetime.utcnow() - cls.start_time

    @classmethod
    def lock(cls):
        cls._lock.acquire()
        cls._locked = True

    @classmethod
    def unlock(cls):
        cls._lock.release()
        cls._locked = False

    @classmethod
    def set_stats_val(cls, name, value):
        cls.lock()
        cls.add_key(name)
        setattr(cls.stats, name, value)
        cls.unlock()
        return value

    @classmethod
    def inc_stats_val(cls, name, value=1):
        cls.lock()
        cls.add_key(name)
        if not hasattr(cls.stats, name):
            setattr(cls.stats, name, 0)
        setattr(cls.stats, name, getattr(cls.stats, name) + value)
        cls.unlock()
        return getattr(cls.stats, name)

    @classmethod
    def get_stats_val(cls, name, default=None):
        cls.lock()
        value = getattr(cls.stats, name, default)
        cls.unlock()
        return value

    @classmethod
    def inc_stats_vals(cls, vals, return_stats=False, **kwargs):
        cls.lock()
        for name, value in vals.items():
            cls.add_key(name)
            if not hasattr(cls.stats, name):
                setattr(cls.stats, name, 0)
            setattr(cls.stats, name, getattr(cls.stats, name) + value)
        if return_stats:
            stats = cls.get_stats(**kwargs)
        cls.unlock()
        if return_stats:
            return stats

    @classmethod
    def get_stats(cls, ignore_keys=(), **kwargs):
        keys = (k for k in cls.stat_keys() if k not in ignore_keys)
        return {k: getattr(cls.stats, k, None) for k in keys}

    @classmethod
    def add_key(cls, name):
        stat_keys = getattr(cls.stats, 'stat_keys', set())
        stat_keys.add(name)
        cls.stats.stat_keys = stat_keys

    @classmethod
    def stat_keys(cls):
        return getattr(cls.stats, 'stat_keys', set())

    @classmethod
    def time_spent(cls):
        return str(datetime.datetime.utcnow() - cls.start_time)

class ExitStatus:
    _listeners = set()
    seconds = 0
    time_start = time.time()

    def __init__(self, message='Success', code=0, target=None, **kwargs):
        self.time_start = kwargs.get('time_start') or self.time_start
        if traceback.sys.exc_info()[0]:
            message = traceback.format_exc()
            code = code or 1

        self.code = code
        self.message = message
        self.target_error = target

    @property
    def seconds(self):
        time_now = time.time()
        return time_now - self.time_start

    def __repr__(self):
        data = 'Finished with code {c} in {s} with message: [{m}]'.format(
            c=self.code,
            s=self.seconds,
            m=self.message
        )
        if self.target_error:
            data += '\nError target: {}'.format(self.target_error)
        return data

    def __iter__(self):
        for f in ['code', 'seconds', 'message', 'target_error']:
            val = getattr(self, f, None)
            if val is not None:
                yield f, val

    def get_error(self):
        if self.target_error:
            return f'target: {self.target_error}\nmessage: {self.message}'

    @classmethod
    def get_status(cls, wait=True):
        if ns_container.error_occurred.is_set():
            wait and ns_container.error_processed.wait()
            result = ns_container.ns_default.error
            status = pickle.loads(result) if isinstance(
                result,
                bytes) else result
        else:
            status = ExitStatus()
        for l in cls._listeners:
            l(status)
        return status

    @classmethod
    def add_exit_listener(cls, listener):
        cls._listeners.add(listener)

def handle_exception_occurred(**kwargs):
    if ns_container.error_processed.is_set():
        return
    result = ExitStatus(**kwargs)
    print(result.message)
    ns_container.ns_default.error = pickle.dumps(result)
    ns_container.error_occurred.set()

def _sigint_default(_signal, frame):
    handle_exception_occurred(code=_signal, message='Keyboard interrupt')

signal.signal(signal.SIGINT, _sigint_default)


# -*- coding: utf-8 -*-
"""
Created on Sun Nov 27 16:15:31 2016

Utility function to count the number of frames for video files in a griven directory

@author: Darshan Shetty
"""
import cv2
import os
def countFramesInFolder(folderLocation):
    totalCount=0;
    videoList=os.listdir(folderLocation)
    for video in videoList:
        filePath=os.path.join(folderLocation,video)
        cap = cv2.VideoCapture(filePath)
        while(cap.isOpened()):
            ret, frame = cap.read()
            if not ret:
                break
            totalCount=totalCount+1
        cap.release()
    return totalCount


from recommenders.recommender_base import RecommenderBase
import numpy as np
import data.data as data
import scipy.sparse as sps
import utils.log as log
import time
from bayes_opt import BayesianOptimization
import sklearn.preprocessing as sk

class Hybrid(RecommenderBase):
    """
    recommender builded passing to the init method an array of extimated R (r_hat_array) that will be combined to obtain an hybrid r_hat
    """
    MAX_MATRIX = 'MAX_MATRIX'
    MAX_ROW = 'MAX_ROW'
    L2 = 'L2'

    def __init__(self, r_hat_array, normalization_mode, urm_filter_tracks):
        self.r_hat_array = r_hat_array
        self.hybrid_r_hat = None
        self.normalized_r_hat_array = None
        self.urm_filter_tracks = urm_filter_tracks
        self._normalization(normalization_mode=normalization_mode)
        print('matrices_normalized')

    def normalize_max_row(self):
        """
        :param userids: user for which compute the predictions
        :return: return an array containing the normalized matrices
        """
        normalized_r_hat_array = []
        count = 0
        for r in self.r_hat_array:
            for row_index in range(r.shape[0]):

                print(row_index*100/r.shape[0])

                row = r.getrow(row_index)
                max_row = row.max()
                r[row_index].data = r[row_index].data/max_row

                #row = r[row_index]
                #max_row = row.max()
                #normalized_row = row/max_row
                #r[row_index] = normalized_row
            normalized_r_hat_array.append(r)
            count+=1
        return normalized_r_hat_array

    def normalize_max_matrix(self):
        """
        :param userids: user for which compute the predictions
        :return: return an array containing the normalized matrices
        """
        normalized_r_hat_array = []
        count = 0
        for r in self.r_hat_array:

            #let the values positives
            #min = r.min()
            #print(min)
            #if min < 0:
            #    r.data = r.data - min

            #avg = np.sum(r.data) / len(r.data)
            #print('avg: {}'.format(avg))
            #r.data = r.data - avg

            max_matrix = r.max()
            print('max: {}'.format(max_matrix))
            r.data = r.data / max_matrix

            #adding confidence
            #r.data=r.data*(r.data-avg)*100
            #for ind in range(len(r.data)):
            #    confidence = (r.data[ind]-avg)*100
            #    r.data[ind] = r.data[ind]*confidence

            normalized_r_hat_array.append(r)
            count += 1
        return normalized_r_hat_array

    def normalize_l2(self):
        normalized_r_hat_array = []
        for r in self.r_hat_array:
            r = sk.normalize(r)
            normalized_r_hat_array.append(r)
        return normalized_r_hat_array

    def _normalization(self, normalization_mode):
        if normalization_mode=='MAX_ROW':
            self.normalized_r_hat_array = self.normalize_max_row()
        elif normalization_mode=='MAX_MATRIX':
            self.normalized_r_hat_array = self.normalize_max_matrix()
        elif normalization_mode == 'NONE':
            self.normalized_r_hat_array = self.r_hat_array
        elif normalization_mode == 'L2':
            self.normalized_r_hat_array = self.normalize_l2()
        else:
            log.error('invalid string for normalization')
            return

    def recommend_batch(self, weights_array, target_userids, N=10, filter_already_liked=True,
                        items_to_exclude=[], verbose=False):
        """
        method used for get the hybrid prediction from the r_hat_matrices passed as parameter during the creation of the recommender
        step1: normalize the matrices, there are two type of normalization
        step2: sum the normalized matrices
        step3: get the prediction from the matrix obtained as sum

        :param weights_array: array of the weights of each r_hat matrix
        :param userids: user for which compute the predictions
        :param normalization_mode: 'MAX_ROW'(default) normalize each row for the max of the row
                                    'MAX_MATRIX' normalize each row for the max of the matrix
        :param N: how many items to predict for each user
        :param filter_already_liked: filter the element with which the user has interacted
        :param items_to_exclude: list of item to exclude from the predictions
        :param verbose:
        :return:
        """

        start = time.time()

        hybrid_r_hat = data.get_empty_urm()

        count = 0
        for m in self.normalized_r_hat_array:
            hybrid_r_hat += m*weights_array[count]
            count += 1

        #filter seen elements
        if filter_already_liked:
            user_profile = self.urm_filter_tracks
            hybrid_r_hat[user_profile.nonzero()] = -np.inf

        """
        # r_hat has only the row for the target playlists, so recreate a matrix with a shape = to the shape of the original_urm
        reconstructed_r_hat = sps.csr_matrix(self.urm.shape)
        reconstructed_r_hat[data.get_target_playlists()] = hybrid_r_hat
        """

        # STEP3
        ranking = np.zeros((len(target_userids), N), dtype=np.int)
        hybrid_r_hat = hybrid_r_hat.todense()

        count = 0
        for row_index in target_userids:
            scores_row = hybrid_r_hat[row_index]

            relevant_items_partition = (-scores_row).argpartition(N)[0, 0:N]
            relevant_items_partition_sorting = np.argsort(-scores_row[0, relevant_items_partition])
            ranking[count] = relevant_items_partition[0, relevant_items_partition_sorting[0, 0:N]]
            count += 1

        print('recommendations created')

        print('{:.2f}'.format(time.time()-start))
        return self._insert_userids_as_first_col(target_userids, ranking)

    def fit(self):
        pass

    def get_r_hat(self, weights_array):
        hybrid_r_hat = data.get_empty_urm()
        count = 0
        for m in self.normalized_r_hat_array:
            hybrid_r_hat += m*weights_array[count]
            count += 1
        return hybrid_r_hat

    def recommend(self, userid, N=10, urm=None, filter_already_liked=True, with_scores=False, items_to_exclude=[]):
        pass

    def run(self):
        pass

    def validateStep(self, **dict):
        # gather saved parameters from self
        targetids = self._validation_dict['targetids']
        urm_test = self._validation_dict['urm_test']
        N = self._validation_dict['N']
        filter_already_liked = self._validation_dict['filter_already_liked']
        items_to_exclude = self._validation_dict['items_to_exclude']

        # build weights array from dictionary
        weights = []
        for i in range(len(dict)):
            w = dict['w{}'.format(i)]
            weights.append(w)

        # evaluate the model with the current weigths
        recs = self.recommend_batch(weights, target_userids=targetids, N=N,
            filter_already_liked=filter_already_liked, items_to_exclude=items_to_exclude, verbose=False)
        return self.evaluate(recs, test_urm=urm_test)

    def validate(self, iterations, urm_test, userids=None,
                        N=10, filter_already_liked=True, items_to_exclude=[], verbose=False):
        # save the params in self to collect them later
        self._validation_dict = {
            'targetids': userids,
            'urm_test': urm_test,
            'N': N,
            'filter_already_liked': filter_already_liked,
            'items_to_exclude': items_to_exclude
        }

        pbounds = {}
        for i in range(len(self.r_hat_array)):
            pbounds['w{}'.format(i)] = (0,1)

        optimizer = BayesianOptimization(
            f=self.validateStep,
            pbounds=pbounds,
            random_state=1,
        )
        optimizer.maximize(
            init_points=2,
            n_iter=iterations,
        )

        print(optimizer.max)
        return optimizer


from scipy.interpolate import CubicSpline
import numpy as np
from sympy import *
from scipy.special import jv, iv
import sort_modes

def calculate_modes(r, h, rho, E, v, freq_setting, freq_value):
    D = E*h**3/(12*(1-v**2))
    print(D)
    freq_list = list()
    Kn_list = list()
    m_list = list()
    n_list = list()

    temporary_freq_list = list()
    temporary_Kn_list = list()
    temporary_m_list = list()
    temporary_n_list = list()

    xpoints = np.linspace(2,30,500)
    ypoints = []

    n = 0
    while n < 10:
        for i in range(len(xpoints)):
            ypoints.append(jv(n,xpoints[i]) * iv(n+1,xpoints[i]) + iv(n,xpoints[i]) * jv(n+1,xpoints[i]))
        cs = CubicSpline(xpoints,ypoints)
        ypoints.clear()
        roots_np_array = CubicSpline.roots(cs)
        roots_list = roots_np_array.tolist()
        while roots_list[0] < 2:
            roots_list.pop(0)
        m = 0
        while m <= 5:
            temporary_freq_list.append((roots_list[m]/r) ** 2 * (D / (rho * h)) ** 0.5 / (2 * np.pi))
            temporary_Kn_list.append(roots_list[m])
            temporary_m_list.append(m)
            temporary_n_list.append(n)
            m += 1
        roots_list.clear()
        n += 1

    freq_list, Kn_list, m_list, n_list = sort_modes.sort_modes(temporary_freq_list, temporary_Kn_list, temporary_m_list, temporary_n_list, freq_setting, freq_value)
    print(freq_list)
    return freq_list, Kn_list, m_list, n_list


import requests
# 导入bs4
from bs4 import BeautifulSoup
import csv

url='http://www.dytt89.com/'
resp=requests.get(url)
resp.encoding='gb2312'

f=open('电影名字.csv',mode='w',encoding='utf-8')
csv_file=csv.writer(f)
# 解析数据
# 1.把页面源代码交给BeautifulSoup处理
# html.parser 告诉处理器这就是html(不加也不影响程序运行)
page=BeautifulSoup(resp.text,"html.parser")
content_div=page.find_all('div',attrs={'class':'co_content222'})[1:] #从第二个开始抓取
# ul=content_div.find('ul')
# li=ul.find_all('li')
for it in content_div:
    uls=it.find('ul')
    lis=uls.find_all('li')[2]
    name=lis.find('a').text
    date=lis.find('span').text
    # print(date)
    csv_file.writerow([name,date])
f.close()
print("over")


# -*- encoding: utf-8 -*-
from flask import Flask
from flask import request, render_template
from klotskigame import KlotskiBoard

app = Flask(__name__)

@app.route('/')
def home(board=None, error_message=None):
    return render_template('index.html', board=board,
                           error_message=error_message)

@app.route('/get_solution', methods=['GET', 'POST'])
def get_solution():
    try:
        if request.method == 'POST':
            line = request.form['board']
            line = line.strip()
            board = []
            for x in line:
                board.append(x)
            if len(board) != 20:
                raise Exception("Wrong input size")
            klotskiBoard = KlotskiBoard()
            klotskiBoard.parse_board(board)
            klotskiBoard.find_solution()
            solution = klotskiBoard.get_solution()
            result = ''
            result += "Steps are:
"
            for p in solution:
                result += p.stringfy()
                result += '
'
            result += 'SOLVED
'
            result += "Number of steps {}".format(len(solution))
            return result
        else:
            return home()
    except Exception as e:
        error_message = "Problem with board input and processing: {}".format(e)
        return home(line, error_message)

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=5000, debug=True)


from django.urls import path

from . import views
app_name='UserProfile'

urlpatterns=[

    path('signin/',views.signin,name='signin'),
    path('',views.profile, name='profile'),
    path('<str:username>',views.friendprofile,name='friendprofile'),
    path('chat/<str:username>',views.chats,name='chats'),
    path('addfriend/',views.addfriend, name='addfriend'),
    path('',views.home, name='home'),
    path('photos/<str:username>',views.Photos,name='Photos'),
    path('notification/',views.note, name='note'),
    #path('comment/',views.comment, name='comment'),
    #path('signin/',views.logout,name='logout'),

]


import boto3

#initialization
athena_client = boto3.client("athena")
s3_client = boto3.client("s3")

class Athena:

    @staticmethod
    def get_query_result(glue_db, query_string, s3_output_location_path, bucket_name):
        ex_id = None
        query_execution = athena_client.start_query_execution(
            QueryString=query_string,
            QueryExecutionContext={
                'Database': glue_db
            },
            ResultConfiguration={
                'OutputLocation': s3_output_location_path,
            }
        )
        while True:
            query_execution_response = athena_client.get_query_execution(
                QueryExecutionId=query_execution["QueryExecutionId"]
            )
            query_state = query_execution_response['QueryExecution']['Status']['State']
            if(query_state == "SUCCEEDED"):
                ex_id = query_execution["QueryExecutionId"]
                break
            elif(query_state == "FAILED"):
                print(query_execution_response['QueryExecution'])
                break
        query_file_path = "resultset/"+ex_id + ".csv"
        fileobj = s3_client.get_object(Bucket=bucket_name, Key=query_file_path)
        return fileobj


from concurrent.futures import process
from celery.exceptions import TimeoutError
from flask import Flask, request
from tasks import process_task
import time

app = Flask(__name__)

@app.route('/')
def index():
    return "Celery With Upstash Redis!"

@app.route('/run', methods = ['POST'])
def run():
    json = request.get_json()
    id = json['id']
    email = json['email']
    diff = json['difficulty']

    process_task.apply_async((diff, id, email), task_id=id)

    return "Your process has been added to the queue.\n"

@app.route('/result', methods = ['POST'])
def result():
    # Here, take the results of the process from celery and do necessary logic with it.
    body = request.get_json()
    id = body['id']

    try:
        result = process_task.AsyncResult(id).get(timeout=1)
        return result
    except TimeoutError:
        return f"Process with id {id} does not exist.\n"

@app.route('/notify', methods = ['POST'])
def notify():
    results = request.get_json()
    email = results['email']

    # Simply handle your notification system: Send email, post via webhook etc.
    # It is also a good place to use celery tasks for sending notifications.
    print(f'Sending Email to {email} the results: {results}')
    return 'OK'


#!/usr/bin/env python
#coding=utf-8

from __future__ import absolute_import
from datetime import timedelta

CELERY_RESULT_BACKEND = 'redis://:admin@10.68.60.111:26379/1'
BROKER_URL = 'amqp://admin:admin@10.68.60.111:5672/devops'
CELERY_TIMEZONE = 'Asia/Shanghai'
BROKER_POOL_LIMIT = None

CELERYBEAT_SCHEDULE = {
    'add-every-30-seconds': {
        'task': 'app.tasks.tasks.add',
        'schedule': timedelta(seconds=30),
        'args': (16, 16)
        },
}


import typing as t
import dataclasses
import secrets as s

from bablyon.warrpers import Request
from bablyon.warrpers import Respone

@dataclasses.dataclass(init=True)
class Config:
    def __init__(
        self,
        secret_key:str=None,
        middlewares:t.List[t.Awaitable]=[],
        request_class:t.Any=Request,
        encoding:str='utf-8'
    ) -> None:
        self.secret_key = secret_key if secret_key is not None else s.token_hex(16)
        self.encoding = encoding
        self.middlewares = middlewares
        self.request_class = request_class

    def __repr__(self) -> str:
        return f'<Config (*****) ({self.encoding}) at 0x({id(self)})>'


#Project
import numpy as np
import matplotlib.pyplot as plt
from random import random
import os
import glob
from astropy.io import fits

def make_circle(x_axis,y_axis,R=510,xc=512.,yc=512.):
    size=x_axis*y_axis           #Initialise size
    x_centre=xc#x_axis/2
    y_centre=yc#y_axis/2
    #Create list
    x=np.arange(size*1.) % x_axis
    y=np.arange(size*1.) % y_axis
    #Reshape to form array
    x = np.reshape(x, (x_axis,y_axis))
    #y array
    y=np.rot90(np.reshape(y, (x_axis,y_axis)))
    #Subtract centre points
    x=x-x_centre
    y=y-y_centre
    #Square array
    x=np.square(x)
    y=np.square(y)
    r1=np.add(x,y)
    #Square root arrays
    r=np.sqrt(r1)
    #Make circle of given stellar radius
    mask=(r<=R)
    r=r*mask
    #Plot stellar model
    #plt.imshow(mask, cmap=plt.cm.binary)
    #plt.ylim([0,1024])
    return r, mask

def limb_darkening(a,b,r,mask):
    #Calculates limb darkening for the star
    mu=np.sqrt(1-r**2)
    LD=r*0.
    LD[mask==1]=1-(a*(1-mu[mask==1]))-(b*((1-mu[mask==1])**2))
    return LD

def gauss(a,sigma,v):
    vrot=(np.arange(1024)-512)/512.*2.5
    #v=np.arange(-12,12,0.02345)
    x=np.shape(vrot)
    y=np.shape(v)
    x=np.rint(x)
    y=np.rint(y)
    gauss=np.zeros((1024,161))
    i=0
    for vx in vrot:
        gauss[i,:]=1-a*np.exp((-(v-vx)**2)/(2*sigma**2))
        i=i+1
    return gauss

def spectrum(array,g,v):
    #Plots the spectrum of the star

    flux=np.zeros(np.shape(v))
    fstar=np.sum(array,0)
    for i in range(0,161):
        flux=flux+fstar[i]*g[i,:]
        i+=1
    flux=flux/np.median(flux[((v<-12)|(v>12))])
    return flux

def planet_motion(aRstar,incl,Rstar,phase,phase_bin,centre,v,amp,line_width):
    #phase=np.arange(-phase,phase,phase_bin)
    x=aRstar*np.sin(2*np.pi*phase)*Rstar+centre
    y=aRstar*np.cos(incl)*np.cos(2*np.pi*phase)*Rstar+centre
    Rplanet = Rstar/10.
    g=gauss(amp,line_width,v)

    star,mask1=make_circle(1025,1025,Rstar)
    star=limb_darkening(0.3,0.3,star/Rstar,mask1)
    star=np.nan_to_num(star)
    print(np.shape(v))
    flux=np.zeros( (111,161) )
    for i in range(0 ,len(phase)):
        planet,mask=make_circle(1025,1025,Rplanet,x[i],y[i])
        system=star*(1.-mask)
        flux[i,:]=spectrum(system,g,v)
        #plt.imshow(system, cmap=plt.cm.afmhot)
        #plt.title(i)
        #plt.plot(v,flux[i,:])
        #plt.pause(0.5)
        #plt.close()
    #plt.imshow(flux, cmap=plt.cm.afmhot)
    return flux

def get_phase():
    path = r'/home/member/a/aishling/Project/ccf'
    filelist = glob.glob(os.path.join(path, '*A.*') )
    phase0 = np.zeros(len(filelist))
    phase=np.zeros(len(filelist))
    data=np.zeros((len(filelist),161))
    i = 0
    T = 2454279.436714
    P = 2.21857567

    for infile in filelist:
        hdulist = fits.open(infile)
        MJD = hdulist[0].header['mjd-obs']
        Date = (MJD + 2400000.5)
        phase0[i] = (Date - T)/P
        OrbitNo = round(phase0[i],0)
        phase[i] = phase0[i] - OrbitNo

        v1=hdulist[0].header['CRVAL1']
        v2=hdulist[0].header['CDELT1']
        v3=hdulist[0].header['NAXIS1']
        v_data=v1+v2*np.arange(v3)+2.55

        im=hdulist[0].data
        spec=im[-1,:]
        spec=spec/np.median(spec[((v_data<-12)|(v_data>12))])
        data[i,:]=spec
        hdulist.close()
        i += 1
    plt.imshow(data)
    return phase, data, v_data

def mcmc(parameters,stepsize,phase,data,v):
    constant=[0,1,2,3,4,5]
    constant[0]=1.           #8.84aRstar
    constant[1]=85.71*np.pi/180.   #incl
    constant[2]=500               #Rstar
    constant[3]=512              #centre
    constant[4]=0.816                   #u1
    constant[5]=0                       #u2

    nstep=200
    npars=np.size(parameters)
    error=0.00015
    u1=0.3
    u2=0.3
    #phase=0.1
    chain=np.zeros((npars,nstep))
    chi_squared=np.zeros(nstep)
    chain[:,0]=parameters

    model=planet_motion(constant[0],constant[1],constant[2],phase,0.001,constant[3],v,parameters[2],parameters[3])
    flux=data

    chisq=np.sum((flux-model)**2/error**2 )
    chi_squared[0]=chisq

    for i in range(1,nstep):
        ##Make a random jump in parameter space
        new_parameters=chain[:,i-1]+np.random.normal(0.,1.,npars)*stepsize
        model=planet_motion(constant[0],constant[1],constant[2],phase,0.001,constant[3],new_parameters[4],new_parameters[2],new_parameters[3])
        nchisq=np.sum((flux-model)**2/error**2 )
        dchi=nchisq-chisq
        ## Going to lower chi squared: Accept new value
        if dchi <=0.:
            chain[:,i]=new_parameters
            chisq=nchisq
        else:
            rn=np.random.uniform()  ##There is still a chance to go to a higher chi squared...
            p=np.exp(-(dchi)/2.)
            if rn <= p:
                chain[:,i]=new_parameters   #accepted higher chi squared, move to new position
                chisq=nchisq
            else:
                chain[:,i]=chain[:,i-1]  ## rejected. Stay at current point
        chi_squared[i]=chisq
    print(new_parameters)
    return chain,chi_squared

Rp_Rstar=0.151
lamda=-0.85*np.pi/180. ##radians
amplitude=0.9
line_width=2.7
vsini=3.5

e_Rp_Rstar=0.00012
e_lamda=0.28
e_amplitude=0.02
e_line_width=0.02
e_vsini=0.2
stepsize=[e_Rp_Rstar,e_lamda,e_amplitude,e_line_width,e_vsini]
phase,data,v=get_phase()
parameters=[Rp_Rstar,lamda,amplitude,line_width,vsini]
mcmc(parameters,stepsize,phase,data,v)


# -*- coding: utf-8 -*-
"""
Created on Tue May 28 14:07:14 2019

@author: Ebi
"""
from keras.datasets import mnist

def plot_history(net_history):
    history = net_history.history
    import matplotlib.pyplot as plt
    losses = history['loss']
    val_losses = history['val_loss']
    accuracies = history['acc']
    val_accuracies = history['val_acc']

    plt.xlabel('Epochs')
    plt.ylabel('Loss')
    plt.plot(losses)
    plt.plot(val_losses)
    plt.legend(['loss', 'val_loss'])

    plt.figure()
    plt.xlabel('Epochs')
    plt.ylabel('Accuracy')
    plt.plot(accuracies)
    plt.plot(val_accuracies)
    plt.legend(['acc', 'val_acc'])

# Load data
(train_images, train_labels), (test_images, test_labels) = mnist.load_data()

# Data attributes
print("train_images dimentions: ", train_images.ndim)
print("train_images shape: ", train_images.shape)
print("train_images type: ", train_images.dtype)

X_train = train_images.reshape(60000, 784)
X_test = test_images.reshape(10000, 784)

X_train = X_train.astype('float32')
X_test = X_test.astype('float32')

X_train /= 255
X_test /= 255

from keras.utils import np_utils
Y_train = np_utils.to_categorical(train_labels)
Y_test = np_utils.to_categorical(test_labels)

#==================================================
# Creating our model
from keras.models import Sequential
from keras.layers import Dense, Dropout
from keras.optimizers import SGD
from keras.losses import categorical_crossentropy

myModel = Sequential()
myModel.add(Dense(500, activation='relu', input_shape=(784,)))
myModel.add(Dropout(20))
myModel.add(Dense(100, activation='relu'))
myModel.add(Dropout(20))
myModel.add(Dense(10, activation='softmax'))

myModel.summary()
myModel.compile(optimizer=SGD(lr=0.001), loss=categorical_crossentropy, metrics=['accuracy'])

#==================================================
# Train our model
network_history = myModel.fit(X_train, Y_train, batch_size=128, epochs=20, validation_split=0.2)
plot_history(network_history)

# Evaluation
test_loss, test_acc = myModel.evaluate(X_test, Y_test)
test_labels_p = myModel.predict(X_test)
import numpy as np
test_labels_p = np.argmax(test_labels_p, axis=1)
preds = myModel.predict_classes(X_test)
print('test_labels:')
print(test_labels)
print('preds:')
print(preds)

# Change layers config
myModel.layers[0].name = 'Layer_0'
myModel.layers[0].trainable = False
myModel.layers[0].get_config()


import re
import string
import nltk
from nltk.corpus import stopwords

def remove_url(text):
    url = re.compile(r'https?://\S+|www\.\S+')
    return url.sub(r'', text)

def remove_emoji(text):
    emoji_pattern = re.compile(
        '['
        u'\U0001F600-\U0001F64F'  # emoticons
        u'\U0001F300-\U0001F5FF'  # symbols & pictographs
        u'\U0001F680-\U0001F6FF'  # transport & map symbols
        u'\U0001F1E0-\U0001F1FF'  # flags (iOS)
        u'\U00002702-\U000027B0'
        u'\U000024C2-\U0001F251'
        ']+',
        flags=re.UNICODE)
    return emoji_pattern.sub(r'', text)

def remove_html(text):
    html = re.compile(r'<.*?>|&([a-z0-9]+|#[0-9]{1,6}|#x[0-9a-f]{1,6});')
    return re.sub(html, '', text)

import string

# Special thanks to https://www.kaggle.com/tanulsingh077 for this function
def clean_text(text):
    '''Make text lowercase, remove text in square brackets,remove links,remove punctuation
    and remove words containing numbers.'''
    text = str(text).lower()
    text = re.sub('\[.*?\]', '', text)
    text = re.sub(
        'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+',
        '',
        text
    )
    text = re.sub('https?://\S+|www\.\S+', '', text)
    text = re.sub('<.*?>+', '', text)
    text = re.sub('[%s]' % re.escape(string.punctuation), '', text)
    text = re.sub('\n', '', text)
    text = re.sub('\w*\d\w*', '', text)

    text = remove_url(text)
    text = remove_emoji(text)
    text = remove_html(text)

    return text

def stemming(text):
    stop_words = stopwords.words('english')
    more_stopwords = ['u', 'im', 'c']
    stop_words = stop_words + more_stopwords

    stemmer = nltk.SnowballStemmer("english")

    # Clean puntuation, urls, and so on
    text = clean_text(text)
    # Remove stopwords and Stemm all the words in the sentence
    text = ' '.join(stemmer.stem(word) for word in text.split(' ') if word not in stop_words)

    return text

# create a function that apply all the functions above
def preprocess_data(text):
    text = remove_url(text)
    text = remove_emoji(text)
    text = remove_html(text)
    text = clean_text(text)
    text = stemming(text)
    return text


"""
We want to test community structure capabilities of SBMs sampled from EAs.  We need the
SBMs such that each node is guaranteed a minimum degree -- so that nodes are unlikely to
be missed by any reasonable window size.  (Otherwise AMI can be a problem when comparing
to systems with missing nodes.)  We do this by merging a SBM with a random graph where all
nodes have minimum degree.

@author: Rich V. Field, Jeremy D. Wendt, Arvind Prasadan

"""

import typing
import networkx as nx
import random
import numpy as np

def min_degree_sbm(num_nodes: int, num_blocks: int, p_in: float, p_out: float,
                   min_degree: int, seed: int=0) -> nx.Graph:
    """
    This function creates a minimum degree SBM approximating the input parameters.  This
    requries that num_nodes / num_blocks is an integer division result.

    Note that when min_degree was 2, this failed on the assert for actual_degrees.  I
    don't know why, so we may need to remove that assert.
    """
    # First make the random graph with minimum-degree:
    degree_list = [min_degree for i in range(num_nodes)]
    rnd_G = nx.configuration_model(degree_list, seed=seed)

    # The returned graph is a multigraph, which may have parallel edges.  To remove any
    # parallel edges from the returned graph...
    rnd_G = nx.Graph(rnd_G)

    # check
    actual_degrees = [d for v, d in rnd_G.degree()]
    for i in range(len(actual_degrees)):
        if actual_degrees[i] != degree_list[i]:
            print('At %d, %d != %d' % (i, actual_degrees[i], degree_list[i]))
    assert actual_degrees == degree_list, 'Random graph generator failed'

    # Make the SBM
    num_nodes_per_block = int(num_nodes / num_blocks)
    assert num_nodes_per_block * num_blocks == num_nodes, 'Input number of nodes not '\
                                                         'divisible by number of blocks'

    sizes = [num_nodes_per_block for b in range(num_blocks)]
    probs = p_out * np.ones((num_blocks, num_blocks))
    for i in range(num_blocks):
        probs[i][i] = p_in

    SBM_G = nx.stochastic_block_model(sizes, probs, seed=seed)

    # Now merge them and return
    return merge_graphs(SBM_G, rnd_G)

def merge_graphs(g1: nx.Graph, g2: nx.Graph) -> nx.Graph:
    """
    A simple helper function that merges two graphs together.  Only used by min_degree_sbm
    for now, but could change over time.  By "merge", we mean that all edges in both
    input graphs are combined into a returned graph
    """
    edge_list_merged = list(set(g1.edges).union(set(g2.edges)))
    return nx.Graph(edge_list_merged)

def reorder_nodes(graph: nx.Graph, seed: int=0) -> (nx.Graph, typing.Dict):
    """
    This helper re-names all the nodes in the input graph to a random other set of names.
    This is necessary for our test so that we can use the same node set but have two
    distinct and disjoint community detection results.
    """
    # First, create the random reordering for the nodes
    num_nodes = graph.number_of_nodes()
    ordered = [i for i in range(num_nodes)]
    random.shuffle(ordered)

    # Now, rename all the nodes and edges
    mapping = {}
    for i, val in enumerate(ordered):
        mapping[i] = val
    return nx.relabel_nodes(graph, mapping, copy=True), mapping

def partition_ids_sbm(num_nodes: int, num_blocks: int, mapping: typing.Dict=None) -> typing.List:
    """
    This helper creates the correct answer partitions for an SBM with the input
    parameters.  If mapping isn't None, this correctly fixes the SBM assignments for the
    mapping.  It's assumed that mapping came from reorder_nodes.
    """
    ids = [int((i / num_nodes) * num_blocks) for i in range(num_nodes)]
    if mapping is not None:
        tmp = [ids[i] for i in range(num_nodes)]
        for key, value in mapping.items():
            tmp[value] = ids[key]
        ids = tmp

    return ids


from haystack.forms import FacetedSearchForm
from haystack.query import SearchQuerySet

class FacetedAllSearchForm(FacetedSearchForm):

    def __init__(self, *args, **kwargs):
        self.selected_date_facets = kwargs.pop("selected_date_facets", [])
        super(FacetedAllSearchForm, self).__init__(*args, **kwargs)

    def no_query_found(self):
        if self.searchqueryset:
            return self.searchqueryset
        sqs =  SearchQuerySet()
        if self.load_all:
            sqs = sqs.load_all()
        return sqs

    def search(self):
        sqs = super(FacetedAllSearchForm, self).search()
        for facet in self.selected_date_facets:
            sqs = sqs.narrow(facet)
        return sqs


from pymatgen.io.cif import CifParser
from urllib.request import urlopen
import pandas as pd
from pymatgen.ext.matproj import MPRester
from pymatgen.ext.matproj import MPRestError
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np

def fc(Rij, iRc, fRc):
    if Rij <= fRc and Rij > iRc:
        return 1
    else:
        return 0

def distance(a, b):
    return np.sqrt((a[0]-b[0])**2+(a[1]-b[1])**2+(a[2]-b[2])**2)

def G1(i, eta, Rs, iRc, fRc, structure):
    sum = 0
    neighbors = structure.get_neighbors(site=structure[i], r=fRc)
    ai = structure[i].species.elements[0].symbol
    for j in range(len(neighbors)):
        aj = neighbors[j].species.elements[0].symbol
        Rij = distance(structure[i].coords, neighbors[j].coords)
        sum += np.exp(-eta*(Rij-Rs)**2)*fc(Rij, iRc, fRc)
    return sum/len(structure)

def G2(i, eta, zeta, Rs, iRc, fRc, structure):
    sum = 0
    neighbors = structure.get_neighbors(site=structure[i], r=fRc)
    # ai = structure[i].species.elements[0].symbol
    for j in range(len(neighbors)):
        # aj = neighbors[j].species.elements[0].symbol
        Rij = distance(structure[i].coords, neighbors[j].coords)
        sum += np.exp(-eta*(Rij-Rs)**2)*fc(Rij, iRc, fRc)*np.exp(-zeta *
                                                                 np.abs(neighbors[j].species.elements[0].Z-structure[i].species.elements[0].Z))
    return sum/len(structure)

def descriptors(cif):
    parser = CifParser.from_string(cif)

    structure = parser.get_structures()
    structure = structure[0]

    # bonds_df = pd.read_csv('bonds.dat', sep='\s+')

    # For dataset_A.csv
    # G1_parameters = {"eta": [2, 4, 6], "Rs": [0, 2, 4], "Rc": [
    #     [0., 2], [2, 4], [4, 6]]}

    # For dataset_B.csv
    G1_parameters = {"eta": [1, 2, 3, 4, 5, 6], "Rs": [0, 1, 2, 3, 4], "Rc": [
        [0., 2], [2, 3], [3, 4], [4, 6]]}

    # For dataset_A.csv
    # G2_parameters = {"eta": [2, 4, 6], "zeta": [2, 4], "Rs": [0, 2, 4], "Rc": [
    #     [0., 2], [2, 4], [4, 6]]}

    # For dataset_B.csv
    G2_parameters = {"eta": [1, 2, 3, 4, 5, 6], "zeta": [1, 2, 3, 4], "Rs": [0, 1, 2, 3, 4], "Rc": [
        [0., 2], [2, 3], [3, 4], [4, 6]]}

    G1_descriptors = []
    G2_descriptors = []

    for eta in G1_parameters["eta"]:
        for Rs in G1_parameters["Rs"]:
            for Rc in G1_parameters["Rc"]:
                G = 0
                for i in range(len(structure)):
                    G += G1(i, eta, Rs, Rc[0], Rc[1], structure)
                G1_descriptors += [G]

    for eta in G2_parameters["eta"]:
        for zeta in G2_parameters["zeta"]:
            for Rs in G2_parameters["Rs"]:
                for Rc in G2_parameters["Rc"]:
                    G = 0
                    for i in range(len(structure)):
                        G += G2(i, eta, zeta, Rs, Rc[0], Rc[1], structure)
                    G2_descriptors += [G]

    descriptors_list = G1_descriptors + G2_descriptors
    # descriptors_list = G2_descriptors

    return descriptors_list