import os
import regex
import concurrent
import pickle
import numpy as np

import torch.nn as nn

from collage.utils import translate, orf_to_coded, prot_to_coded
from collage.tensorize import record_slice_to_arrays, tensorize_batch

def reverse_complement(sequence: str) -> str:
    temp_seq = sequence.upper()
    temp_seq = temp_seq.replace('A', 't').replace(
        'T', 'a').replace('C', 'g').replace('G', 'c')
    # temp_seq = temp_seq.replace('K','m').replace('M','k').replace('R','y').replace('Y','r')
    # temp_seq = temp_seq.replace('B','v').replace('V','b').replace('H','d').replace('D','h')
    return temp_seq.upper()[::-1]

def orf_searcher(dna_sequence: str,
                 min_length: int = 30,
                 both_strands: bool = False,
                 longest_only: bool = True,
                 sequence_name: str = 'Sequence',
                 dir=None) -> list:
    '''
    Generate a list of records describing potential ORFs in a given DNA sequence
    '''

    ORF_search = regex.compile('ATG(?:...){2,}?(?:TAG|TAA|TGA|TRA|TAR)')

    records = []
    for strand in range(both_strands + 1):
        if strand == 0:  # For strand
            sequence = dna_sequence
            strand_label = '+'
        else:           # Rev strand
            sequence = reverse_complement(dna_sequence)
            strand_label = '-'

        # if sequence == 303030303030:
        #    print( sequence_name )
        if type(sequence) != type(''):
            print(sequence, type(sequence))
        orfs = ORF_search.findall(sequence, overlapped=True)

        for orf in set(orfs):

            protein = translate(orf)
            if protein.count('.') != 1:
                continue  # Limit to 1 stop codon
            if len(protein) <= min_length:
                continue  # Ensure # of AAs >= min length

            if strand_label == '+':
                start_pos = dna_sequence.find(orf) + 1
            else:
                start_pos = dna_sequence.find(reverse_complement(orf)) + 1
            orf_length = len(orf)
            end_pos = start_pos + orf_length - 1

            record = {'Name': sequence_name,
                      'ORF': orf,
                      'ORF_coded': orf_to_coded(orf, True),
                      'Translation': protein,
                      'Translation_coded': prot_to_coded(protein),
                      'Start_position': start_pos,
                      'End_position': end_pos,
                      'Strand': strand_label,
                      'Length': len(protein),
                      'n': 1,
                      'Codon_weights': [1.0] * orf_length}  # Temporary patch on Codon_weights, will likely delete later
            records.append(record)

    # Prune records of internal ORFs
    if longest_only:
        longest_orfs = dict(
            [((r['Strand'], r['End_position']), {'Length': 0}) for r in records])
        for record in records:
            key = (record['Strand'], record['End_position'])
            # print( record[ 'Length' ], longest_orfs[ key ] )
            if record['Length'] > longest_orfs[key]['Length']:
                longest_orfs[key] = record
        records = list(longest_orfs.values())

    if not dir:
        return records
    else:
        # Dump the records to the provided directory
        out_filename = sequence_name.replace('/', '_').replace('\\', '_') + '.pkl'
        out_path = os.path.join(dir, out_filename)
        pickle.dump(records, open(out_path, 'wb'))
        return None

def library_to_orf_records(sequence_library: dict,
                           unique_orf_only: bool = False,
                           unique_prot_only: bool = False,
                           min_length: int = 30,
                           both_strands: bool = False,
                           longest_only: bool = True,
                           executor=None) -> list:
    '''
    Process a DNA library to ORF records
    '''

    if unique_prot_only:
        unique_orf_only = True  # Force unique ORF identifier before doing unique protein search

    if not executor:
        orf_records = []
        for name in sequence_library:
            orf_records += orf_searcher(dna_sequence=sequence_library[name],
                                        min_length=min_length,
                                        both_strands=both_strands,
                                        longest_only=longest_only,
                                        sequence_name=name)
    else:
        # library_elements = list( sequence_library.items() )
        # orf_records = [ ]
        # library_items = list( sequence_library.items() )
        # for i in range( 0, len( library_items ), executor_batch_size):
        #    print( i )
        #    batch = list( library_items[ i : i + executor_batch_size ] )
        # temp_dir = 'tmp' + str( round(time.time()) )
        # os.mkdir( temp_dir )
        futures = [executor.submit(orf_searcher,
                                   sequence,
                                   min_length,
                                   both_strands,
                                   longest_only,
                                   name) for name, sequence in sequence_library.items()]
        concurrent.futures.wait(futures)
        # print( 'ORFs processed' )
        # record_files = os.listdir( temp_dir )
        # record_sets = [ pickle.load( open( os.path.join( temp_dir, f ), 'rb') ) for f in record_files ]
        # print( 'Files read' )
        # orf_records = [ x for x in record_sets ]
        # orf_records = [ x for f in record_files
        #                for x in pickle.load( open( os.path.join( temp_dir, f ), 'rb') ) ]

        # shutil.rmtree( temp_dir )

        # results = [ f.result() for f in futures ]
        # orf_records = [ x for x in results ]
        # print( results )
        orf_records = [x for f in futures for x in f.result()]

    if unique_orf_only:
        unique_orfs = dict([(r['ORF'], []) for r in orf_records])
        for r in orf_records:
            unique_orfs[r['ORF']].append(r)
        new_records = []
        for records in unique_orfs.values():
            # print( records )
            total_n = np.sum([r['n'] for r in records])
            n_per_strand = dict([(s, np.sum([r['Strand'] == s for r in records])) for s in ['+', '-']])
            modal_strand = '-' if n_per_strand['-'] > n_per_strand['+'] else '+'
            # print( modal_strand )
            starts = [r['Start_position']
                      for r in records if r['Strand'] == modal_strand]
            start_to_n = dict([(s, starts.count(s))
                              for s in sorted(set(starts))])
            start_to_n = dict(sorted(start_to_n.items(), key=lambda x: x[1], reverse=True))
            modal_start = list(start_to_n)[0]
            # print( modal_start )
            modal_records = [r for r in records if r['Strand'] == modal_strand and r['Start_position'] == modal_start]
            modal_record = dict(modal_records[0])
            modal_record['n'] = total_n
            new_records.append(modal_record)
        orf_records = new_records

    return orf_records

# Draft batch function for prediciton (which is a bit different from training)

def library_scorer(model: nn.Module,
                   orf_records: list,
                   max_context: int = 500,
                   max_sites_per_batch: int = 2000,
                   device: str = 'cpu') -> list:
    '''
    Update ORF records to include CoLLAGE site scores
    '''
    nllloss = nn.NLLLoss(reduction='none')

    # Sort records based on length, longest first, work on copy
    orf_records = sorted(
        orf_records, key=lambda x: x['Length'], reverse=True)

    # Loop over records to create batches with orf indices
    batches = []
    running_batch = []
    n_records = len(orf_records)
    for i, record in enumerate(orf_records):
        orf_records[i]['CoLLAGE_scores'] = []
        if record['Length'] <= max_context:
            # Add elements to running_batch with appropriate padding
            if len(running_batch) == 0:
                # Adding the first entry, and since records are sorted,
                # this will always be the longest for the batch
                seq_len_for_batch = record['Length']
                n_sequences_per_batch = int(max_sites_per_batch / seq_len_for_batch)
                last_idx_in_batch = min([n_records - 1, i + n_sequences_per_batch - 1])
            array = record_slice_to_arrays(record, 0, record['Length'], seq_len_for_batch, i)
            running_batch.append(array)
            if i == last_idx_in_batch:
                batches.append(tensorize_batch(running_batch))
                running_batch = []

        else:
            # Sequence will need to be sequentially processed
            # Number of predictions = Length - max_content + 1
            # E.g. 501 residues with max_content = 500 will take 2 predictions
            # Don't bother adding to running_batch

            n_predictions = record['Length'] - max_context + 1

            sliced_arrays = [record_slice_to_arrays(record, s, s + max_context, max_context, i)
                             for s in range(n_predictions)]

            n_slices_per_batch = int(max_sites_per_batch / max_context)

            for k in range(0, len(sliced_arrays), n_slices_per_batch):
                batch = sliced_arrays[k: k + n_slices_per_batch]
                batches.append(tensorize_batch(batch))

    # Run CoLLAGE model on the batches
    model.eval()
    for batch in batches:
        prot, cds, weight, gene = [x.to(device) for x in batch]
        targets = cds[:, 1:]
        pred = model(prot, cds[:, :-1]).transpose(1, -1)
        logLs = -nllloss(pred, targets).detach().numpy()

        orf_idxs = gene.detach().flatten().numpy()
        for orf_idx, logL in zip(orf_idxs, logLs):
            if len(orf_records[orf_idx]['CoLLAGE_scores']) == 0:
                orf_records[orf_idx]['CoLLAGE_scores'] = list(logL)
            else:
                orf_records[orf_idx]['CoLLAGE_scores'].append(logL[-1])

    return orf_records


#!/usr/local/bin/python3
import argparse
import os
import sys
import tempfile

import getpass
import iterfzf
from datetime import datetime, timezone
from pathlib import Path

import git_pki.gpg_wrapper
from git_pki import __version__
from git_pki.custom_types import AddIdentityRequest, KeyChange, ImportRequest, RevokeIdentityRequest, Request
from git_pki.exceptions import Git_PKI_Exception
from git_pki.git_wrapper import Git
from git_pki.utils import file_exists, format_key, mkdir, read_multiline_string, sha1_encode
from git_pki import gpg_wrapper

class KeyChangeListener:
    def __init__(self, gpg):
        self.__gpg = gpg

    def key_added(self, path):
        print(f"A {path}")
        self.__gpg.import_public_key(path)

    def key_updated(self, path):
        print(f"M {path}")
        self.__gpg.import_public_key(path)

    def key_removed(self, path):
        print(f"R {path}")
        fingerprint = self.__gpg.file_key_fingerprint(path)
        self.__gpg.remove_public_key(fingerprint)

class GPKI:
    def __init__(self, home):
        self.__file_gpghome = mkdir(f"{home}/vault/private", 0o700)
        self.__file_repository = mkdir(f"{home}/vault/public")
        self.__review_dir = mkdir(f"{home}/reviews")
        self.__gpg = git_pki.gpg_wrapper.GnuPGHandler(self.__file_gpghome)
        self.__git = Git(self.__file_repository)

        listener = KeyChangeListener(self.__gpg)
        self.__git.update(listener)

    def generate_identity(self, name, email, description, passphrase=None):
        existing_key = self.__gpg.private_key_fingerprint(name)
        if existing_key is not None:
            # If key exists, confirm removal of the private version and move public one to the archive
            response = input(f"Replace existing identity of {existing_key}? [yN] ")
            if response.lower() != "y":
                return
            self.revoke(priv_key_name=existing_key)

        fingerprint = self.__gpg.generate_key(name, email, description, passphrase=passphrase)
        if fingerprint is None:
            return

        key = self.__gpg.export_public_key(fingerprint)
        file = Path(f"{self.__git.identity_dir}/{name}/{fingerprint}")
        self.__export_key(key, Path(file))
        self.__git.push_branch(f"{name}/{fingerprint}", f"Publish key {name}/{fingerprint}")
        print(key)

    def revoke(self, priv_key_name=None):
        if priv_key_name is None:
            available_signatories = map(format_key, self.__gpg.private_keys_list())
            selection = iterfzf.iterfzf(available_signatories, prompt="Select private key to revoke ")
            to_revoke = None if selection is None else selection.split()[0]
            if to_revoke is None:
                return
            priv_key = self.__gpg.get_private_key_by_id(to_revoke)
        else:
            priv_key = self.__gpg.get_private_key_by_id(priv_key_name)

        passphrase = getpass.getpass(f"Specify passphrase for the existing key of [{priv_key.name}]: ")
        self.__gpg.remove_private_key(priv_key.fingerprint, passphrase)
        if input("Invalidate previous public key? [yN]\n").lower() == "y":  # remove this question, it's pointless to have public key when nobody has private key
            self.__gpg.remove_public_key(priv_key.fingerprint)
            ans = input("Specify expiration time in format YYYY-MM-DDTHH:mm:ss or leave empty to take current timestamp.")
            if ans == '':
                revocation_timestamp = datetime.now(timezone.utc)
            else:
                try:
                    revocation_timestamp = datetime.fromisoformat(ans)
                except ValueError:
                    Git_PKI_Exception(f"Unrecognized date: {ans}")
            revocation_timestamp = revocation_timestamp.replace(tzinfo=timezone.utc).strftime('%Y-%m-%d %H:%M:%S%z')

            # make RevokeIdentityRequest
            revoke_file = Path(f"{self.__git.identity_dir}/{priv_key.name}/{priv_key.fingerprint}_revoked")
            mkdir(revoke_file.parent)
            with open(revoke_file, 'w') as f:
                f.write(revocation_timestamp)
            self.__git.push_branch(f"{priv_key.name}/{priv_key.fingerprint}_revoked", f"Revoke key {priv_key.name}/{priv_key.fingerprint}")

    def list_signatories(self):
        print("fingerprint                              created-on expires-on\tidentity\temail\tdescription")
        for key in self.__gpg.private_keys_list():
            # TODO (#25): align the text correctly
            print(f"{key}")

    def list_recipients(self):
        print("fingerprint                              created-on expires-on\tidentity\temail\tdescription")
        for key in self.__gpg.public_keys_list():
            # TODO (#25): align the text correctly
            print(f"{key}")

    def encrypt(self, source, target, passphrase=None, select_all_recipients=False):
        if target is not None and os.path.isfile(target):
            if input(f"Target file already exist, do you want to overwrite? [yN] ").lower() != 'y':
                return
        available_recipients = list(map(format_key, self.__gpg.public_keys_list()))
        if select_all_recipients:
            selection = available_recipients
        else:
            selection = iterfzf.iterfzf(available_recipients, prompt="Select recipients (use tab to selected entry): ", multi=True)
            if selection is None:
                return

        recipients = [item.split()[0] for item in selection]

        available_signatories = map(format_key, self.__gpg.private_keys_list())
        selection = iterfzf.iterfzf(available_signatories, prompt="Select signatory or press ctrl+d to not sign ")
        signatory = None if selection is None else selection.split()[0]

        if not passphrase:
            passphrase = getpass.getpass(f"Specify passphrase for [{selection[0]}]: ")

        self.__gpg.encrypt(recipients, signatory, source, target, passphrase)

    def decrypt(self, source, target, passphrase=None, sync=False):
        if sync:
            self.sync()
        if file_exists(target):
            if input(f"Target file already exist, do you want to overwrite? [yN] ").lower() != 'y':
                return

        if source is None:
            data = []
            print("Paste message, then press enter and ctrl+d")
            for line in sys.stdin:
                data.append(line)
            source = "".join(data)
            recipients = self.__gpg.get_recipients_from_message(source)
        elif os.path.isfile(source):
            with open(source, 'rb') as src_file:
                recipients = self.__gpg.get_recipients_from_file(src_file)
        else:
            print(f"Specified source file: {source} was not found, aborting.")
            return

        # Check if we have at least one private key to decrypt message
        priv_key = None
        for rec in recipients[::-1]:
            priv_key = self.__gpg.get_private_key_by_id(rec)
            if priv_key is not None:
                break

        if priv_key is None:
            raise Git_PKI_Exception("Could not find private key to decrypt message. Are you correct recipient?")
        if passphrase is None:
            passphrase = getpass.getpass(f"Specify passphrase for {priv_key.name}: ")

        self.verify_message(source, passphrase, sync)
        self.__gpg.decrypt(source, target, passphrase)

    def verify_message(self, message, passphrase, synced):
        valid_key_list = []
        revoked_key_list = []
        signature_verification = self.__gpg.verify_signature(message, passphrase)
        for signature in signature_verification:
            key = self.__gpg.get_public_key_by_id(signature.signatory_fingerprint)
            if key is None:
                if synced:
                    raise Git_PKI_Exception("Could not verify message: signatory from outside organisation.")
                else:
                    self.sync()
                    key = self.__gpg.get_public_key_by_id(signature.signatory_fingerprint)
                    if key is None:
                        raise Git_PKI_Exception("Could not verify message: signatory from outside organisation.")

            if self.is_key_revoked(key) and self.get_revocation_time(key) < datetime.fromtimestamp(float(signature.timestamp), tz=timezone.utc):
                revoked_key_list.append(key)
            else:
                valid_key_list.append(key)

        if len(revoked_key_list) == 0:
            return
        elif len(valid_key_list) != 0:
            if input(f"Message signed with revoked keys: {' '.join([key.name for key in revoked_key_list])}. Proceed anyways? [yN]\n").lower() != 'y':
                raise Git_PKI_Exception("Operation aborted by user.")
        else:
            raise Git_PKI_Exception(
                "Could not decrypt message signed with revoked key and message was signed after revocation time.")

    def import_keys(self, files):
        if not files:
            print("Paste the key and then press ctrl+d on an empty line")
            data = read_multiline_string()
            file = tempfile.mkstemp()[1]
            with open(file, "w") as output:
                output.write(data)
            files = [file]

        fingerprints = []
        imported = False
        for file in files:
            for fingerprint in self.__import_key(file):
                fingerprints.append(fingerprint)
                name = self.__gpg.public_key_name(fingerprint)
                file = f"{self.__git.identity_dir}/{name}/{fingerprint}"
                key = self.__gpg.export_public_key(fingerprint)
                self.__export_key(key, Path(file))
                imported = True

        if not imported:
            return
        branch_name = f'import/{sha1_encode("".join(fingerprints))}'
        message = f"Import keys {', '.join(fingerprints)}"
        self.__git.push_branch(branch_name, message)

    def export_keys(self, names, target_file, mode=None):
        if file_exists(target_file):
            if not mode:
                choices = ['append', 'cancel', 'overwrite']
                mode = iterfzf.iterfzf(choices, prompt=f"Output file: {target_file} already exists, select: <append> to add keys to the file, <overwrite> to remove existing file, or <cancel> to abort")
            if mode == 'overwrite':
                os.remove(target_file)
            if mode == 'cancel':
                return
        for name in names:
            key = self.__gpg.export_public_key(name)
            if not key:
                print(f"{name}: Failed\n")
            else:
                print(f"{name}:\n{key}\n")
                if target_file:
                    self.__export_key(key, Path(target_file))
                    print(f"{name}: exported to file: {target_file}")

    @staticmethod
    def __export_key(key, path):
        mkdir(path.parent)
        with open(path, "a") as file:
            file.write(key)

    def __import_key(self, path):
        if not file_exists(path):
            print(f"File {path} not found, aborting.")
            return []
        keys_from_file = self.__gpg.scan(path)
        if not keys_from_file:
            print(f"File {path} does not contain any key, aborting.")
            return []
        print(f"File {path} contains:")
        for key in keys_from_file:
            print(f"{key.fingerprint} key of {key.name} valid between {key.created_on} and {key.expires_on}")
        if input("is that OK? [yN] ").lower() != 'y':
            return []

        with open(path, 'rb') as file:
            imported = self.__gpg.import_public_key(file.read())

        successfully_imported_keys, is_import_successful = self.__get_successfully_imported_keys_and_status(imported)
        self.__print_import_summary(keys_from_file, imported)

        if not is_import_successful:
            print("\nThere were errors while importing keys, reverting changes.")
            self.remove_keys(successfully_imported_keys)
            keys_to_load = {key for key in keys_from_file if key.fingerprint in successfully_imported_keys}
            self.__load_keys_from_git(keys_to_load)
            return []

        return successfully_imported_keys

    @staticmethod
    def __print_import_summary(keys, imported_status):
        key_statuses = list(zip(keys, imported_status))
        succeded = [item for item in key_statuses if item[1]['ok'] == '1']
        unchanged = [item for item in key_statuses if item[1]['ok'] == '0' and item[1]['text'].endswith("Not actually changed\n")]
        failed = [key for key in key_statuses if key not in succeded and key not in unchanged]
        print("Import Summary:")
        if failed:
            print("\nFailed:")
            for fail in failed:
                print(f"{fail[1]['fingerprint']}, reason:  {fail[1]['text'].replace('Not actually changed', '').strip()}")
        if unchanged:
            print("\nUnchanged:")
            for unch in unchanged:
                print(format_key(unch[0]))
        if succeded:
            print("\nSucceded:")
            for succ in succeded:
                print(format_key(succ[0]))
        print('\n')

    @staticmethod
    def __get_successfully_imported_keys_and_status(imported_keys):
        import_successful = True
        successfully_imported = []
        for key_status in imported_keys:
            reason = key_status["text"]
            if key_status["ok"] == '0':
                if reason == 'Not actually changed\n':
                    continue
                import_successful = False
            else:
                successfully_imported.append(key_status['fingerprint'].lower())
        return successfully_imported, import_successful

    def __load_keys_from_git(self, key_list):
        backup_files = [self.__git.path_to(f"identity/{key.name}/{key.fingerprint}") for key in key_list]
        for file in backup_files:
            if file_exists(file):
                with open(file, "rb") as f:
                    self.__gpg.import_public_key(f.read())

    def remove_keys(self, fingerprint_list):
        for fingerprint in fingerprint_list:
            self.__gpg.remove_public_key(fingerprint)

    def review_requests(self):
        git = self.__git

        def map_change(change):
            if change.op == 'A':
                path = reviewed.path_to(change.path)
                return KeyChange(added=list(self.__gpg.scan(path)), removed=[])
            if change.op == 'R':
                path = git.path_to(change.path)
                return KeyChange(added=[], removed=list(self.__gpg.scan(path)))
            if change.op == 'M':
                removed = git.path_to(change.path)
                added = reviewed.path_to(change.path)
                return KeyChange(added, removed)

        self.merge_revoked()

        unmerged = list(git.list_branches_unmerged_to_remote_counterpart_of(git.current_branch()))
        if not unmerged:
            return
        for i, request in enumerate(unmerged):
            print(f"{i}) {request.title}")

        try:
            selected = int(input(f"Select request to review (0-{len(unmerged)-1}): "))
        except ValueError:
            raise Git_PKI_Exception("Please pass the integer value to select request.")

        request = git.get_specified_request(unmerged[selected])
        if not git.is_mergeable_to(self.__git.master_branch, request.branch.full_name):
            print("Warning, cannot perform `git merge` automatically")

        changes = list(git.file_diff(request.branch.full_name))
        with git.open_worktree(self.__review_dir, request.branch.full_name) as reviewed:
            for change in map(map_change, changes):
                self.__run_checks(change, request, reviewed)
            print("Requested changes:")
            for change in map(map_change, changes):
                print(change)

        msg = 'Approve this changes? Answer "y" to merge them.'
        if input(msg).lower() != 'y':
            if input(f"\nDelete branch {request.branch} [yN] ?").lower() == 'y':
                git.remove_remote_branch(request.branch.name)
                print(f'\nSuccessfully deleted branch {request.branch}')
        else:
            self.merge_changes(request)

    def merge_changes(self, request):
        print(f'Merging branch {request.branch.name} into master...')
        self.__git.merge(request.branch.full_name)
        self.__git.push(self.__git.master_branch)
        self.__git.remove_remote_branch(request.branch.name)

    def merge_revoked(self):
        self.__git.fetch()
        self.__git.merge(f'origin/{self.__git.master_branch}')
        unmerged_branches = list(self.__git.list_branches_unmerged_to_remote_counterpart_of(self.__git.current_branch()))
        for branch in unmerged_branches:
            request = self.__git.get_specified_request(branch)
            if isinstance(request, RevokeIdentityRequest):
                if self.__git.is_mergeable_to(self.__git.master_branch, request.branch.full_name):
                    self.merge_changes(request)
                    couterbranch_candidate = request.branch.full_name.replace('_revoked', '')
                    try:
                        self.__git.merge(couterbranch_candidate)
                    except EnvironmentError:
                        pass
                else:
                    print(f"Error: Cannot automerge branch {request.branch.name}")

    def is_key_expired(self, key):
        if not key:
            return True
        if key.expires_on is None:
            return False
        return datetime.now() > datetime.strptime(key.expires_on, "%Y-%m-%d")

    def is_any_key_valid(self, path):
        for key in self.__gpg.scan(path):
            if not self.is_key_expired(key):
                return True
        return False

    def does_fingerprint_match_file(self, path):
        key = self.__gpg.scan(path)
        if not key:
            return False
        else:
            return str(path).endswith(key[0].fingerprint)

    def __run_checks(self, change, request, reviewed):
        if isinstance(request, ImportRequest):
            name = change.added[0].name if change.added else change.removed[0].name
            fingerprint = change.added[0].fingerprint if change.added else change.removed[0].fingerprint
        else:
            name = request.name
            fingerprint = request.fingerprint
        filepath = reviewed.path_to(os.path.join('identities', name, fingerprint))
        if not self.does_fingerprint_match_file(filepath):
            raise Git_PKI_Exception('File name and fingerprint are no equal.')
        if not self.is_any_key_valid(filepath):
            raise Git_PKI_Exception(f"The file {filepath} does not contain any valid key, aborting.")

    def is_key_revoked(self, key):
        return os.path.isfile(self.__git.path_to(os.path.join('identities', key.name, key.fingerprint + '_revoked')))

    def get_revocation_time(self, key):
        path = self.__git.path_to(os.path.join('identities', key.name, key.fingerprint + '_revoked'))
        if os.path.isfile(path):
            with open(path, 'r') as revoke_file:
                return datetime.strptime(revoke_file.read().strip(), '%Y-%m-%d %H:%M:%S%z')
        else:
            return datetime.strptime('2000-01-01 00:00:00+00:00', '%Y-%m-%d %H:%M:%S%z')

    def sync(self, keep_rejected_keys=False):
        self.__git.fetch(prune=True)
        self.__git.merge(f'origin/{self.__git.master_branch}')

        for root, dirs, files in os.walk(os.path.join(self.__git.root_dir, 'identities')):
            for file in files:
                if "revoke" in file:
                    continue
                identity_path = os.path.join(root, file)
                with open(identity_path, "rb") as f:
                    self.__gpg.import_public_key(f.read())
                if gpg_wrapper.verbose:
                    print(f"Loaded key: name: {os.path.dirname(identity_path).split('/')[-1]}, fingerprint: {file}")
        print("Successfully loaded all valid keys.")

        remote_branches = [rbranch.replace('origin/', '') for rbranch in self.__git.get_remote_branches()]
        branches_to_check = [lbranch for lbranch in self.__git.get_local_branches() if lbranch not in remote_branches]
        for branch in branches_to_check:
            if not self.__git.is_merged_to(self.__git.master_branch, branch):
                if not keep_rejected_keys:
                    self.__revert_pr(branch)
                    self.__git.remove_local_branch(branch)

    def __revert_pr(self, branch):
        request = self.__git.get_specified_request(Request(branch, self.__git.commit_title(branch)))
        if isinstance(request, RevokeIdentityRequest):
            return
        else:
            with self.__git.open_worktree(self.__review_dir, request.branch.name) as work_tree:
                if isinstance(request, AddIdentityRequest):
                    keys_to_remove = self.__gpg.scan(work_tree.path_to(os.path.join('identities', request.name, request.fingerprint)))
                    passphrase = getpass.getpass(f"Specify passphrase to remove existing key of [{keys_to_remove[0].name}]: ")
                    self.__gpg.remove_public_key(keys_to_remove[0].fingerprint)
                    self.__gpg.remove_private_key(keys_to_remove[0].fingerprint, passphrase)
                else:
                    keys_to_remove = []
                    for root, _, files in os.walk(work_tree.path_to(os.path.join('identities'))):
                        for file in files:
                            if file in request.fingerprints:
                                keys_to_remove.extend(self.__gpg.scan(os.path.join(root, file)))
                    for key in keys_to_remove:
                        self.__gpg.remove_public_key(key.fingerprint)

def create_gpki_parser():
    common_args_parser = argparse.ArgumentParser(
        prog='git pki', argument_default=argparse.SUPPRESS, add_help=False)
    common_args_parser.add_argument('-h', '--help')
    common_args_parser.add_argument(
        '--version', '-v', action='version', version=f'%(prog)s version {__version__}')
    common_args_parser.add_argument('--verbose', action='store_true', default=False)

    cli_parser = argparse.ArgumentParser(
        prog='git pki',
        argument_default=argparse.SUPPRESS,
        add_help=False,
        parents=[common_args_parser]
    )

    subparsers = cli_parser.add_subparsers(dest='command')

    encrypt_parser = subparsers.add_parser(
        'encrypt',
        argument_default=argparse.SUPPRESS,
        usage=argparse.SUPPRESS,
        add_help=False,
        parents=[common_args_parser])
    encrypt_parser.add_argument('--input', '-i', default=None)
    encrypt_parser.add_argument('--output', '-o', default=None)
    encrypt_parser.add_argument('--password', '-p', default=None)
    encrypt_parser.add_argument('--all', '-a', action='store_true', default=False)

    decrypt_parser = subparsers.add_parser(
        'decrypt',
        argument_default=argparse.SUPPRESS,
        usage=argparse.SUPPRESS,
        add_help=False,
        parents=[common_args_parser])
    decrypt_parser.add_argument('--input', '-i', default=None)
    decrypt_parser.add_argument('--output', '-o', default=None)
    decrypt_parser.add_argument('--password', '-p', default=None)
    decrypt_parser.add_argument('--sync', '-s', action='store_true', default=False)

    new_identity_parser = subparsers.add_parser(
        'identity',
        argument_default=argparse.SUPPRESS,
        usage=argparse.SUPPRESS,
        add_help=False,
        parents=[common_args_parser])
    new_identity_parser.add_argument('name', nargs='?')
    new_identity_parser.add_argument('--email', default=None)
    new_identity_parser.add_argument('--description', default=None)

    import_key_parser = subparsers.add_parser(
        'import',
        argument_default=argparse.SUPPRESS,
        usage=argparse.SUPPRESS,
        add_help=False,
        parents=[common_args_parser])
    import_key_parser.add_argument('--input', '-i', nargs='+', default=None)

    export_key_parser = subparsers.add_parser(
        'export',
        argument_default=argparse.SUPPRESS,
        usage=argparse.SUPPRESS,
        add_help=False,
        parents=[common_args_parser])
    export_key_parser.add_argument('key_names', nargs='+')
    export_key_parser.add_argument('--output', '-o', default=None)

    subparsers.add_parser(
        'recipients',
        argument_default=argparse.SUPPRESS,
        usage=argparse.SUPPRESS,
        add_help=False,
        parents=[common_args_parser])

    subparsers.add_parser(
        'signatories',
        argument_default=argparse.SUPPRESS,
        usage=argparse.SUPPRESS,
        add_help=False,
        parents=[common_args_parser])

    subparsers.add_parser(
        'review',
        argument_default=argparse.SUPPRESS,
        usage=argparse.SUPPRESS,
        add_help=False,
        parents=[common_args_parser])

    sync_parser = subparsers.add_parser(
        'sync',
        argument_default=argparse.SUPPRESS,
        usage=argparse.SUPPRESS,
        add_help=False,
        parents=[common_args_parser])
    sync_parser.add_argument('--keep-rejected-keys', action='store_true', default=False)

    subparsers.add_parser(
        'revoke',
        argument_default=argparse.SUPPRESS,
        usage=argparse.SUPPRESS,
        add_help=False,
        parents=[common_args_parser])

    return cli_parser

def launch(parsed_cli):
    gpg_wrapper.verbose = parsed_cli.verbose

    gpki = GPKI("/tmp/foobarbaz")
    cmd = parsed_cli.command

    if cmd == 'decrypt':
        gpki.decrypt(parsed_cli.input, parsed_cli.output, parsed_cli.password, parsed_cli.sync)
    elif cmd == 'encrypt':
        gpki.encrypt(parsed_cli.input, parsed_cli.output, parsed_cli.password, parsed_cli.all)
    elif cmd == 'identity':
        if 'name' not in parsed_cli:
            raise Git_PKI_Exception("Name is mandatory while creating new identity.")
        gpki.generate_identity(parsed_cli.name, parsed_cli.email, parsed_cli.description)
    elif cmd == 'import':
        gpki.import_keys(parsed_cli.input)
    elif cmd == 'export':
        gpki.export_keys(parsed_cli.key_names, parsed_cli.output)
    elif cmd == 'recipients':
        gpki.list_recipients()
    elif cmd == 'signatories':
        gpki.list_signatories()
    elif cmd == 'review':
        gpki.review_requests()
    elif cmd == 'sync':
        gpki.sync(parsed_cli.keep_rejected_keys)
    elif cmd == 'revoke':
        gpki.revoke()
    else:
        # Some help should be printed here
        raise Git_PKI_Exception(f"Command {cmd} is not a valid git-pki command.")

def main():
    args = sys.argv[1:]
    cli_parser: argparse.ArgumentParser = create_gpki_parser()
    parsed_cli = cli_parser.parse_args(args)
    launch(parsed_cli)

if __name__ == "__main__":
    main()


import sys
import re
import os.path
from pcapfile import savefile
import xlwt
import matplotlib.pyplot as plt
from datetime import datetime
from PIL import Image

def graphFiltered(data, ipToSearch):

    # SRC PART
    tabSrc = {}
    for result in data:
        if result[0] == ipToSearch:
            tabSrc[result[1]] = result[2]

    if any(tabSrc):
        ipsSrc = []
        numberSrc = []
        for key, value in tabSrc.items():
            ipsSrc.append(key)
            numberSrc.append(value)
    else:
        print("Aucune donnée en IP Source")

    # DST PART
    tabDst = {}
    for result in data:
        if result[1] == ipToSearch:
            tabDst[result[0]] = result[2]

    if any(tabDst):
        ipsDst = []
        numberDst = []
        for key, value in tabDst.items():
            ipsDst.append(key)
            numberDst.append(value)

    else:
        print("Aucune donnée en IP Destination")

    ### DISPLAY
    if any(tabSrc) and any(tabDst):
        fig, (ax1, ax2) = plt.subplots(1, 2)
        fig.tight_layout(pad=3.0)

        ax1.bar(ipsSrc, numberSrc, color = 'pink')
        ax1.set_ylabel('Occurences')
        ax1.set_title('IP as Source')

        ax2.bar(ipsDst, numberDst, color = 'red')
        ax2.set_ylabel('Occurences')
        ax2.set_title('IP as Destination')

    elif len(tabSrc) > 0 and len(tabDst) <= 0:
        fig, ax1 = plt.subplots()

        ax1.bar(ipsSrc, numberSrc, color = 'pink')
        ax1.set_ylabel('Occurences')
        ax1.set_title('IP as Source')

    elif len(tabDst) > 0 and len(tabSrc) <= 0:
        fig, ax2 = plt.subplots()

        ax2.bar(ipsDst, numberDst, color = 'red')
        ax2.set_ylabel('Occurences')
        ax2.set_title('IP as Destination')

    plt.savefig('filteredGraphs.png', dpi=200)

    plt.show()

def countSrc(data):
    numberSrc = 0
    tab = {}
    for x in data:
        ipSrc = x[0]
        occurences = x[2]
        if ipSrc in tab:
            oldValue = tab[ipSrc]
            tab[ipSrc] = oldValue + occurences
        else:
            tab[ipSrc] = occurences

    return(tab)

def countDst(data):
    numberDst = 0
    tab = {}
    for x in data:
        ipDst = x[1]
        occurences = x[2]
        if ipDst in tab:
            oldValue = tab[ipDst]
            tab[ipDst] = oldValue + occurences
        else:
            tab[ipDst] = occurences

    return(tab)

def autopct_format(values):
    def my_format(pct):
        total = sum(values)
        val = int(round(pct*total/100.0))
        return '{v:d}'.format(v=val)
    return my_format

def generate_graph(data):
    nouveauTableau = []
    for i in range(len(data)):
        tmpTableau = []
        tmpTableau += data[i][0].split('  ')
        tmpTableau.append(data[i][1])
        nouveauTableau.append(tmpTableau)

    basicGraphs(nouveauTableau)
    filterGraph = str(input("Voulez-vous un graphique ciblé sur une IP ? (o/N) "))
    if filterGraph.lower() == "o":
        ipFilter = str(input("Quelle est l'IP que vous voulez filtrer ? "))
        graphFiltered(nouveauTableau, ipFilter)

def basicGraphs(data):
    # IP SRC PART
    labelsSrc = []
    occSrc = []
    tabCountSrc = countSrc(data)
    for key, value in tabCountSrc.items():
        labelsSrc.append(key)
        occSrc.append(value)

    # IP DST PART
    labelsDst = []
    occDst = []
    tabCountDst = countDst(data)
    for key, value in tabCountDst.items():
        labelsDst.append(key)
        occDst.append(value)

    # GENERATE CHARTS
    fig, (ax1, ax2) = plt.subplots(2)

    fig.tight_layout(pad=3.0)

    ax1.pie(occSrc,autopct=autopct_format(occSrc), shadow=True, startangle=90)
    ax1.legend(labelsSrc, title="IP SOURCE")
    ax1.axis('equal')
    ax1.set_title("IP'S AS SOURCE")

    ax2.pie(occDst, autopct=autopct_format(occDst), shadow=True, startangle=90)
    ax2.legend(labelsDst, title="IP DESTINATION")
    ax2.axis('equal')
    ax2.set_title("IP'S AS DESTINATION")

    plt.savefig('basicGraphs.png', dpi=200)
    plt.show()

def validate_ip(ip_str):
    reg = r"^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$"
    if re.match(reg, ip_str):
        return True
    else:
        return False

def extract_pcap(pcapfile):
    try:
        open_pcap = open(pcapfile, 'rb')
    except IOError:
        print("Le fichier n'existe pas !")

    pcap_file = savefile.load_savefile(open_pcap, layers=2, verbose=True)
    packets_ip = pcap_file.packets
    compteur = 0

    pair_matches = []
    matches = []
    clean_matches = []
    clean_pairs = re.compile('ipv4 packet from ([^<]* to [^<]*) carrying')

    for i in range(len(packets_ip)):
        pair_matches.append(str(pcap_file.packets[i].packet.payload))

    for i in pair_matches:
        clean_pair = re.findall(clean_pairs, i)
        if len(clean_pair) > 0:
            matches.append(clean_pair[0].replace('b', '').replace("'", ""))
            tmp = clean_pair[0].replace('to', '').replace('b', '').replace("'", "")
            clean_matches.append(tmp)

    results = [[x, matches.count(x)] for x in set(matches)]
    clean_results = [[x, clean_matches.count(x)] for x in set(clean_matches)]

    for i in results:
        print(str(i[0]) + ' -> ' + str(i[1]) + ' fois')

    toGraphics = str(input("Voulez-vous générer des graphiques ? (o/N) "))
    if toGraphics.lower() == 'o':
        generate_graph(clean_results)
        toExport = str(input("Voulez-vous exporter en CSV ? (o/N) "))
        if toExport.lower() == 'o':
            export_as_csv(clean_results)
        else:
            print("Merci d'avoir utilisé notre PcapParser !")
            sys.exit(0)
    else:
        toExport = str(input("Voulez-vous exporter en CSV ? (o/N) "))
        if toExport.lower() == 'o':
            export_as_csv(clean_results)
        else:
            print("Merci d'avoir utilisé notre PcapParser !")
            sys.exit(0)

def export_as_csv(results):

    wbk = xlwt.Workbook('utf-8')
    saveFileXLS = os.getcwd() + '/pcap_results.xls'
    headings_font = xlwt.easyxf('font: color-index blue, bold on; align: horiz centre;')
    normal_font = xlwt.easyxf('font: bold off; align: horiz centre')

    sheet0 = wbk.add_sheet('IPs extraites', cell_overwrite_ok=True)
    sheet0.write(0, 0, 'IP Source', headings_font)
    sheet0.write(0, 1, 'IP Destination', headings_font)
    sheet0.write(0, 2, 'Occurences', headings_font)

    ##############

    if os.path.exists("basicGraphs.png"):
        sheet1 = wbk.add_sheet('BasicGraphs', cell_overwrite_ok=True)
        imgBasicGraph = Image.open("basicGraphs.png")
        r, g, b, a = imgBasicGraph.split()
        imgBasicGraph = Image.merge("RGB", (r, g, b))
        imgBasicGraph.save('BasicGraphs.bmp')
        sheet1.insert_bitmap('BasicGraphs.bmp', 1, 1)

    if os.path.exists("filteredGraphs.png"):
        sheet2 = wbk.add_sheet('FilteredGraphs', cell_overwrite_ok=True)
        imgFilteredGraph = Image.open("filteredGraphs.png")
        r, g, b, a = imgFilteredGraph.split()
        imgFilteredGraph = Image.merge("RGB", (r, g, b))
        imgFilteredGraph.save('filteredGraphs.bmp')
        sheet2.insert_bitmap('filteredGraphs.bmp', 2, 1)

    for i in range(len(results)):
        ips = results[i][0].split()
        col = 0
        sheet0.write(i + 1, col, ips[0], normal_font)
        col += 1
        sheet0.write(i + 1, col, ips[1], normal_font)
        col += 1
        sheet0.write(i + 1, col, results[i][1], normal_font)
    print('Rapport crée !')
    wbk.save(saveFileXLS)

def getFileName():
    pcap_file = str(input("Veuillez donner le nom du fichier pcap à parser (sans extension) : "))
    pcap_file += ".pcap"

    if os.path.exists(pcap_file):
        extract_pcap(pcap_file)
    else:
        print("Le fichier n'existe pas !")
        sys.exit(0)

def saveAndMove():
    dateChaine = ""
    today = datetime.now()
    dateChaine += str(today.day) + "_" + str(today.month) + "_" + str(today.year) + "-" + str(today.hour) + ":" + str(today.minute) + ":" + str(today.second)
    os.mkdir("save_" + dateChaine)

    files = os.listdir(".")

    for f in files:
        if(".bmp" in f or ".png" in f or ".xls" in f):
            os.replace(f, "save_" + dateChaine + "/"+ f)

def main():
    getFileName()
    saveAndMove()

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
import pickle
import os
import pandas as pd
import math
import chardet
from collections import Counter
import tensorflow as tf
from sklearn.cross_validation import train_test_split
from sklearn.ensemble import RandomForestClassifier
import numpy as np
from imblearn.over_sampling import RandomOverSampler
pkl_file = open('2378_data_directory\word2vec-64dim.pkl', 'rb')
word_vectors = pickle.load(pkl_file)
pkl_file.close()
pkl_file = open('2378_data_directory\myfile.pkl', 'rb')
dicts = pickle.load(pkl_file)
pkl_file.close()
df_new = pd.read_pickle('2378_data_directory\df.pkl')
# print(df_new)
df_new = df_new.sort_values("price_list", ascending=False)
# print(df_new1)
price_list = [[i] for i in df_new['price_list']]
content_list = [i for i in df_new['content_list']]
name_list = [i for i in df_new['name_list']]

def get_token_dict_uni_gram(text):
    token_list = list()
    for i in range(len(text)):
        token_list.append(text[i])
    return token_list

def trans_article(text):
    token_list = get_token_dict_uni_gram(text)
    one_hot = list()
    for token in token_list:
        one_hot.append(dicts[token])
    return one_hot

def numpy_padding(data):
    # Get lengths of each row of data
    lens = np.array([len(i) for i in data])
    # Mask of valid places in each row
    mask = np.arange(lens.max()) < lens[:, None]
    # Setup output array and put elements from data into masked positions
    out = np.zeros(mask.shape, dtype=data.dtype)
    out[mask] = np.concatenate(data)
    return out

def trans_tokens_to_matrix(tokens):
    matrix = list()
    for i in tokens:
        i = int(i)
        if i == 0:
            matrix.append([0 for i in range(64)])
        else:
            matrix.append(word_vectors[i])
    return np.array(matrix)

content_vectors = list()
for i in range(len(content_list)):
    content_vectors.append(trans_article(content_list[i]))
content_vectors = np.array(content_vectors)
print(content_vectors.shape)
# content_vectors = numpy_padding(content_vectors)
# print(content_vectors.shape)

content_matrix = list()
for i in content_vectors:
    content_matrix.append(trans_tokens_to_matrix(i))
content_matrix = np.array(content_matrix)

avg_word_vec = list()
for i in content_matrix:
    if len(i) == 0:
        b = [0 for k in range(64)]
    else:
        b = i[0]
        for j in range(len(i)-1):
            b += i[j+1]
        b /= len(i)
    avg_word_vec.append(b)
avg_word_vec = np.array(avg_word_vec)
dim = 64
X_train_vec, X_test_vec, y_train_price, y_test_price = train_test_split(avg_word_vec, price_list, test_size=0.1, random_state=1)
# ros = RandomOverSampler(random_state=0)
# X_resampled, y_resampled = ros.fit_sample(X_train, y_train)
y_train_class = list()
for i in y_train_price:
    if i[0] > 1:
        y_train_class.append(2)
    elif 1 >= i[0] >= 0.5:
        y_train_class.append(1)
    else:
        y_train_class.append(0)
forest = RandomForestClassifier(n_estimators=5000, random_state=1, n_jobs=-1, max_depth=100)
forest.fit(X_train_vec, y_train_class)
y_test_class = list()
for i in y_test_price:
    if i[0] > 1:
        y_test_class.append(2)
    elif 1 >= i[0] >= 0.5:
        y_test_class.append(1)
    else:
        y_test_class.append(0)
print(len(X_train_vec), len(y_train_price), len(y_train_class))
'''
使用分类器分类结果
'''
y_test_class_pred = forest.predict(X_test_vec)
not_equ = 0
for i in range(len(y_test_class)):
    if y_test_class[i] != y_test_class_pred[i]:
        not_equ += 1
print(len(X_test_vec), len(y_test_price), len(y_test_class), len(y_test_class_pred))
'''
网络结构
'''
print("----------train begin------------")
query_batch = tf.placeholder(tf.float32, shape=[None, dim], name='QueryBatch')
price_batch = tf.placeholder(tf.float32, shape=[None, 1], name='PriceBatch')

L1_N = 300
l1_par_range = np.sqrt(6.0 / (dim + L1_N))
weight1 = tf.Variable(tf.random_uniform([dim, L1_N], - l1_par_range, l1_par_range))
bias1 = tf.Variable(tf.random_uniform([L1_N], - l1_par_range, l1_par_range))
query_l1 = tf.matmul(query_batch, weight1) + bias1
query_l1_out = tf.nn.relu(query_l1)
query_l1_out = tf.nn.dropout(query_l1_out, keep_prob=0.6)

L2_N = 300
l2_par_range = np.sqrt(6.0 / (L1_N + L2_N))
weight2 = tf.Variable(tf.random_uniform([L1_N, L2_N], - l2_par_range, l2_par_range))
bias2 = tf.Variable(tf.random_uniform([L2_N], - l2_par_range, l2_par_range))
query_l2 = tf.matmul(query_l1_out, weight2) + bias2
query_l2_out = tf.nn.relu(query_l2)
query_l2_out = tf.nn.dropout(query_l2_out, keep_prob=0.6)

L3_N = 128
l3_par_range = np.sqrt(6.0 / (L2_N + L3_N))
weight3 = tf.Variable(tf.random_uniform([L2_N, L3_N], - l3_par_range, l3_par_range))
bias3 = tf.Variable(tf.random_uniform([L3_N], - l3_par_range, l3_par_range))
query_l3 = tf.matmul(query_l2_out, weight3) + bias3
query_l3_out = tf.nn.relu(query_l3)
query_l3_out = tf.nn.dropout(query_l3_out, keep_prob=0.6)

L4_N = 1
l4_par_range = np.sqrt(6.0 / (L3_N + L4_N))
weight4 = tf.Variable(tf.random_uniform([L3_N, L4_N], - l4_par_range, l4_par_range))
bias4 = tf.Variable(tf.random_uniform([L4_N], - l4_par_range, l4_par_range))
query_l4 = tf.add(tf.matmul(query_l3_out, weight4), bias4, name="Prediction")
query_l4_out = tf.nn.dropout(query_l4, keep_prob=0.6)
loss = tf.abs(tf.div(tf.subtract(query_l4, price_batch), price_batch))
cost = tf.reduce_mean(loss)
train_step = tf.train.GradientDescentOptimizer(learning_rate=0.0001).minimize(cost)
training_epochs = 5000
display_step = 200
saver = tf.train.Saver()
savedir = "model/"

'''对不同类别做回归'''
price_0_x_train = list()
price_0_y_train = list()
price_1_x_train = list()
price_1_y_train = list()
price_2_x_train = list()
price_2_y_train = list()
for i in range(len(X_train_vec)):
    if y_train_class[i] == 0:
        price_0_x_train.append(X_train_vec[i])
        price_0_y_train.append(y_train_price[i])
    elif y_train_class[i] == 1:
        price_1_x_train.append(X_train_vec[i])
        price_1_y_train.append(y_train_price[i])
    else:
        price_2_x_train.append(X_train_vec[i])
        price_2_y_train.append(y_train_price[i])

pred_class_0_x_test = list()
pred_class_0_y_test = list()
pred_class_1_x_test = list()
pred_class_1_y_test = list()
pred_class_2_x_test = list()
pred_class_2_y_test = list()

for i in range(len(X_test_vec)):
    if y_test_class_pred[i] == 0:
        pred_class_0_x_test.append(X_test_vec[i])
        pred_class_0_y_test.append(y_test_price[i])
    elif y_test_class_pred[i] == 1:
        pred_class_1_x_test.append(X_test_vec[i])
        pred_class_1_y_test.append(y_test_price[i])
    else:
        pred_class_2_x_test.append(X_test_vec[i])
        pred_class_2_y_test.append(y_test_price[i])
aape_total = 0.0
test_total = 0
with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
    for epoch in range(training_epochs):
        _, c = sess.run([train_step, cost], feed_dict={query_batch: price_0_x_train, price_batch: price_0_y_train})
        avg_cost = c / 1
        predict = sess.run(query_l4, feed_dict={query_batch: pred_class_0_x_test})
        predict = np.array(predict)
        predict = predict.reshape(len(predict))
        predict = (predict)
        test_set_y = np.array(pred_class_0_y_test)
        test_set_y = test_set_y.reshape(len(test_set_y))
        aape = 0.0
        for i in range(len(predict)):
            aape += abs((predict[i] - test_set_y[i]) / test_set_y[i])
        aape_total += aape
        test_total += len(predict)
        print("aape:", aape / len(predict))

with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
    for epoch in range(training_epochs):
        _, c = sess.run([train_step, cost], feed_dict={query_batch: price_1_x_train, price_batch: price_1_y_train})
        avg_cost = c / 1
        predict = sess.run(query_l4, feed_dict={query_batch: pred_class_1_x_test})
        predict = np.array(predict)
        predict = predict.reshape(len(predict))
        predict = (predict)
        test_set_y = np.array(pred_class_1_y_test)
        test_set_y = test_set_y.reshape(len(test_set_y))
        aape = 0.0
        for i in range(len(predict)):
            aape += abs((predict[i] - test_set_y[i]) / test_set_y[i])
        aape_total += aape
        test_total += len(predict)
        print("aape:", aape / len(predict))

with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
    for epoch in range(training_epochs):
        _, c = sess.run([train_step, cost], feed_dict={query_batch: price_2_x_train, price_batch: price_2_y_train})
        avg_cost = c / 1
        predict = sess.run(query_l4, feed_dict={query_batch: pred_class_2_x_test})
        predict = np.array(predict)
        predict = predict.reshape(len(predict))
        predict = (predict)
        test_set_y = np.array(pred_class_2_y_test)
        test_set_y = test_set_y.reshape(len(test_set_y))
        aape = 0.0
        for i in range(len(predict)):
            aape += abs((predict[i] - test_set_y[i]) / test_set_y[i])
        aape_total += aape
        test_total += len(predict)
        print("aape:", aape / len(predict))

print("total_aape", aape_total / test_total)


from cmu_graphics import *
from start import *
from game import *
from help import *
from loadBoard import *
from collections import defaultdict

#Extra Stuff:
#1. Hopefully the UI doesn't look terrible; you can see buttons
#   and messages in classes.py
#2. I think the backtracker is decent? idk I tried my best;
#   all the testing can be seen in test.ipynb
#3. added X-Wing hint generation (ctrl f xwing in game.py)

#reload buttons when resize
def onResize(app):
    app.boardLeft = app.width/10
    app.boardTop = app.height/8
    app.boardWidth = app.boardHeight = min(app.width, app.height)*3/4
    Button.buttons = defaultdict(list)
    start_makeButtons(app)
    game_makeButtons(app)
    loadBoard_makeButtons(app)
    help_makeButtons(app)

def onAppStart(app):
    app.difficulty = 'easy'
    app.font = 'monospace'
    app.showHints = app.wrongLabels = True
    app.saveBoard = False
    app.saveBoardPath = "solvedBoards/"
    app.stepsPerSecond = 120

def main():
    runAppWithScreens(initialScreen='start', height=800, width=1000)

main()


from collections import namedtuple

import errno
import fnmatch
import logging
import os
import shutil

from guild import file_util
from guild import run_manifest

log = logging.getLogger("guild")

class MergeError(Exception):
    pass

class StopMerge(Exception):
    """Raised during a merge to indicate the operation stopped early."""
    def __init__(self, target_file, msg=None):
        super().__init__([target_file, msg])
        self.target_file = target_file
        self.msg = msg

RunMerge = namedtuple(
    "RunMerge",
    [
        "run",
        "target_dir",
        "copy_all",
        "skip_sourcecode",
        "skip_deps",
        "exclude",
        "to_copy",
        "to_skip",
    ],
)

CopyFile = namedtuple(
    "MergeFile",
    [
        "file_type",
        "run_path",
        "target_path",
    ],
)
CopyFile.__doc__ = """
Represents a merge file to be copied.

`file_type` is one of:

  s - source code file
  d - project-local dependency source
  o - 'other' file - i.e. a non-source and not a dependency, likely
      generated by the run
"""

SkipFile = namedtuple(
    "SkipFile",
    [
        "file_type",
        "run_path",
        "target_path",
        "reason",
    ],
)
SkipFile.__doc__ = """
Represents a merge file that is skipped rather than copied.

`file_type` is one of:

  s - source code file
  d - project-local dependency source
  o - 'other' file - i.e. a non-source and not a dependency, likely
      generated by the run

`reason` is a code indicating why the file is skipped.

  x   - path matches a user-provided exclude pattern
  s   - user opted to skip source code files
  d   - user opted to skip dependencies
  npd - file is a non-project dependencies (either from a non-project
        location or from a project-local unpacked archive)
  u   - file is unchanged
  ?   - file from manifest is unknown (not a source code file or
        dependency source)
"""

_ManifestEntry = namedtuple(
    "_ManifestEntry",
    [
        "file_type",
        "run_path",
        "file_hash",
        "source",
        "source_subpath",
    ],
)
_ManifestEntry.__doc__ = """
Represents an entry from the run manifest, which is used to record
known files at the time of run init.

Expected values for `file_type` are 's' and 'd' for source code file
and project-local dependency respectively.
"""

def init_run_merge(
    run,
    target_dir,
    copy_all=False,
    skip_sourcecode=False,
    skip_deps=False,
    exclude=None,
    prefer_nonsource=False,
):
    manifest_index = _init_manifest_index(run)
    merge = RunMerge(
        run,
        target_dir,
        copy_all=copy_all,
        skip_sourcecode=skip_sourcecode,
        skip_deps=skip_deps,
        exclude=exclude,
        to_copy=[],
        to_skip=[],
    )
    _apply_run_files_to_merge(run, manifest_index, merge)
    _prune_overlapping_targets(merge, prefer_nonsource)
    return merge

def _init_manifest_index(run):
    """Returns a dict keyed by run path containing _ManifestEntry items."""
    try:
        manifest = run_manifest.manifest_for_run(run.dir)
    except FileNotFoundError as e:
        raise MergeError(f"run manifest does not exist for run {run.id}") from e
    else:
        index = {}
        for manifest_entry in manifest:
            index_entry = _manifest_index_entry(manifest_entry, run)
            if not index_entry:
                continue
            index[index_entry.run_path] = index_entry
        return index

def _manifest_index_entry(entry, run):
    if len(entry) not in (4, 5):
        log.warning("unexpected manfiest entry for run %s: %s", run.id, entry)
        return None
    file_type = entry[0]
    run_path = entry[1]
    file_hash = entry[2]
    source = entry[3]
    source_subpath = entry[4] if len(entry) == 5 else None
    return _ManifestEntry(file_type, run_path, file_hash, source, source_subpath)

def _apply_run_files_to_merge(run, manifest_index, merge):
    for run_path in sorted(_iter_run_files(run)):
        _apply_run_file_to_merge(run_path, manifest_index, merge)

def _iter_run_files(run):
    for path in file_util.find(
        run.dir, followlinks=True, includedirs=True, unsorted=True
    ):
        full_path = os.path.join(run.dir, path)
        if os.path.isfile(full_path):
            yield path

def _apply_run_file_to_merge(run_path, manifest_index, merge):
    normalized_run_path = run_manifest.normalize_path(run_path)
    manifest_entry = manifest_index.get(normalized_run_path)
    if manifest_entry:
        _apply_manifest_entry_to_merge(manifest_entry, merge)
    else:
        _apply_unknown_file_to_merge(normalized_run_path, merge)

def _apply_manifest_entry_to_merge(entry, merge):
    if entry.file_type == "s":
        _apply_sourcecode_entry(entry, merge)
    elif entry.file_type == "d":
        _apply_dep_entry(entry, merge)
    else:
        log.warning(
            "unknown manifest file type '%s' for run file '%s'",
            entry.file_type,
            entry.run_path,
        )
        _skip_unknown_path(entry.run_path, entry.source, entry.file_type, merge)

def _apply_sourcecode_entry(entry, merge):
    if merge.skip_sourcecode:
        _skip_sourcecode_entry(entry.run_path, entry.source, merge)
    elif _is_excluded_path(entry.source, merge):
        _skip_excluded(entry.run_path, entry.source, entry.file_type, merge)
    elif _is_unchanged(entry.run_path, entry.source, merge):
        _skip_unchanged(entry.run_path, entry.source, entry.file_type, merge)
    else:
        _copy_sourcecode_entry(entry, merge)

def _skip_sourcecode_entry(run_path, target_path, merge):
    merge.to_skip.append(SkipFile("s", run_path, target_path, "s"))

def _is_excluded_path(path, merge):
    if not merge.exclude:
        return False
    return any(fnmatch.fnmatch(path, pattern) for pattern in merge.exclude)

def _skip_excluded(run_path, target_path, file_type, merge):
    merge.to_skip.append(SkipFile(file_type, run_path, target_path, "x"))

def _is_unchanged(run_path, target_path, merge):
    """Returns True if a run path is unchanged in the target.

    Returns False if the merge is not configured with a target dir
    (i.e. `merge.target_dir` is None.
    """
    dest = os.path.join(merge.target_dir, target_path)
    if not os.path.isfile(dest):
        return False
    src = os.path.join(merge.run.dir, run_path)
    return not file_util.files_differ(src, dest)

def _skip_unchanged(run_path, target_path, file_type, merge):
    merge.to_skip.append(SkipFile(file_type, run_path, target_path, "u"))

def _copy_sourcecode_entry(entry, merge):
    merge.to_copy.append(CopyFile("s", entry.run_path, entry.source))

def _apply_dep_entry(entry, merge):
    if merge.skip_deps:
        _skip_dep_entry(entry, merge)
    else:
        target_path = _target_path_for_dep_entry(entry)
        if target_path:
            if _is_excluded_path(target_path, merge):
                _skip_excluded(entry.run_path, target_path, entry.file_type, merge)
            elif _is_unchanged(entry.run_path, target_path, merge):
                _skip_unchanged(entry.run_path, target_path, entry.file_type, merge)
            else:
                _copy_dep_entry(entry, target_path, merge)
        else:
            _skip_non_project_dep(entry.run_path, merge)

def _skip_dep_entry(entry, merge):
    merge.to_skip.append(
        SkipFile("d", entry.run_path, _target_path_for_dep_entry(entry), "d")
    )

def _target_path_for_dep_entry(entry):
    if not entry.source_subpath and entry.source.startswith("file:"):
        return entry.source[5:]
    return None

def _copy_dep_entry(entry, project_path, merge):
    merge.to_copy.append(CopyFile("d", entry.run_path, project_path))

def _skip_non_project_dep(path, merge):
    merge.to_skip.append(SkipFile("d", path, None, "npd"))

def _skip_unknown_path(run_path, target_path, file_type, merge):
    merge.to_skip.append(SkipFile(file_type, run_path, target_path, "?"))

def _apply_unknown_file_to_merge(run_path, merge):
    if _is_guildfile(run_path):
        return
    # We don't know anything about the file - assume it corresponds to
    # the path in the project for the purpose of mergine.
    target_path = run_path
    if not merge.copy_all:
        _skip_unknown_path(run_path, target_path, "o", merge)
    elif _is_excluded_path(run_path, merge):
        _skip_excluded(run_path, target_path, "o", merge)
    elif _is_unchanged(run_path, target_path, merge):
        _skip_unchanged(run_path, target_path, "o", merge)
    else:
        _copy_other_path(run_path, target_path, merge)

def _is_guildfile(path):
    return path.startswith(".guild")

def _copy_other_path(run_path, target_path, merge):
    merge.to_copy.append(CopyFile("o", run_path, target_path))

def apply_run_merge(merge, pre_copy=None):
    for cf in _sorted_copy_files(merge.to_copy):
        _apply_copy_file_for_merge(cf, merge, pre_copy)

def _sorted_copy_files(files):
    return sorted(files, key=lambda cf: cf.target_path)

def _apply_copy_file_for_merge(cf, merge, pre_copy):
    src = os.path.join(merge.run.dir, cf.run_path)
    dest = os.path.join(merge.target_dir, cf.target_path)
    _apply_pre_copy(pre_copy, merge, cf, src, dest)
    _copy_file(src, dest)

def _apply_pre_copy(pre_copy, merge, cf, src, dest):
    if not pre_copy:
        return
    pre_copy(merge, cf, src, dest)

def _copy_file(src, dest):
    try:
        _shutil_copy(src, dest)
    except IOError as e:
        if e.errno != errno.ENOENT:
            raise
        os.makedirs(os.path.dirname(dest))
        _shutil_copy(src, dest)

def _shutil_copy(src, dest):
    try:
        shutil.copy(src, dest)
    except shutil.SameFileError:
        pass

def _prune_overlapping_targets(merge, prefer_nonsource=False):
    """Removes merge `to_copy` files with overlapping target paths.

    Overlapping targets may occur because Guild supports two categries
    of files for merge: source code and non-source code. Non-source
    code files include dependencies and generated files.

    `merge.files` may be modified as a result of calling this function.

    By default, the pruning prefers source code files over non-source
    code files. In the default case, when a target path exists both as
    source code and as non-source code, the source code file is
    retained and the non-source code file is pruned.

    To prefer non-source code files over source code files, specify
    `prefer_nonsource=True`.
    """
    source_lookup = {cf.target_path for cf in merge.to_copy if cf.file_type == "s"}
    nonsource_lookup = {cf.target_path for cf in merge.to_copy if cf.file_type != "s"}
    merge.to_copy[:] = [
        cf for cf in merge.to_copy if _keep_for_prune_overlapping(
            cf,
            source_lookup,
            nonsource_lookup,
            prefer_nonsource,
        )
    ]

def _keep_for_prune_overlapping(cf, source_lookup, nonsource_lookup, prefer_nonsource):
    return (
        not (cf.target_path in source_lookup and cf.target_path in nonsource_lookup)
        or (cf.file_type != "s" if prefer_nonsource else cf.file_type == "s")
    )


import requests
from loguru import logger

from utils.config import config

def is_unlock(name, mix_port, curr_node, total_node):
    try:
        headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
                                 "Chrome/64.0.3282.119 Safari/537.36"}
        flag = 0
        if not config['fastMode']:
            # 自制剧
            r1 = requests.get("https://www.netflix.com/title/70242311", proxies={
                "http": f"socks5h://127.0.0.1:{mix_port}",
                "https": f"socks5h://127.0.0.1:{mix_port}"
            }, timeout=5, headers=headers)
            if r1.status_code == 200:
                flag += 1
        else:
            flag += 1

        # 非自制剧
        r2 = requests.get("https://www.netflix.com/title/70143836", proxies={
            "http": f"socks5h://127.0.0.1:{mix_port}",
            "https": f"socks5h://127.0.0.1:{mix_port}"
        }, timeout=5, headers=headers)
        if r2.status_code == 200:
            flag += 1

        # 解锁测试
        if flag == 0:
            logger.info(f"({curr_node}/{total_node}) {name}: None.")
            ntype = "None"
        elif flag == 1:
            if config['fastMode']:
                logger.info(f"({curr_node}/{total_node}) {name}: None.")
                ntype = "None"
            else:
                logger.info(f"({curr_node}/{total_node}) {name}: Only Original.")
                ntype = "Only Original"
        else:
            logger.info(f"({curr_node}/{total_node}) {name}: Full Unlock.")
            ntype = "Full Unlock"

    except Exception as e:
        logger.error(f"({curr_node}/{total_node}) {name}：Error. {str(e.args[0].args[0])}")
        ntype = "Error"
    return ntype


import splitfolders as sf
import os
import argparse

# Change the working directory to the location of the script
os.chdir(os.path.dirname(os.path.abspath(__file__)))

def split(from_path, to_path, train, val, test, seed):
    """Splits the dataset into train, val, and test where train is 80%, val is 10% and test is 10%

    from_path (str): Path to folder containing unsplit images
    to_path (str): Path to folder where the split images will be stored
    train (float): Percentage of data to trainable data
    val (float): Percentage of data to be validation data
    test (float): Percentage of data to be test data
    seed (float): Seed for the image selection

    """
    sf.ratio(from_path, to_path, seed=seed, ratio=(train,val,test))

def define_arguments(parser):
    """Defines the arguments for the script

    Args:
        parser (argparse.ArgumentParser): The parser to add the arguments to
    """
    parser.add_argument('-s', '--source', type=str, help='Path to folder containing unsplit images')
    parser.add_argument('-d', '--destionation', type=str, help='Path to folder where the split images will be stored')
    parser.add_argument('--train', type=float, default=0.8, help='Training set precentage (default: 0.8)')
    parser.add_argument('--val', type=float, default=0.1, help='Validation set precentage (default: 0.1)')
    parser.add_argument('--test', type=float, default=0.1, help='Test set precentage (default: 0.1)')
    parser.add_argument('--seed', type=float, default=1337, help='Seed for the image selection (default: 1337)')

if __name__ == "__main__":
    parser = argparse.ArgumentParser(prog = 'DATA SET SPLITTER', description='Split dataset into train, val, and test set')
    define_arguments(parser)
    args = parser.parse_args()
    split(args.source, args.destionation, args.train, args.val, args.test, args.seed)


#!/usr/bin/env python3

from configparser import ConfigParser

from requests import get

# Set up and performa the request

config = ConfigParser()
config.read('secrets.ini')

base_url = 'https://api.typeform.com/v1/form/' + config['TypeForm']['UID']
args = {'completed': 'true',
        'key': config['TypeForm']['key'],
        }

result = get(base_url, args)

with open('responses/0.json', 'w') as outfile:
    outfile.write(result.text)

# data = result.json()

# submit_dates = [d['metadata']['date_submit'] for d in data['responses']]


from pathlib import Path

# укажите путь к каталогу с файлами
path_dir = Path()

path_file1 = path_dir / 'file1_1.txt'
path_file2 = path_dir / 'file1_2.txt'

with path_file1.open() as fh_in:
    print(f"{type(fh_in)}")
    print(f"{fh_in.read()}")

with path_file2.open(encoding='866') as fh_in:
    text = fh_in.read()

print(text)

text.split()


'''
    计算训练集各类的数量
'''

from tqdm import tqdm
label_dic = {}

num = []
with open('./label/csvfile/CuisineList.txt') as f:
    with open('./train.txt','r') as txt:
        for i, per_laebl in enumerate(f.readlines()):
            label_dic[str(i)] = per_laebl[:-1]

        lines = txt.readlines()
        for line in lines:
            nums = (line.split())[1]
            num.append(nums)

total_num = [0 for i in range(172)]

with tqdm(total = 172) as pbar:
    with open('./total.txt','a+') as total:
        for i in range(172):
            for x in num:
                if x == str(i):
                    total_num[i] += 1
            total.write('第{}类数量为:'.format(i) + str(total_num[i]) + '\n')
            pbar.update(1)

x = 0
for i in range(len(total_num)):
    x += int(total_num[i])
print('图片总数量为:{}'.format(x))

total.close()


import discord
import codecs
import time
import sys
import traceback
import yaml
import aiohttp

from discord.ext import commands
from discord.ext.commands.view import StringView
from cogs.utils.settingsmanager import Settings
from cogs.utils.localizer import Localizer
from cogs.utils.localizer import LocalizerWrapper
from cogs.utils.alias import Aliaser
from cogs.utils.context import Context
from cogs.utils.logger import BotLogger
from cogs.helpformatter import commandhelper
from cogs.utils.paginator import Scroller

with codecs.open("data/config.yaml", 'r', encoding='utf8') as f:
    conf = yaml.load(f, Loader=yaml.SafeLoader)

initial_extensions = [
    'cogs.cogs',
    'cogs.settings',
    'cogs.misc',
    'cogs.helpformatter'
]

on_ready_extensions = [
    'cogs.music',
    'cogs.musicevents'
]

def _get_prefix(bot, message):
    if not message.guild:
        prefix = bot.settings.default_prefix
        return commands.when_mentioned_or(prefix)(bot, message)
    prefixes = bot.settings.get(message.guild, 'prefixes', 'default_prefix')
    return commands.when_mentioned_or(*prefixes)(bot, message)

class Bot(commands.Bot):
    def __init__(self, debug: bool=False):
        super().__init__(command_prefix=_get_prefix,
                         description=conf["bot"]["description"])

        self.settings = Settings(**conf['default server settings'])
        self.APIkeys = conf.get('APIkeys', {})

        self.localizer = Localizer(conf.get('locale path', "./localization"), conf.get('locale', 'en_en'))
        self.aliaser = Aliaser(conf.get('locale path', "./localization"), conf.get('locale', 'en_en'))

        self.session = aiohttp.ClientSession(loop=self.loop)

        self.debug = debug
        self.main_logger = logger
        self.logger = self.main_logger.bot_logger.getChild("Bot")
        self.logger.debug("Debug: %s" % debug)

        for extension in initial_extensions:
            try:
                self.load_extension(extension)
            except Exception:
                self.logger.exception("Loading of extension %s failed" % extension)

    async def on_command_error(self, ctx, err):
        if not self.debug:
            if (isinstance(err, commands.MissingRequiredArgument) or
                    isinstance(err, commands.BadArgument)):
                paginator = commandhelper(ctx, ctx.command, ctx.invoker, include_subcmd=False)
                scroller = Scroller(ctx, paginator)
                await scroller.start_scrolling()

            if isinstance(err, commands.CommandInvokeError):
                self.logger.debug("Error running command: %s\n Traceback: %s"
                                 % (ctx.command, traceback.format_exception(err.__traceback__)))
                pass

            elif isinstance(err, commands.NoPrivateMessage):
                self.logger.debug("Error running command: %s\n Traceback: %s"
                                 % (ctx.command, traceback.format_exception(err.__traceback__)))
                await ctx.send('That command is not available in DMs')

            elif isinstance(err, commands.CommandOnCooldown):
                await ctx.send(f"{ctx.message.author.mention} Command is on cooldown. Try again in `{err.retry_after:.1f}` seconds.")

            elif isinstance(err, commands.CheckFailure):
                self.logger.debug("Error running command: %s\n Traceback: %s"
                                 % (ctx.command, traceback.format_exception(err.__traceback__)))
                pass

            elif isinstance(err, commands.CommandNotFound):
                pass
        else:
            tb = err.__traceback__
            traceback.print_tb(tb)
            print(err)

    async def on_message(self, message):
        if message.author.bot:
            return
        await self.process_commands(message)

    async def process_commands(self, message):
        ctx = await self.get_context(message, cls=Context)

        # Replace aliases with commands
        ctx = self.aliaser.get_command(ctx)

        # Add the localizer
        if ctx.command and ctx.command.cog_name:
            ctx.localizer = LocalizerWrapper(self.localizer, ctx.locale, ctx.command.cog_name.lower())
        else:
            ctx.localizer = LocalizerWrapper(self.localizer, ctx.locale, None)

        await self.invoke(ctx)

    async def on_ready(self):
        if not hasattr(self, 'uptime'):
            self.uptime = time.time()

        for extension in on_ready_extensions:
            try:
                self.logger.debug("Loading extension %s" % extension)
                self.load_extension(extension)
            except Exception as e:
                self.logger.exception("Loading of extension %s failed" % extension)

        print(f'\nLogged in as: {self.user.name}' +
              f' in {len(self.guilds)} servers.')
        print(f'Version: {discord.__version__}\n')

        await self.change_presence(activity=discord.Game(type=0,
                                   name=conf["bot"]["playing status"]),
                                   status=discord.Status.online)

    def run(self):
        try:
            super().run(conf["bot"]["token"], reconnect=True)
        except Exception as e:
            tb = e.__traceback__
            traceback.print_tb(tb)
            print(e)

def run_bot(debug: bool=False):
    bot = Bot(debug=debug)
    bot.run()

if __name__ == '__main__':
    state = False
    if 'debug' in sys.argv:
        state = True
    logger = BotLogger(state, conf.get('log_path', '/data/logs'))
    run_bot(debug=state)


"""
+
NAME:
  mmrs_abtov2v3

PURPOSE:
  Convert MRS local alpha,beta coordinates to JWST v2,v3 coordinates

CALLING SEQUENCE:
  mmrs_abtov2v3,a,b,band,[refdir=]

INPUTS:
  a       - Alpha coordinate in arcsec
  b       - Beta coordinate in arcsec
  band    - band name (e.g, '1A')

OPTIONAL INPUTS:
  refdir - Root directory for distortion files

OUTPUT:
  v2      - V2 coordinate in arcsec
  v3      - V3 coordinate in arcsec

OPTIONAL OUTPUT:
  XAN     - XAN coordinate in arcsec
  YAN     - YAN coordinate in arcsec

COMMENTS:
  Works with CDP5 delivery files.  Inverse function is mmrs_v2v3toab.pro

EXAMPLES:

BUGS:

PROCEDURES CALLED:

INTERNAL SUPPORT ROUTINES:

REVISION HISTORY:
  30-Jul-2015  Written by David Law (dlaw@stsci.edu)
  27-Oct-2015  Add conversion to REAL V2,V3 (D. Law)
  16-Nov-2015  Add conversion to XAN,YAN (D. Law)
  24-Jan-2016  Update reference files to CDP5 (D. Law)
  17-Oct-2016  Input/output v2/v3 in arcsec (D. Law)
  13-Dec-2017  Update directory path for new STScI-MIRI workspace
  10-Oct-2018  Update directory path for new miricoord structure
-
------------------------------------------------------------------------------"""

import numpy as np
from astropy.io import fits

def mmrs_abtov2v3(a,b,band,cdpDir=None,fileversion = "06.04.00"):

    if cdpDir is None:
        cdpDir='/Users/ioannisa/Desktop/python/miri_devel/cdp_data/'

    # Ensure we're not using integer inputs
    adbl=a.astype('float')
    bdbl=b.astype('float')

    # Determine input reference FITS file
    distcdp = {}
    distcdp["3C"] = "MIRI_FM_MIRIFULONG_34LONG_DISTORTION_%s.fits" %fileversion
    distcdp["3B"] = "MIRI_FM_MIRIFULONG_34MEDIUM_DISTORTION_%s.fits" %fileversion
    distcdp["3A"] = "MIRI_FM_MIRIFULONG_34SHORT_DISTORTION_%s.fits" %fileversion

    distcdp["1C"] = "MIRI_FM_MIRIFUSHORT_12LONG_DISTORTION_%s.fits" %fileversion
    distcdp["1B"] = "MIRI_FM_MIRIFUSHORT_12MEDIUM_DISTORTION_%s.fits" %fileversion
    distcdp["1A"] = "MIRI_FM_MIRIFUSHORT_12SHORT_DISTORTION_%s.fits" %fileversion

    distcdp["4C"] = distcdp["3C"]
    distcdp["4B"] = distcdp["3B"]
    distcdp["4A"] = distcdp["3A"]

    distcdp["2C"] = distcdp["1C"]
    distcdp["2B"] = distcdp["1B"]
    distcdp["2A"] = distcdp["1A"]

    # Read alpha,beta -> XAN,YAN table
    convtable=fits.open(cdpDir+distcdp[band])['albe_to_XANYAN'].data
    # Determine which rows we need
    v2index=np.where(convtable.field(0)=='T_CH{}_XAN'.format(band))[0][0]
    v3index=np.where(convtable.field(0)=='T_CH{}_YAN'.format(band))[0][0]
    # Trim to relevant v2, v3 rows for this band
    conv_v2=list(convtable[v2index])
    conv_v3=list(convtable[v3index])

    # WARNING index 0 is just text, the coefficients start from index 1
    xan=conv_v2[1]+conv_v2[2]*adbl + conv_v2[3]*bdbl + conv_v2[4]*adbl*bdbl
    yan=conv_v3[1]+conv_v3[2]*adbl + conv_v3[3]*bdbl + conv_v3[4]*adbl*bdbl

    # convert to arcsec from arcmin
    v2=xan*60.
    v3=-(yan+7.8)*60.
    xan=xan*60.
    yan=yan*60.

    return v2,v3,xan,yan


import torch
import torch.nn as nn
from torch import optim
from torchvision import datasets, transforms
from torch.utils.data import random_split, DataLoader

train_data = datasets.MNIST('data', train=True, download=True, transform=transforms.ToTensor())
train, val = random_split(train_data, [55000, 5000])
train_loader = DataLoader(train, batch_size=32)
val_loader = DataLoader(val, batch_size=32)

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()

        self.features = nn.Sequential(nn.Conv2d(in_channels=1, out_channels=5, kernel_size=3, padding=1),
                                      nn.ReLU(inplace=True), nn.BatchNorm2d(5),
                                      nn.Conv2d(in_channels=5, out_channels=10, kernel_size=3, padding=1),
                                      nn.MaxPool2d(2, 2), nn.ReLU(inplace=True), nn.BatchNorm2d(10),
                                      nn.Conv2d(in_channels=10, out_channels=20, kernel_size=3, padding=1),
                                      nn.ReLU(inplace=True), nn.BatchNorm2d(20),
                                      nn.Conv2d(in_channels=20, out_channels=40, kernel_size=3, padding=1),
                                      nn.MaxPool2d(2, 2), nn.ReLU(inplace=True), nn.BatchNorm2d(40))

        self.classifier = nn.Sequential(nn.Linear(7 * 7 * 40, 1024), nn.ReLU(inplace=True),
                                       	nn.Linear(1024, 2048), nn.ReLU(inplace=True),
                                        nn.Linear(2048, 10))

    def forward(self, x):

        x = self.features(x)
        x = x.view(-1, 7 * 7 * 40)
        x = self.classifier(x)
        return x

# Instantiation of the network
model = Net()

# Instantiation of the cross-entropy loss
loss = nn.CrossEntropyLoss()

# Instantiation of the Adam optimizer
optimizer = optim.Adam(model.parameters(), lr=3e-4, weight_decay=0.001)

nb_epochs = 10
for epoch in range(nb_epochs):
  losses = list()
  accuracies = list()
  model.train()
  for batch in train_loader:
    x, y = batch
    optimizer.zero_grad()

    # Computing the forward pass
    outputs = model(x)

    # Computing the loss function
    J = loss(outputs, y)

    # Computing the gradients
    J.backward()

    # Updating the weights
    optimizer.step()
    losses.append(J.item())
    accuracies.append(y.eq(outputs.detach().argmax(dim=1)).float().mean())
  print(f'Epoch {epoch + 1}', end=', ')
  print(f'training loss: {torch.tensor(losses).mean() : .2f}', end=', ')
  print(f'training accuracy: {torch.tensor(accuracies).mean(): .2f}')

  losses = list()
  accuracies = list()
  model.eval()
  for batch in val_loader:
    x, y = batch

    # Computing the forward pass
    with torch.no_grad():
      outputs = model(x)

    # Computing the loss function
    J = loss(outputs, y)
    losses.append(J.item())
    accuracies.append(y.eq(outputs.detach().argmax(dim=1)).float().mean())
  print(f'Epoch {epoch + 1}', end=', ')
  print(f'validation loss: {torch.tensor(losses).mean() : .2f}', end=', ')
  print(f'validation accuracy: {torch.tensor(accuracies).mean(): .2f}')


from datetime import datetime
from django import template
from django.shortcuts import render, redirect
from django.contrib import messages
from django.shortcuts import get_object_or_404
from account.models import User, UserProfile
from .models import JobPost
from .forms import JobPostForm, CompanyProfileForm
from account.states import STATUS_CHOICES, JOB_STATUS

def employer_profile(request, id=0):
    user = User.objects.get(pk=id)

    if request.method == 'GET':
        if id == user:
            form = CompanyProfileForm()
        else:
            user = UserProfile.objects.get(user_id=id)
            form = CompanyProfileForm(instance=user)
        return render(request, 'employers/company_profile_form.html', {'form': form})
    else:
        if id == user:
            form = CompanyProfileForm(request.POST, request.FILES)
        else:
            user = UserProfile.objects.get(user_id=id)
            form = CompanyProfileForm(request.POST, request.FILES, instance=user or None)
        if form.is_valid():
            profile = form.save(commit=False)
            profile.user = request.user
            profile.save()
        messages.success(request, 'Profile Upadated Successfully !')
        return redirect('employer-dashboard', request.user.pk)

def employer_dashboard(request, id):
    user    = get_object_or_404(User, pk=id)
    posts   = JobPost.objects.all().filter(user=user).order_by('-date_posted')
    context = {
        'user': user,
        'posts': posts,
    }

    return render(request, 'employers/dashboard.html', context)

def post(request):
    if request.method == 'POST':
        form = JobPostForm(request.POST)
        if form.is_valid():
            post_form = form.save(commit=False)
            post_form.user = request.user
            post_form.save()
            messages.success(request, 'Job Listed Successfuly')
            return redirect('employer-dashboard', request.user.pk)
        else:
            context = {
                'form':form
            }
            return render(request, 'employers/post.html', context)
    else:
        context = {
                'form':JobPostForm()
            }
        return render(request, 'employers/post.html', context)

def edit_post(request, id):
    job = get_object_or_404(JobPost, pk=id)
    if request.method == 'POST':
        form = JobPostForm(request.POST, instance=job)
        if form.is_valid():
            edited_form = form.save(commit=False)
            edited_form.user = request.user
            edited_form.save()
            messages.success(request, 'Job Updated Successfuly')
            return redirect('employer-dashboard', request.user.pk)
        else:
            context = {
                'form':form
            }
            return render(request, 'employers/post.html', context)
    else:
        context = {
                'form':JobPostForm(instance=job)
            }
        return render(request, 'employers/post.html', context)

def delete_job(request, id):
    jobpost = JobPost.objects.all().get(pk=id)
    jobpost.delete()
    messages.success(request, 'Job successfully deleted')
    return redirect('employer-dashboard', jobpost.user_id)


# pylint: disable=no-self-use,invalid-name
import itertools
import math

from pytest import approx, raises
import torch

from allennlp.modules import ConditionalRandomField
from allennlp.modules.conditional_random_field import allowed_transitions
from allennlp.common.checks import ConfigurationError
from allennlp.common.testing import AllenNlpTestCase

from mylib.modules.marginal_conditional_random_field import MarginalConditionalRandomField

class TestConditionalRandomField(AllenNlpTestCase):
    def setUp(self):
        super().setUp()
        self.logits = torch.Tensor([
            [[0, 0, .5, .5, .2], [0, 0, .3, .3, .1], [0, 0, .9, 10, 1]],
            [[0, 0, .2, .5, .2], [0, 0, 3, .3, .1], [0, 0, .9, 1, 1]],
        ])

        self.orig_tags = torch.LongTensor([
            [2, 3, 4],
            [3, 2, 2]
        ])

        self.tags = torch.FloatTensor([
            [[0, 0, 1, 0, 0], [0, 0, 0, 1, 0], [0, 0, 0, 0, 1]],
            [[0, 0, 0, 1, 0], [0, 0, 1, 0, 0], [0, 0, 1, 0, 0]]
        ])
        self.tags = torch.clamp(self.tags + 1e-30, max=1)

        self.transitions = torch.Tensor([
            [0.1, 0.2, 0.3, 0.4, 0.5],
            [0.8, 0.3, 0.1, 0.7, 0.9],
            [-0.3, 2.1, -5.6, 3.4, 4.0],
            [0.2, 0.4, 0.6, -0.3, -0.4],
            [1.0, 1.0, 1.0, 1.0, 1.0]
        ])

        self.transitions_from_start = torch.Tensor([0.1, 0.2, 0.3, 0.4, 0.6])
        self.transitions_to_end = torch.Tensor([-0.1, -0.2, 0.3, -0.4, -0.4])

        # Use the CRF Module with fixed transitions to compute the log_likelihood
        self.crf = MarginalConditionalRandomField(5)
        self.crf.transitions = torch.nn.Parameter(self.transitions)
        self.crf.start_transitions = torch.nn.Parameter(self.transitions_from_start)
        self.crf.end_transitions = torch.nn.Parameter(self.transitions_to_end)

    def score(self, logits, tags):
        """
        Computes the likelihood score for the given sequence of tags,
        given the provided logits (and the transition weights in the CRF model)
        """
        # Start with transitions from START and to END
        total = self.transitions_from_start[tags[0]] + self.transitions_to_end[tags[-1]]
        # Add in all the intermediate transitions
        for tag, next_tag in zip(tags, tags[1:]):
            total += self.transitions[tag, next_tag]
        # Add in the logits for the observed tags
        for logit, tag in zip(logits, tags):
            total += logit[tag]
        return total

    def test_forward_works_without_mask(self):
        log_likelihood = self.crf(self.logits, self.tags).item()

        # Now compute the log-likelihood manually
        manual_log_likelihood = 0.0

        # For each instance, manually compute the numerator
        # (which is just the score for the logits and actual tags)
        # and the denominator
        # (which is the log-sum-exp of the scores for the logits across all possible tags)
        for logits_i, tags_i in zip(self.logits, self.orig_tags):
            numerator = self.score(logits_i.detach(), tags_i.detach())
            all_scores = [self.score(logits_i.detach(), tags_j)
                          for tags_j in itertools.product(range(5), repeat=3)]
            denominator = math.log(sum(math.exp(score) for score in all_scores))
            # And include them in the manual calculation.
            manual_log_likelihood += numerator - denominator

        # The manually computed log likelihood should equal the result of crf.forward.
        assert manual_log_likelihood.item() == approx(log_likelihood)

    def test_forward_works_with_mask(self):
        # Use a non-trivial mask
        mask = torch.LongTensor([
            [1, 1, 1],
            [1, 1, 0]
        ])

        log_likelihood = self.crf(self.logits, self.tags, mask).item()

        # Now compute the log-likelihood manually
        manual_log_likelihood = 0.0

        # For each instance, manually compute the numerator
        #   (which is just the score for the logits and actual tags)
        # and the denominator
        #   (which is the log-sum-exp of the scores for the logits across all possible tags)
        for logits_i, tags_i, mask_i in zip(self.logits, self.orig_tags, mask):
            # Find the sequence length for this input and only look at that much of each sequence.
            sequence_length = torch.sum(mask_i.detach())
            logits_i = logits_i.data[:sequence_length]
            tags_i = tags_i.data[:sequence_length]

            numerator = self.score(logits_i, tags_i)
            all_scores = [self.score(logits_i, tags_j)
                          for tags_j in itertools.product(range(5), repeat=sequence_length)]
            denominator = math.log(sum(math.exp(score) for score in all_scores))
            # And include them in the manual calculation.
            manual_log_likelihood += numerator - denominator

        # The manually computed log likelihood should equal the result of crf.forward.
        assert manual_log_likelihood.item() == approx(log_likelihood)

    def get_all_sequences(self, tags):
        import itertools
        newseq = []
        for marginal in tags:
            newmarginal = []
            for i, val in enumerate(marginal):
                if val > 0:
                    newmarginal.append(i)
            newseq.append(newmarginal)

        all_seqs = itertools.product(*newseq)

        return all_seqs

    def test_forward_works_with_mask_and_marginal(self):
        # Use a non-trivial mask
        mask = torch.LongTensor([
            [1, 1, 1],
            [1, 1, 0]
        ])

        self.tags = torch.FloatTensor([
            [[1, 1, 1, 1, 1], [0, 0, 0, 1, 0], [1, 1, 1, 1, 1]],
            [[0, 0, 0, 1, 1], [0, 0, 1, 0, 0], [0, 0, 1, 0, 0]]
        ])

        self.get_all_sequences(self.tags[0].long())

        log_likelihood = self.crf(self.logits, torch.clamp(self.tags + 1e-30, max=1), mask).item()

        # Now compute the log-likelihood manually
        manual_log_likelihood = 0.0

        # For each instance, manually compute the numerator
        #   (which is just the score for the logits and actual tags)
        # and the denominator
        #   (which is the log-sum-exp of the scores for the logits across all possible tags)
        for logits_i, tags_i, mask_i in zip(self.logits, self.tags, mask):
            # Find the sequence length for this input and only look at that much of each sequence.
            sequence_length = torch.sum(mask_i.detach())
            logits_i = logits_i.data[:sequence_length]
            tags_i = tags_i.data[:sequence_length]

            all_seqs = self.get_all_sequences(tags_i)

            num_scores = []
            for seq in all_seqs:
                num_scores.append(self.score(logits_i, seq))

            numerator = math.log(sum(math.exp(score) for score in num_scores))

            all_scores = [self.score(logits_i, tags_j)
                          for tags_j in itertools.product(range(5), repeat=sequence_length)]
            denominator = math.log(sum(math.exp(score) for score in all_scores))
            # And include them in the manual calculation.
            manual_log_likelihood += numerator - denominator

        # The manually computed log likelihood should equal the result of crf.forward.
        assert manual_log_likelihood == approx(log_likelihood)