import json
import os

EXCEPTIONS_KYUJITAI = {
    '缺缺': '欠缺',
    '共倭國': '共和國',
}

EXCEPTIONS_SHINJITAI = {
    '欠欠': '欠缺',
}

class BasicConverter(object):
    """
    Basic converter, only converting Shinjitai to Kyujitai, and vice versa (WITHOUT kakikae)
    """

    def __init__(self):

        # Determine Shinjitai/Kyujitai database path
        current_path = os.path.abspath(os.path.dirname(__file__))
        kyujitai_db_path = os.path.join(current_path, 'kyujitai.json')

        # Parse Kyujitai database
        kyujitai_db_file = open(kyujitai_db_path, 'r', encoding="utf-8")
        self.kyujitai_data = json.load(kyujitai_db_file)
        kyujitai_db_file.close()

        # Build Shinjitai to Kyujitai conversion databases
        self.shinjitai_to_kyujitai_database = {}
        self.kyujitai_to_shinjitai_database = {}

        # create Shinjitai/Kyujitai dictionaries
        for entry in self.kyujitai_data:
            shinjitai = entry[0]
            kyujitai = entry[1]
            self.shinjitai_to_kyujitai_database[shinjitai] = kyujitai
            self.kyujitai_to_shinjitai_database[kyujitai] = shinjitai

    def shinjitai_to_kyujitai(self, input_string):

        # convert individual characters
        for char in self.shinjitai_to_kyujitai_database:
            input_string = input_string.replace(char, self.shinjitai_to_kyujitai_database[char])

        # process conversion exceptions
        for word in EXCEPTIONS_KYUJITAI:
            input_string = input_string.replace(word, EXCEPTIONS_KYUJITAI[word])

        return input_string

    def kyujitai_to_shinjitai(self, input_string):

        # convert individual characters
        for char in self.kyujitai_to_shinjitai_database:
            input_string = input_string.replace(char, self.kyujitai_to_shinjitai_database[char])

        # process conversion exceptions
        for word in EXCEPTIONS_SHINJITAI:
            input_string = input_string.replace(word, EXCEPTIONS_SHINJITAI[word])

        return input_string

class KyujitaiConverter(object):
    """
    Full converter, converting Shinjitai to Kyujitai, and vice versa (WITH kakikae)
    """

    def __init__(self):

        # Use BasicConverter to convert individual Shinjitai/Kyujitai characters
        self.basic_converter = BasicConverter()

        # Determine Kakikae databases paths
        current_path = os.path.abspath(os.path.dirname(__file__))
        kakikae_simplified_db_path = os.path.join(current_path, 'kakikae_simplified.json')
        kakikae_variants_db_path = os.path.join(current_path, 'kakikae_variants.json')

        # Parse Kakikae database
        kakikae_simplified_db_file = open(kakikae_simplified_db_path, 'r', encoding="utf-8")
        self.kakikae_database_simplified = json.load(kakikae_simplified_db_file)
        kakikae_simplified_db_file.close()
        kakikae_variants_db_file = open(kakikae_variants_db_path, 'r', encoding="utf-8")
        self.kakikae_database_variants = json.load(kakikae_variants_db_file)
        kakikae_variants_db_file.close()

        # Build Kakikae conversion databases
        self.kakikae_encode_database = {}
        self.kakikae_decode_database = {}

        # First step: index of all potential words (build database keys)
        for entry in self.kakikae_database_simplified:
            new_char = entry['new']
            # Encode (Shinjitai to Kyujitai) database keys
            for word in entry.get('words'):
                # add word if not already in database
                if word not in self.kakikae_encode_database:
                    self.kakikae_encode_database[word] = word
            # Decode (Kyujitai to Shinjitai) database keys
            for old_char in entry['old']:
                for word in entry.get('words'):
                    word = word.replace(new_char, old_char)
                    if word not in self.kakikae_decode_database:
                        self.kakikae_decode_database[word] = word
        # Variants database only used for decoding (Kyujitai to Shinjitai conversion)
        for entry in self.kakikae_database_variants:
            new_char = entry['new']
            for old_char in entry['old']:
                for word in entry.get('words'):
                    word = word.replace(new_char, old_char)
                    if word not in self.kakikae_decode_database:
                        self.kakikae_decode_database[word] = word

        # Second step: replace Kakikae characters (build database values)
        for entry in self.kakikae_database_simplified:
            new_char = entry['new']
            base_old_char = entry['old'][0]
            # Encode (Shinjitai to Kyujitai) database values
            for word in entry.get('words'):
                self.kakikae_encode_database[word] = self.kakikae_encode_database[word].replace(new_char, base_old_char)
            # Decode (Kyujitai to Shinjitai) database values
            for old_char in entry['old']:
                for word in entry.get('words'):
                    word = word.replace(new_char, old_char)
                    self.kakikae_decode_database[word] = self.kakikae_decode_database[word].replace(old_char, new_char)
        # Variants database only used for decoding (Kyujitai to Shinjitai conversion)
        for entry in self.kakikae_database_variants:
            new_char = entry['new']
            for old_char in entry['old']:
                for word in entry.get('words'):
                    word = word.replace(new_char, old_char)
                    self.kakikae_decode_database[word] = self.kakikae_decode_database[word].replace(old_char, new_char)

    def shinjitai_to_kyujitai(self, input_string):

        # revert douon no kanji ni yoru kakikae
        for word in self.kakikae_encode_database:
            input_string = input_string.replace(word, self.kakikae_encode_database[word])

        # convert remaining individual characters
        input_string = self.basic_converter.shinjitai_to_kyujitai(input_string)

        return input_string

    def kyujitai_to_shinjitai(self, input_string):

        # convert individual characters first
        input_string = self.basic_converter.kyujitai_to_shinjitai(input_string)

        # apply douon no kanji ni yoru kakikae
        for word in self.kakikae_decode_database:
            input_string = input_string.replace(word, self.kakikae_decode_database[word])

        return input_string


import pghistory
import model_utils.fields
import mptt.fields
import django.utils.timezone
from django.contrib.auth.models import Group
from django.db import models
from model_utils.models import SoftDeletableModel, TimeStampedModel
from mptt.models import MPTTModel, TreeForeignKey

from attendees.whereabouts.models import Organization

class Menu(MPTTModel, TimeStampedModel, SoftDeletableModel):
    """
    Todo: Raise ValueError if the instance.get_level() > 1 due to Boostrap 4 dropdown-submenu limit,
    or smartmenus will be needed.

    UniqueConstraint(fields=['organization', 'category', 'html_type', 'url_name'], condition=models.Q(is_removed=False))
    cannot be used since some menu repatitive for different user groups

    """

    ATTENDEE_CREATE_VIEW = "attendee_create_view"
    ATTENDEE_UPDATE_VIEW = "attendee_update_view"
    ATTENDEE_UPDATE_SELF = "attendee_update_self"

    id = models.BigAutoField(
        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
    )

    organization = models.ForeignKey(
        Organization,
        null=False,
        blank=False,
        default=0,
        on_delete=models.SET(0),
        help_text="Organization of the menu",
    )

    category = models.CharField(
        max_length=32,
        null=False,
        blank=False,
        db_index=True,
        default="main",
        help_text="Type of menu, such as 'main', 'side', etc",
    )

    parent = TreeForeignKey(
        "self", on_delete=models.SET(-1), null=True, blank=True, related_name="children"
    )

    html_type = models.CharField(
        max_length=50,
        blank=True,
        null=False,
        help_text="HTML tags such as div or a. For API it can be blank",
    )

    urn = models.CharField(
        max_length=255,
        blank=True,
        null=True,
        help_text="use relative path (including leading & ending slash '/') such as /app/division/assembly/page-name",
    )

    url_name = models.SlugField(
        max_length=255,
        blank=False,
        null=False,
        db_index=True,
        help_text="view name of the path, such as 'assembly_attendances', 'divider between index and register links', "
        "etc. For API it's class name",
    )

    display_name = models.CharField(
        max_length=50,
        blank=False,
        null=False,
        help_text="description of the path, such as 'Character index page', 'divider between index and register "
        "links', etc",
    )

    display_order = models.SmallIntegerField(default=0, blank=False, null=False)

    infos = models.JSONField(
        null=True,
        blank=True,
        default=dict,
        help_text="HTML attributes & more such as {'class': 'dropdown-item'}. Please keep {} here even no data.",
    )

    auth_groups = models.ManyToManyField(
        Group, through="MenuAuthGroup", related_name="auth_groups"
    )

    class MPTTMeta:
        order_insertion_by = ["display_order", "display_name"]

    class Meta:
        db_table = "users_menus"

    def __str__(self):
        return "%s %s %s URN: ...%s" % (
            self.organization_slug,
            self.category.upper(),
            self.display_name,
            self.urn[-40:] if self.urn else "",
        )

    @property
    def organization_slug(self):
        return self.organization.slug if self.organization else ""

    @staticmethod
    def user_can_create_attendee(user, url_name=ATTENDEE_CREATE_VIEW):
        return Menu.objects.filter(
            auth_groups__in=user.groups.all(),
            url_name=url_name,
            menuauthgroup__write=True,
            is_removed=False,
        ).exists()

class MenusHistory(pghistory.get_event_model(
    Menu,
    pghistory.Snapshot('menu.snapshot'),
    pghistory.BeforeDelete('menu.before_delete'),
    name='MenusHistory',
    related_name='history',
)):
    pgh_id = models.BigAutoField(primary_key=True, serialize=False)
    pgh_created_at = models.DateTimeField(auto_now_add=True)
    pgh_label = models.TextField(help_text='The event label.')
    pgh_obj = models.ForeignKey(db_constraint=False, on_delete=models.deletion.DO_NOTHING, related_name='history', to='users.menu')
    id = models.IntegerField(db_index=True)
    created = model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')
    modified = model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')
    organization = models.ForeignKey(db_constraint=False, default=0, help_text='Organization of the menu', on_delete=models.deletion.DO_NOTHING, related_name='+', related_query_name='+', to='whereabouts.organization')
    is_removed = models.BooleanField(default=False)
    tree_id = models.PositiveIntegerField(editable=False)
    level = models.PositiveIntegerField(editable=False)
    lft = models.PositiveIntegerField(editable=False)
    rght = models.PositiveIntegerField(editable=False)
    display_order = models.SmallIntegerField(default=0)
    category = models.CharField(default='main', help_text="Type of menu, such as 'main', 'side', etc", max_length=32)
    url_name = models.SlugField(db_index=False, help_text="view name of the path, such as 'assembly_attendances', 'divider between index and register links', etc. For API it's class name", max_length=255)
    display_name = models.CharField(help_text="description of the path, such as 'Character index page', 'divider between index and register links', etc", max_length=50)
    infos = models.JSONField(blank=True, default=dict, help_text="HTML attributes & more such as {'class': 'dropdown-item'}. Please keep {} here even no data.", null=True)
    parent = mptt.fields.TreeForeignKey(blank=True, db_constraint=False, null=True, on_delete=models.deletion.DO_NOTHING, related_name='+', related_query_name='+', to='users.menu')
    urn = models.CharField(blank=True, help_text="use relative path (including leading & ending slash '/') such as /app/division/assembly/page-name", max_length=255, null=True)
    html_type = models.CharField(blank=True, help_text='HTML tags such as div or a. For API it can be blank', max_length=50)
    pgh_context = models.ForeignKey(db_constraint=False, null=True, on_delete=models.deletion.DO_NOTHING, related_name='+', to='pghistory.context')

    class Meta:
        db_table = "users_menushistory"


import pygame
import random
import math

# Initialize the game engine
pygame.init()

BLACK = [0, 0, 0]
WHITE = [255, 255, 255]

WIDTH = 500
HEIGHT = 500
# Set the height and width of the screen
SIZE = [WIDTH, HEIGHT]

screen = pygame.display.set_mode(SIZE)
pygame.display.set_caption("Ball Animation")

BLACK = (  0,   0,   0)
WHITE = (255, 255, 255)
RED   = (255,   0,   0)
# Create an empty array
ball_list = pygame.sprite.Group()

def sign(n):
    if n > 0:
        return -1
    else:
        return 1

def random_color():
    return (random.randrange(0, 244), random.randrange(0, 244), random.randrange(0, 244))

class Ball(pygame.sprite.Sprite):

    def __init__(self, size, x, y, velocity, theta, elasticity, forces, friction_coefficient):

        super().__init__()

        self.size = size
        self.friction_coefficient = friction_coefficient
        self.elasticity = elasticity

        self.image = pygame.Surface([self.size, self.size])
        self.image.fill(WHITE)
        self.image.set_colorkey(WHITE)

        self.rect = self.image.get_rect()

        self.rect.x = x
        self.rect.y = HEIGHT - y

        self.v_x = velocity * math.cos(theta)
        self.v_y = -1 * velocity * math.sin(theta)

        self.a_x = 0
        self.a_y = 0

        self.forces = forces

        # Draw the ellipse
        pygame.draw.ellipse(self.image, (random.randrange(0, 244), random.randrange(0, 244), random.randrange(0, 244)),[0, 0, self.size, self.size])

    def apply_force(self, force):
        self.a_x += force[0]
        self.a_y += -1 * force[1]

    def update(self):

        for force in self.forces:
            self.apply_force(force)

        #bogus friction (messes things up when v is too high or too low)
        friction_x = (self.v_x ** 2)*sign(self.v_x)*self.friction_coefficient
        friction_y = (self.v_y ** 2)*sign(self.v_y)*self.friction_coefficient*-1

        #friction_y is multiplied by -1 because y increases as the ball goes down

        self.apply_force([friction_x, friction_y])

        self.v_x += self.a_x
        self.v_y += self.a_y

        self.rect.x += self.v_x
        self.rect.y += self.v_y

        self.a_x = 0
        self.a_y = 0

        self.collision()

    def collision(self):
        if self.rect.x > WIDTH-self.size or self.rect.x < self.size:
            # Reset it just above the top
            self.v_x = -1 * self.v_x * self.elasticity

            if(self.rect.x < self.size):
                self.rect.x = self.size

            if(self.rect.y > WIDTH-self.size):
                self.rect.y = WIDTH-self.size

        if self.rect.y < 0 or self.rect.y > HEIGHT - self.size:

            if(self.rect.y < 0):
                self.rect.y = 0

            if(self.rect.y > HEIGHT - self.size):
                self.rect.y = HEIGHT - self.size

            self.v_y = -1 * self.v_y * self.elasticity

clock = pygame.time.Clock()

all_sprite_list = pygame.sprite.Group()

all_sprite_list.add(Ball(x=50,
            y=50,
            size=10,
            velocity=50,
            theta=math.pi/2.5,
            forces=[[0, -0.9]],
            elasticity=.9,
            friction_coefficient=0.01
            ))

# Loop until the user clicks the close button.
done = False
while not done:

    for event in pygame.event.get():   # User did something
        if event.type == pygame.QUIT:  # If user clicked close
            done = True   # Flag that we are done so we exit this loop

    all_sprite_list.draw(screen)
    all_sprite_list.update()

    pygame.display.flip()
    clock.tick(100)

pygame.quit()


import argparse
import logging
import os
import shutil
import sys
import tempfile
import traceback

import yaml

from cros.factory.test.env import paths
from cros.factory.utils import file_utils
from cros.factory.utils.process_utils import Spawn

DESCRIPTION = '''
Audits source hashes logged during system finalization
to verify that no unauthorized changes have been made to
a factory bundle.  Hashes are compared against a "golden"
(known to be correct) set of hashes.

Each value for GOLDEN or SAMPLE may be any of the following:

- the 'py' directory of a factory source tree
- a factory log bundle (with name ending in '.tar.xz')
- a factory toolkit (with name ending in '.run')
- an event log file (in which case the last 'source_hashes' event
  logged is examined)

For example, to compare multiple factory logs against a "golden"
toolkit:

  audit_source_hashes --golden install_factory_toolkit.run logs/*.tar.xz
'''

class AuditException(Exception):
  pass

def GetHashes(path):
  """Gets a dictionary of hashes for the given path.

  Args:
    path: A path to any of the accepted file types (see DESCRIPTION).

  Returns:
    A dictionary of paths to their hashes.
  """
  if os.path.isdir(path):
    if os.path.basename(path) != 'py':
      raise AuditException(
          '%s is a directory, but not a path to a "py" source directory' % path)
    return file_utils.HashSourceTree(path)['hashes']

  if path.endswith('.run'):
    tmpdir = tempfile.mkdtemp(prefix='toolkit.')
    try:
      # Extract and hash toolkit
      Spawn([path, '--tar', '-axf', '-C', tmpdir, './usr/local/factory/py'],
            check_call=True, log=True)
      return file_utils.HashSourceTree(
          os.path.join(tmpdir, 'usr', 'local', 'factory', 'py'))['hashes']
    finally:
      shutil.rmtree(tmpdir)

  # None of the above: it's either an event log file, or a report
  # containing an event log file.
  if path.endswith('.tar.xz'):
    # Looks like a report.  Extract the event log data.
    proc = Spawn(['tar', '-Oaxf', path, 'events'], read_stdout=True)
    if proc.returncode:
      raise AuditException(
          'Unable to read events from report %s (tar returned %d)' % (
              path, proc.returncode))
    data = proc.stdout_data
  else:
    # Assume it's an event log.  Look for the event specifically
    # to avoid a bunch of unnecessary YAML parsing.
    data = file_utils.ReadFile(path)

  events = data.split('\n---\n')
  for e in reversed(events):
    if not e.startswith('EVENT: source_hashes'):
      continue
    data = yaml.load(e)

    hash_function = data.get('hash_function')
    if hash_function != file_utils.SOURCE_HASH_FUNCTION_NAME:
      raise ValueError(
          'Expected hash function %r but got %r' % (
              file_utils.SOURCE_HASH_FUNCTION_NAME,
              hash_function))
    return data['hashes']
  raise AuditException(
      'No source_hashes event in event log %s' % path)

def FindMismatches(golden_hashes, sample_hashes, sample, out):
  error_count = [0]

  def ReportLine(path, msg):
    if error_count[0] == 0:
      out.write('In sample %s:\n' % sample)
    out.write('- %s: %s\n' % (path, msg))
    error_count[0] += 1

  all_keys = sorted(set(sample_hashes.keys()) |
                    set(golden_hashes.keys()))

  for k in all_keys:
    sample_value = sample_hashes.get(k)
    golden_value = golden_hashes.get(k)
    if golden_value is None and sample_value is not None:
      ReportLine(k, 'unexpected file encountered in sample')
    elif sample_value is None and golden_value is not None:
      ReportLine(k, 'missing from sample')
    elif sample_value != golden_value:
      ReportLine(k, 'hash mismatch (expected %s, found %s)' % (
          golden_value, sample_value))

  return error_count[0]

def AuditHashes(golden, samples, out):
  """Audits source hashes (see DESCRIPTION).

  Args:
    golden: Path to golden used for analysis.
    samples: A list of samples to compare against.

  Returns:
    True if all values are correct, False otherwise.
  """
  golden_hashes = GetHashes(golden)

  total_bad_samples = 0
  total_mismatched_hashes = 0
  total_exceptions = 0
  for s in samples:
    try:
      sample_hashes = GetHashes(s)
      mismatched_hashes = FindMismatches(golden_hashes,
                                         sample_hashes, s, out)
    except Exception:
      out.write('Error processing sample %s\n' % s)
      traceback.print_exc(file=out)
      total_exceptions += 1
      total_bad_samples += 1
    else:
      total_mismatched_hashes += mismatched_hashes
      if mismatched_hashes:
        total_bad_samples += 1

  if total_bad_samples:
    out.write('\n'
              'Found %d mismatched hashes and %d exceptions.\n'
              'FAILED (%d/%d samples passed).\n' % (
                  total_mismatched_hashes,
                  total_exceptions,
                  len(samples) - total_bad_samples,
                  len(samples)))
    return False
  out.write('PASSED (%d/%d samples passed).\n' % (
      len(samples), len(samples)))
  return True

def main(argv=None, out=sys.stderr):
  parser = argparse.ArgumentParser(
      description=DESCRIPTION,
      formatter_class=argparse.RawDescriptionHelpFormatter)
  parser.add_argument(
      '--golden', '-g', metavar='GOLDEN',
      help='Source of "golden" (correct) hashes; defaults to this source tree')
  parser.add_argument(
      'samples', metavar='SAMPLE', nargs='+',
      help='Samples to check against the golden')
  args = parser.parse_args(sys.argv[1:] if argv is None else argv)
  logging.basicConfig(level=logging.WARNING)

  if args.golden is None:
    args.golden = os.path.join(paths.FACTORY_DIR, 'py')

  sys.exit(0 if AuditHashes(args.golden, args.samples, out) else 1)

if __name__ == '__main__':
  main()


#!/usr/bin/env python

import csv
import os
import re
import sys
import yaml
import copy

signs = {
    r'\(f\)': "female",
    r'\(m\)': "male",
    r'\(v\)': "verb",
    r'\(adj\.\)': "adjective",
    r'\(adj\.': "adjective",
    r'\(adv\.\)': "adverb",
    r'\(adv\)': "adverb",
    r'\(prep\.\)': "preposition",
    r'\([pP]ron\.\)': "pronoun",
}

accents = {
    'á': 'a',
    'í': 'i',
    'é': 'e',
    'ó': 'o',
    'ú': 'u',
    'ü': 'u',
}
words_regex = f"^[ a-zA-Z{''.join(sorted(accents.keys()))}.-]+(\.\.\.|!|\?)?$"

def remove_accent(ladino):
    ladino = re.sub(r'á', 'a', ladino)
    ladino = re.sub(r'í', 'i', ladino)
    ladino = re.sub(r'é', 'e', ladino)
    ladino = re.sub(r'ó', 'o', ladino)
    ladino = re.sub(r'ú', 'u', ladino)

    return ladino

def get_grammar_types():
    types = {
        "female": False,
        "verb": False,
        "adjective": False,
        "adverb": False,
        "preposition": False,
        "pronoun": False,
    }
    for sign, grammar_type in signs.items():
        regex = r'^(.*?)\s+' + sign + r'\s*$'
        match = re.search(regex, ladino)
        if match:
            types[grammar_type] = True
            ladino = match.group(1)

def _err(msg):
    print(f"ERROR {msg}", file=sys.stderr)

def save_words(all_words):
    with open("words.txt", "w") as fh:
        for word in all_words:
            print(word, file=fh)

def save_librelingo_format(dictionary):
    data = {
        'Two-way-dictionary': dictionary,
    }
    filename = "ladinokomunita.yaml"
    #print(data)
    with open(filename, 'w') as fh:
        fh.write("# This is a generated file. Do not edit manually!\n\n")
        fh.write("Skill:\n")
        fh.write("  Name: Ladinokomunita\n")
        fh.write("  Id: 10000\n\n")
        fh.write("New words: []\n")
        fh.write("Phrases: []\n\n")
        yaml.dump(data, fh, Dumper=yaml.Dumper, allow_unicode=True, indent=4)

def save_yaml_format(full):
    filename = "dictionary.yaml"
    data = {
        'words': full
    }
    with open(filename, 'w') as fh:
        yaml.dump(full, fh, Dumper=yaml.Dumper, allow_unicode=True, indent=4)

def _extract_words(word):
    if ',' in word:
        return re.split(r'\s*,\s*', word)
    else:
        return word

def main():
    #filename = "diksionario_biervos_excel_corrected.csv"
    filename = "biervos.csv"
    dictionary = []
    full = []
    all_words = set()
    with open(filename) as fh:
        # Palavra,Eshemplos,Espanyol,English,Turkish,Origen,id,Portuguese,French
        rd = csv.DictReader(fh, delimiter=',')

        count = 50
        for row in rd:
            ladino = row['Palavra']
            eshemplos = row.get('Eshemplos')
            examples = []
            if eshemplos is not None and eshemplos != '':
                examples = [{
                    'ladino': eshemplos,
                }]
            entry = {
                'origen'     : row['Origen'],
                'id'         : row['id'],
                'examples'   : examples,
                'versions': [{
                    'translations': {
                        'english'    : _extract_words(row['English']),
                        'spanish'    : _extract_words(row['Espanyol']),
                        'turkish'    : _extract_words(row['Turkish']),
                        'portuguese' : _extract_words(row['Portuguese']),
                        'french'     : _extract_words(row['French']),
                    },
                }]
            }
            #print(ladino)
            #print(english)
            if ladino is None:
                _err(f"Ladino is None in {row}")
                continue

            match = re.search(r'^(?P<words>[^()]+?)\s*(\((?P<grammar>.*)\))?\s*$', ladino)
            if match is None:
                _err(f"Ladino does not match it is '{ladino}'")
                continue
            grammar = match.group('grammar')
            words_str = match.group('words')
            #print(words_str)
            words = re.split(r'\s*[/,]\s*', words_str)
            word = words[0]
            match = re.search(words_regex, word)
            if not match:
                _err(f"Word '{word}' does not match our rules from Ladino {ladino}")
                continue
            #print(word)
            if entry['versions'][0]['translations']['english'].__class__.__name__ == 'list':
                translated_words = entry['versions'][0]['translations']['english']
            else:
                translated_words = [entry['versions'][0]['translations']['english']]
            for translated_word in translated_words:
                dictionary.append({translated_word: word})
            plain_word = remove_accent(word)
            all_words.add(plain_word)
            #if plain_word in full:
            #    prev = copy.deepcopy(full[plain_word])
            #    #prev_accented = prev.pop('accented')
            #    _err(f"Ladino word '{plain_word}' already exists. (original '{word}' row: {row})")
            #    print(prev, file=sys.stderr)
            #    print(entry, file=sys.stderr)
            #    print('-----', file=sys.stderr)
            #    continue
            #full[plain_word] = copy.deepcopy(entry)
            data = copy.deepcopy(entry)
            data['versions'][0]['accented'] = word
            data['versions'][0]['ladino'] = plain_word
            #grammar_types = get_grammar_types(grammar)
            print(grammar)
            if grammar == 'v':
                grammar = 'verb'
                data['conjugations'] = {}
            elif grammar == 'f':
                grammar = None
                data['versions'][0]['gender'] = 'feminine'
                data['versions'][0]['number'] = 'singular'
            elif grammar == 'm':
                grammar = None
                data['versions'][0]['gender'] = 'masculine'
                data['versions'][0]['number'] = 'singular'
            elif grammar is None:
                pass
            elif grammar in ['adv.', 'adv']:
                grammar = 'adverb'
            elif grammar == 'adj.':
                grammar = 'adjective'
            elif grammar == 'prep.':
                grammar = 'preposition'
            elif grammar in ['pron.', 'Pron.']:
                grammar = 'pronoun'
            else: #grammar in ['adj.+ m/f', 'm+pron.', 'n+adj.', 'm/p', 'm/f']: #, 'm), arabá (f', 'distribuir', 'la', 'en teatro']:
                data['comments'] = [f'grammar: {grammar}']
                grammar = None
            data['grammar'] = grammar

            if len(words) > 1:
                data['versions'][0]['alternative-spelling'] = []
                for alt_word in words[1:]:
                    data['versions'][0]['alternative-spelling'].append({
                        'ladino': remove_accent(alt_word),
                        'accented': alt_word,
                    })

            if len(sys.argv) == 2 and sys.argv[1] == 'all':
                full.append(data)
            else:
                # Normally just save one:
                word_file = f"words/{data['versions'][0]['ladino'].lower()}.yaml"
                if os.path.exists(word_file):
                    exit(f"File {word_file} already exists")
                print(f"{word_file}\nword: {data['versions'][0]['ladino'].lower()}\nid: {data['id']}")
                with open(word_file, 'w') as fh:
                    yaml.dump(data, fh, Dumper=yaml.Dumper, allow_unicode=True, indent=4)
                count -= 1
                if count <= 0:
                    exit()

    #save_librelingo_format(dictionary)
    save_yaml_format(sorted(full, key=lambda this: this['versions'][0]['ladino']))
    #save_words(all_words)

            #if grammar is None:
            #    print(ladino)

            #ladino_words = [ladino]
            #if re.search(r'/', ladino):
            #    ladino_words = ladino.split('/')
            #elif re.search(r',', ladino):
            #    ladino_words = ladino.split(',')
            #elif re.search(r';', ladino):
            #    ladino_words = ladino.split(';')

            #err = False
            #for ladino in ladino_words:
            #    if not re.search(r'^[A-Za-z ]+(\s?\.\.\.)?$', ladino):
            #        #print(f"ERROR ladino word '{ladino}' has unhandled characer in this row: {row}")
            #        err = True
            #        continue
            #if err:
            #    continue

            #for ladino in ladino_words:
            #    ladino = ladino.strip(' ')
            #    #if re.search('^la\s+[a-zA-Z]+$', ladino):
            #    print(ladino)

            #print(row)
            #exit()

if __name__ == '__main__':
    main()


#!/usr/bin/python

import time
import subprocess
import concurrent.futures
import numpy as np
from decimal import *
from threading import Event
from sense_hat import SenseHat
from MotorDriver import *
from ConnectionSenseHat import *
from modules.PCA9685 import PCA9685
from multiprocessing import Value, Array, Process
from gpiozero import RotaryEncoder, Button
from gpiozero.pins.pigpio import PiGPIOFactory
#from WitMotionSensorConnection import JY901S

U_MAX = 100
U_MIN = 0
SPEED = 0
OFFSET = 1
FWD = 1
BCK = 0

# ロータリーエンコーダのピン設定
PIN_ROTAR_A1 = 17
PIN_ROTAR_A2 = 27
PIN_ROTAR_D1 = 22
PIN_ROTAR_D2 = 23

# ロータリーエンコーダ初期化
rotation_a = FWD
rotation_d = FWD
def rotated_clockwise_a():
    global rotation_a
    rotation_a = BCK

def rotated_clockwise_d():
    global rotation_d
    rotation_d = BCK

def rotated_counter_clockwise_a():
    global rotation_a
    rotation_a = FWD

def rotated_counter_clockwise_d():
    global rotation_d
    rotation_d = FWD

# Wit Motion
'''
obj = None
device = None
# WitMotion角度センサ初期化
devicePath = "/dev/ttyUSB0"
if os.path.exists("/dev/ttyUSB1"):
    devicePath = "/dev/ttyUSB1"
obj = JY901S()
device = obj.getDevice()
device.serialConfig.portName = devicePath   # Set serial port
device.serialConfig.baud = 9600                     # Set baud rate
device.openDevice()                                 # Open serial port
# Read configuration information
obj.readConfig(device)
# Data update event
device.dataProcessor.onVarChanged.append(obj.onUpdate)
'''
'''
# sense hatを初期化
sense = SenseHat()
angle = []
# sense hat指示値読み替え用配列定義
for i in range(360):
    if i <= 180:
        angle.append(i)
    else:
        angle.append(i - 360)
'''
# sense hatの指示値読み取りをスレッド化
def read_sense_hat_angle(th_angle_y):
    while True:
        angle_y = subprocess.run("cat /opt/angle_y", encoding='utf-8', stdout=subprocess.PIPE, shell=True)
        angle_y = str(angle_y.stdout)
        if angle_y == '' or angle_y == None:
            continue
        else:
            th_angle_y.value = int(angle_y)

getcontext().prec = 28
def request_controler(th_angle_y):
    factory = PiGPIOFactory()
    rotary_encoder_a = RotaryEncoder(
            PIN_ROTAR_A1, PIN_ROTAR_A2, wrap=True, max_steps=180, pin_factory=factory
    )
    rotary_encoder_a.steps = 0
    rotary_encoder_a.when_rotated_clockwise = rotated_clockwise_a
    rotary_encoder_a.when_rotated_counter_clockwise = rotated_counter_clockwise_a

    rotary_encoder_d = RotaryEncoder(
            PIN_ROTAR_D1, PIN_ROTAR_D2, wrap=True, max_steps=180, pin_factory=factory
    )
    rotary_encoder_d.steps = 0
    rotary_encoder_d.when_rotated_clockwise = rotated_clockwise_d
    rotary_encoder_d.when_rotated_counter_clockwise = rotated_counter_clockwise_d

    # 積算開始
    start = time.time()

    before_count_a = 0
    before_count_d = 0
    rt_a_counter = 0
    rt_d_counter = 0
    prev_angle_y = 0

    direction = [
        'forward',
        'backward',
    ]

    # PWM制御
    pwm = PCA9685(0x40, debug=False)
    pwm.setPWMFreq(50)
    motor_driver = MotorDriver()
    pwm.setPWMFreq(50000000.0)

    # 制御パラメータ設定
    k1 = Decimal(str(363.41817119))
    k2 = Decimal(str(43.69756113))
    k3 = Decimal(str(7.46787171))
    k4 = Decimal(str(6.50978926))

    # モータ初期化
    direction_a = 0
    direction_d = 0
    motor_driver.MotorRun(pwm, 0, direction[direction_a], SPEED)
    motor_driver.MotorRun(pwm, 1, direction[direction_d], SPEED)

    getcontext().prec = 28
    while True:
        # 振子の回転角度を取得
        angle_y = Decimal(str(th_angle_y.value))
        delta_angle_y = Decimal(angle_y - prev_angle_y)
        rt_a_counter = rotary_encoder_a.steps
        rt_d_counter = rotary_encoder_d.steps
        # １ステップ前からのモータ回転角度
        delta_theta_a = Decimal(abs(abs(rt_a_counter) - abs(before_count_a)))
        delta_theta_d = Decimal(abs(abs(rt_d_counter) - abs(before_count_d)))
        if rotation_a == BCK:
            delta_theta_a = -delta_theta_a
        if rotation_d == BCK:
            delta_theta_d = -delta_theta_d
        # motorの振子に対する角度
        motor_a_angle = delta_theta_a - delta_angle_y
        motor_d_angle = delta_theta_d - delta_angle_y
        '''
        if np.sign(delta_theta_a) == np.sign(delta_angle_y):
            motor_a_angle = delta_theta_a - delta_angle_y
        else:
            motor_a_angle = delta_angle_y - delta_theta_a
        if np.sign(delta_theta_d) == np.sign(delta_angle_y):
            motor_d_angle = delta_theta_d - delta_angle_y
        else:
            motor_d_angle = delta_angle_y - delta_theta_d
        '''
        delta_time = Decimal(time.time() - start)
        start = time.time()
        # 振子の角速度
        if delta_angle_y == 0:
            angular_velocity_y = Decimal(0)
        else:
            angular_velocity_y = Decimal(delta_angle_y/delta_time)
        # モータの角速度
        velocity_a = Decimal(delta_theta_a/delta_time)
        velocity_d = Decimal(delta_theta_d/delta_time)

        # U(t)を導出
        ua = ((angle_y * k1) + (angular_velocity_y * k2) + (motor_a_angle * k3) + (velocity_a * k4))
        ud = ((angle_y * k1) + (angular_velocity_y * k2) + (motor_d_angle * k3) + (velocity_d * k4))

        # 回転方向を設定
        if ua < 0:
            direction_a = BCK
        else:
            direction_a = FWD

        if ud < 0:
            direction_d = BCK
        else:
            direction_d = FWD
        ua = abs(ua)
        ud = abs(ud)

        # 系に対してPWMデューティ比として与える制御入力をスケーリング
        if ua != U_MIN:
            ua = round(Decimal(0.02) * ua)
        if ud != U_MIN:
            ud = round(Decimal(0.02) * ud)

        # デューティ比の最大値を超過する場合は足切り
        if ua > U_MAX:
            ua = U_MAX
        if ud > U_MAX:
            ud = U_MAX

        print("angle_y  : " + str(angle_y)
                    + "\tangular_velocity_y  : " + str(angular_velocity_y)
                    + "\t motor angle : " + str(motor_a_angle)
                    + "\t motor velocity : " + str(velocity_a)
                    + "\t motor delta_theta : " + str(delta_theta_a)
                    + "\t direction : " + direction[direction_a]
                    + "\t u(t)  : " + str(ua)
                    + "\t delta : " + str(delta_time))

        # deleta theata計測用
        before_count_a = rt_a_counter
        before_count_d = rt_d_counter
        prev_angle_y = angle_y

        # モータへデューティ比を印加
        motor_driver.MotorRun(pwm, 0, direction[direction_a], ua)
        motor_driver.MotorRun(pwm, 1, direction[direction_d], ua)
        # ハードウェア側の応答を見て待ち時間を設定
        #time.sleep(0.002)

if __name__ == "__main__":
    th_angle_y = Value('i', 0)
    p1 = Process(target=request_controler, args=[th_angle_y])
    p2 = Process(target=read_sense_hat_angle, args=[th_angle_y])
    p1.start()
    p2.start()


import math
import os
import tempfile
from copy import deepcopy
from unittest import TestCase

import torch
from mmcv.runner import load_checkpoint, save_checkpoint

from mmcls.models.backbones import DistilledVisionTransformer
from .utils import timm_resize_pos_embed

class TestDeiT(TestCase):

    def setUp(self):
        self.cfg = dict(
            arch='deit-base', img_size=224, patch_size=16, drop_rate=0.1)

    def test_init_weights(self):
        # test weight init cfg
        cfg = deepcopy(self.cfg)
        cfg['init_cfg'] = [
            dict(
                type='Kaiming',
                layer='Conv2d',
                mode='fan_in',
                nonlinearity='linear')
        ]
        model = DistilledVisionTransformer(**cfg)
        ori_weight = model.patch_embed.projection.weight.clone().detach()
        # The pos_embed is all zero before initialize
        self.assertTrue(torch.allclose(model.dist_token, torch.tensor(0.)))

        model.init_weights()
        initialized_weight = model.patch_embed.projection.weight
        self.assertFalse(torch.allclose(ori_weight, initialized_weight))
        self.assertFalse(torch.allclose(model.dist_token, torch.tensor(0.)))

        # test load checkpoint
        pretrain_pos_embed = model.pos_embed.clone().detach()
        tmpdir = tempfile.gettempdir()
        checkpoint = os.path.join(tmpdir, 'test.pth')
        save_checkpoint(model, checkpoint)
        cfg = deepcopy(self.cfg)
        model = DistilledVisionTransformer(**cfg)
        load_checkpoint(model, checkpoint, strict=True)
        self.assertTrue(torch.allclose(model.pos_embed, pretrain_pos_embed))

        # test load checkpoint with different img_size
        cfg = deepcopy(self.cfg)
        cfg['img_size'] = 384
        model = DistilledVisionTransformer(**cfg)
        load_checkpoint(model, checkpoint, strict=True)
        resized_pos_embed = timm_resize_pos_embed(
            pretrain_pos_embed, model.pos_embed, num_tokens=2)
        self.assertTrue(torch.allclose(model.pos_embed, resized_pos_embed))

        os.remove(checkpoint)

    def test_forward(self):
        imgs = torch.randn(3, 3, 224, 224)

        # test with_cls_token=False
        cfg = deepcopy(self.cfg)
        cfg['with_cls_token'] = False
        cfg['output_cls_token'] = True
        with self.assertRaisesRegex(AssertionError, 'but got False'):
            DistilledVisionTransformer(**cfg)

        cfg = deepcopy(self.cfg)
        cfg['with_cls_token'] = False
        cfg['output_cls_token'] = False
        model = DistilledVisionTransformer(**cfg)
        outs = model(imgs)
        self.assertIsInstance(outs, tuple)
        self.assertEqual(len(outs), 1)
        patch_token = outs[-1]
        self.assertEqual(patch_token.shape, (3, 768, 14, 14))

        # test with output_cls_token
        cfg = deepcopy(self.cfg)
        model = DistilledVisionTransformer(**cfg)
        outs = model(imgs)
        self.assertIsInstance(outs, tuple)
        self.assertEqual(len(outs), 1)
        patch_token, cls_token, dist_token = outs[-1]
        self.assertEqual(patch_token.shape, (3, 768, 14, 14))
        self.assertEqual(cls_token.shape, (3, 768))
        self.assertEqual(dist_token.shape, (3, 768))

        # test without output_cls_token
        cfg = deepcopy(self.cfg)
        cfg['output_cls_token'] = False
        model = DistilledVisionTransformer(**cfg)
        outs = model(imgs)
        self.assertIsInstance(outs, tuple)
        self.assertEqual(len(outs), 1)
        patch_token = outs[-1]
        self.assertEqual(patch_token.shape, (3, 768, 14, 14))

        # Test forward with multi out indices
        cfg = deepcopy(self.cfg)
        cfg['out_indices'] = [-3, -2, -1]
        model = DistilledVisionTransformer(**cfg)
        outs = model(imgs)
        self.assertIsInstance(outs, tuple)
        self.assertEqual(len(outs), 3)
        for out in outs:
            patch_token, cls_token, dist_token = out
            self.assertEqual(patch_token.shape, (3, 768, 14, 14))
            self.assertEqual(cls_token.shape, (3, 768))
            self.assertEqual(dist_token.shape, (3, 768))

        # Test forward with dynamic input size
        imgs1 = torch.randn(3, 3, 224, 224)
        imgs2 = torch.randn(3, 3, 256, 256)
        imgs3 = torch.randn(3, 3, 256, 309)
        cfg = deepcopy(self.cfg)
        model = DistilledVisionTransformer(**cfg)
        for imgs in [imgs1, imgs2, imgs3]:
            outs = model(imgs)
            self.assertIsInstance(outs, tuple)
            self.assertEqual(len(outs), 1)
            patch_token, cls_token, dist_token = outs[-1]
            expect_feat_shape = (math.ceil(imgs.shape[2] / 16),
                                 math.ceil(imgs.shape[3] / 16))
            self.assertEqual(patch_token.shape, (3, 768, *expect_feat_shape))
            self.assertEqual(cls_token.shape, (3, 768))
            self.assertEqual(dist_token.shape, (3, 768))


import re
import pathlib
from setuptools import setup, find_packages
from setuptools.command.test import test as TestCommand

BASE_PATH = pathlib.Path(__file__).parent
try:
    version = re.findall(r"""^__version__ = "([^']+)"\r?$""",
                         (BASE_PATH / "aioftp" / "__init__.py").read_text(),
                         re.M)[0]
except IndexError:
    raise RuntimeError("Unable to determine version.")

class NoseTestCommand(TestCommand):

    def finalize_options(self):
        TestCommand.finalize_options(self)
        self.test_args = []
        self.test_suite = True

    def run_tests(self):
        import pathlib
        import shutil
        import nose

        for path in pathlib.Path("tests").glob("*"):
            if path.is_dir():
                shutil.rmtree(str(path))
        cov = pathlib.Path(".coverage")
        if cov.exists():
            cov.unlink()
        nose.main(argv=["nosetests"])

setup(
    name="aioftp",
    version=version,
    description=("ftp client/server for asyncio"),
    long_description=(BASE_PATH / "README.rst").read_text(),
    classifiers=[
        "Development Status :: 5 - Production/Stable",
        "Programming Language :: Python",
        "Programming Language :: Python :: 3",
        "Topic :: Internet :: File Transfer Protocol (FTP)",
    ],
    author="pohmelie",
    author_email="multisosnooley@gmail.com",
    url="https://github.com/aio-libs/aioftp",
    license="Apache 2",
    packages=find_packages(),
    python_requires=" >= 3.5.3",
    install_requires=[],
    tests_require=["nose", "coverage", "trustme"],
    cmdclass={"test": NoseTestCommand},
    include_package_data=True
)


#!/usr/bin/env python3
import argparse
import concurrent.futures
import os
import time
import random
from concurrent.futures import ThreadPoolExecutor

import requests

from tqdm import tqdm

user_agents = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36",
    "Mozilla/5.0 (Windows NT 5.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36",
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36",
    "Mozilla/4.0 (compatible; MSIE 9.0; Windows NT 6.1)",
    "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)",
    "Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Gecko",
    "Mozilla/5.0 (Windows NT 6.2; WOW64; Trident/7.0; rv:11.0) like Gecko",
    "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0)",
    "Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)",
    "Mozilla/5.0 (Windows NT 6.1; Win64; x64; Trident/7.0; rv:11.0) like Gecko",
    "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)",
    "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)",
    "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)",
]
visitvsco = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
    "Accept-Encoding": "gzip, deflate",
    "Accept-Language": "en-US,en;q=0.9",
    "Connection": "keep-alive",
    "Host": "vsco.co",
    "Upgrade-Insecure-Requests": "1",
    "User-Agent": random.choice(user_agents),
}

visituserinfo = {
    "Accept": "*/*",
    "Accept-Encoding": "gzip, deflate",
    "Accept-Language": "en-US,en;q=0.9",
    "Connection": "keep-alive",
    "Host": "vsco.co",
    "Referer": "https://vsco.co/bob/images/1",
    "User-Agent": random.choice(user_agents),
}

media = {
    "Accept": "*/*",
    "Accept-Encoding": "gzip, deflate",
    "Accept-Language": "en-US,en;q=0.9",
    "Connection": "keep-alive",
    "Host": "vsco.co",
    "Referer": "https://vsco.co/bob/images/1",
    "User-Agent": random.choice(user_agents),
    "X-Client-Build": "1",
    "X-Client-Platform": "web",
}

class Scraper(object):
    def __init__(self, username):
        self.username = username
        self.session = requests.Session()
        self.session.get(
            "https://vsco.co/content/Static/userinfo?callback=jsonp_%s_0"
            % (str(round(time.time() * 1000))),
            headers=visituserinfo,
        )
        self.uid = self.session.cookies.get_dict()["vs"]
        path = os.path.join(os.getcwd(), self.username)
        # If the path doesn't exist, then create it
        if not os.path.exists(path):
            os.makedirs(path)
        os.chdir(path)
        self.path = path
        self.newSiteId()
        self.buildJSON()
        self.totalj = 0
        self.imagelist = []

    def newSiteId(self):
        """
        Gets the unique id used per vsco user, it can be found in the cookies
        :params: none
        :return: returns the site id of a user in case you were curious
        """
        res = self.session.get(
            "https://vsco.co/ajxp/%s/2.0/sites?subdomain=%s" % (self.uid, self.username)
        )
        self.siteid = res.json()["sites"][0]["id"]
        self.sitecollectionid = res.json()["sites"][0]["site_collection_id"]
        return self.siteid

    def buildJSON(self):
        """
        Creates the urls used to grab json data for a user
        :params: none
        :return: returns the main media url by default
        """
        self.mediaurl = "https://vsco.co/ajxp/%s/2.0/medias?site_id=%s" % (
            self.uid,
            self.siteid,
        )
        self.journalurl = "https://vsco.co/ajxp/%s/2.0/articles?site_id=%s" % (
            self.uid,
            self.siteid,
        )
        self.collectionurl = "https://vsco.co/ajxp/%s/2.0/collections/%s/medias?" % (
            self.uid,
            self.sitecollectionid,
        )
        return self.mediaurl

    def getCollection(self):
        """
        Downloads the collection posts from the user
        :params: none
        :return: none
        """
        self.imagelist = []
        path = os.path.join(os.getcwd(), "collection")
        if not os.path.exists(path):
            os.makedirs(path)
        os.chdir(path)
        self.getCollectionList()
        with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
            future_to_url = {
                executor.submit(self.download_img_normal, lists): lists
                for lists in self.imagelist
            }
            for future in tqdm(
                    concurrent.futures.as_completed(future_to_url),
                    total=len(self.imagelist),
                    desc="Downloading collection posts of %s" % self.username,
                    unit=" posts",
            ):
                liste = future_to_url[future]
                try:
                    data = future.result()
                except Exception as exc:
                    print("%r crashed %s" % (liste, exc))
        os.chdir("..")

    def getCollectionList(self):
        """
        Starts setting up to download the collection

        Does magical stuff with the concurrent future
        :params: none
        :return: none
        """
        self.pbar = tqdm(
            desc="Finding new collection posts of %s" % self.username, unit=" posts"
        )
        with ThreadPoolExecutor(max_workers=5) as executor:
            future_to_url = {
                executor.submit(self.makeCollectionList, num): num for num in range(5)
            }
            for future in concurrent.futures.as_completed(future_to_url):
                num = future_to_url[future]
                try:
                    data = future.result()
                except Exception as exc:
                    print("%r crashed %s" % (num, exc))
        self.pbar.close()

    def makeCollectionList(self, num):
        """
        Determines what file type a media item is, then appends the correct url
        :params: num - this does some magic, no idea why it works or why I did it
        :return: a boolean on whether the list was successfully made
        """
        num += 1
        z = self.session.get(
            self.collectionurl,
            params={"size": 100, "page": num},
            headers=media,
        ).json()["medias"]

        count = len(z)
        while count > 0:
            for url in z:
                if (
                        "%s.jpg" % str(url["upload_date"])[:-3] in os.listdir()
                        or "%s.mp4" % str(url["upload_date"])[:-3] in os.listdir()
                ):
                    continue
                if url["is_video"] is True:
                    self.imagelist.append(
                        [
                            "https://%s" % url["video_url"],
                            str(url["upload_date"])[:-3],
                            True,
                        ]
                    )
                    self.pbar.update()
                else:
                    self.imagelist.append(
                        [
                            "https://%s" % url["responsive_url"],
                            str(url["upload_date"])[:-3],
                            False,
                        ]
                    )
                    self.pbar.update()
            num += 5
            z = self.session.get(
                self.collectionurl,
                params={"size": 100, "page": num},
                headers=media,
            ).json()["medias"]
            count = len(z)
        return True

    def getJournal(self):
        """
        Downloads the journal posts from the user
        :params: none
        :return: none
        """
        self.getJournalList()
        self.progbarj = tqdm(
            total=self.totalj,
            desc="Downloading journal posts of %s" % self.username,
            unit=" posts",
        )
        for x in self.works:
            path = os.path.join(os.getcwd(), x[0])
            if not os.path.exists(path):
                os.makedirs(path)
            os.chdir(path)
            x.pop(0)
            with ThreadPoolExecutor(max_workers=5) as executor:
                future_to_url = {
                    executor.submit(self.download_img_journal, part): part for part in x
                }
                for future in concurrent.futures.as_completed(future_to_url):
                    part = future_to_url[future]
                    try:
                        data = future.result()
                    except Exception as exc:
                        print("%r crashed %s" % (part, exc))
            os.chdir(os.path.normpath(os.getcwd() + os.sep + os.pardir))
        self.progbarj.close()

    def getJournalList(self):
        """
        Opens initial journal data of a user and creates the journal folder

        Then it does some magical bs, I made this years ago no idea how it works

        :params: none
        :return: none
        """
        self.works = []
        self.jour_found = self.session.get(
            self.journalurl, params={"size": 10000, "page": 1}, headers=media
        ).json()["articles"]
        self.pbarjlist = tqdm(
            desc="Finding new journal posts of %s" % self.username, unit=" posts"
        )
        for x in self.jour_found:
            self.works.append([x["permalink"]])
        path = os.path.join(os.getcwd(), "journal")
        if not os.path.exists(path):
            os.makedirs(path)
        os.chdir(path)
        with ThreadPoolExecutor(max_workers=5) as executor:
            future_to_url = {
                executor.submit(self.makeListJournal, len(self.jour_found), val): val
                for val in range(len(self.jour_found))
            }
            for future in concurrent.futures.as_completed(future_to_url):
                val = future_to_url[future]
                try:
                    data = future.result()
                except Exception as exc:
                    print("%r crashed %s" % (val, exc))
        self.pbarjlist.close()

    def makeListJournal(self, num, loc):
        """
        Makes the list of all journal entries on the users page
        :params: num, loc
        :return: a boolean on whether the journal media was able to be grabbed
        """
        for item in self.jour_found[loc]["body"]:
            if item["type"] == "image":
                if os.path.exists(
                        os.path.join(os.getcwd(), self.jour_found[loc]["permalink"])
                ):
                    if "%s.jpg" % str(item["content"][0]["id"]) in os.listdir(
                            os.path.join(os.getcwd(), self.jour_found[loc]["permalink"])
                    ):
                        continue
                self.works[loc].append(
                    [
                        "https://%s" % item["content"][0]["responsive_url"],
                        item["content"][0]["id"],
                        "img",
                    ]
                )
            elif item["type"] == "video":
                if os.path.exists(
                        os.path.join(os.getcwd(), self.jour_found[loc]["permalink"])
                ):
                    if "%s.mp4" % str(item["content"][0]["id"]) in os.listdir(
                            os.path.join(os.getcwd(), self.jour_found[loc]["permalink"])
                    ):
                        continue
                self.works[loc].append(
                    [
                        "https://%s" % item["content"][0]["video_url"],
                        item["content"][0]["id"],
                        "vid",
                    ]
                )
            elif item["type"] == "text":
                if os.path.exists(
                        os.path.join(os.getcwd(), self.jour_found[loc]["permalink"])
                ):
                    if "%s.txt" % str(item["content"]) in os.listdir(
                            os.path.join(os.getcwd(), self.jour_found[loc]["permalink"])
                    ):
                        continue
                self.works[loc].append([item["content"], "txt"])
            self.totalj += 1
            self.pbarjlist.update()
        return True

    def download_img_journal(self, lists):
        """
        Downloads the journal media in specified ways depending on the type of media

        Since Journal items can be text files, images, or videos, I had to make 3
        different ways of downloading

        :params: lists - No idea why I named it this, but it's a media item
        :return: a boolean on whether the journal media was able to be downloaded
        """
        if lists[1] == "txt":
            with open("%s.txt" % str(lists[0]), "w") as f:
                f.write(lists[0])
        if lists[2] == "img":
            if "%s.jpg" % lists[1] in os.listdir():
                return True
            with open("%s.jpg" % str(lists[1]), "wb") as f:
                f.write(requests.get(lists[0], stream=True).content)

        elif lists[2] == "vid":
            if "%s.mp4" % lists[1] in os.listdir():
                return True
            with open("%s.mp4" % str(lists[1]), "wb") as f:
                for chunk in requests.get(lists[0], stream=True).iter_content(
                        chunk_size=1024
                ):
                    if chunk:
                        f.write(chunk)
        self.progbarj.update()
        return True

    def getImages(self):
        """
        Makes a list of all media items in a page

        I clearly recall straight copy pasting the concurrent futures thing from stack
        overflow. Still have no idea how it works

        :params: none
        :return: none
        """
        self.imagelist = []
        self.getImageList()
        with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
            future_to_url = {
                executor.submit(self.download_img_normal, lists): lists
                for lists in self.imagelist
            }
            for future in tqdm(
                    concurrent.futures.as_completed(future_to_url),
                    total=len(self.imagelist),
                    desc="Downloading posts of %s" % self.username,
                    unit=" posts",
            ):
                liste = future_to_url[future]
                try:
                    data = future.result()
                except Exception as exc:
                    print("%r crashed %s" % (liste, exc))

    def getImageList(self):
        """
        tqdm is a great library isn't it

        :params: none
        :return: none
        """
        self.pbar = tqdm(desc="Finding new posts of %s" % self.username, unit=" posts")
        with ThreadPoolExecutor(max_workers=5) as executor:
            future_to_url = {
                executor.submit(self.makeImageList, num): num for num in range(5)
            }
            for future in concurrent.futures.as_completed(future_to_url):
                num = future_to_url[future]
                try:
                    data = future.result()
                except Exception as exc:
                    print("%r crashed %s" % (num, exc))
        self.pbar.close()

    def makeImageList(self, num):
        """
        At this point I'm only doing these comments out of a general sense I should comment
        all these functions after that one PR did it

        Don't ask me how it works, frankly I'm surprised it does

        :params: num - I remember looking at this after starting to merge the prs and wondering why I did num += 5. Still couldn't tell you.
        :return: a boolean on whether the journal media was able to be grabbed
        """
        num += 1
        z = self.session.get(
            self.mediaurl, params={"size": 100, "page": num}, headers=media
        ).json()["media"]
        count = len(z)
        while count > 0:
            for url in z:
                if (
                        "%s.jpg" % str(url["upload_date"])[:-3] in os.listdir()
                        or "%s.mp4" % str(url["upload_date"])[:-3] in os.listdir()
                ):
                    continue
                if url["is_video"] is True:
                    self.imagelist.append(
                        [
                            "https://%s" % url["video_url"],
                            str(url["upload_date"])[:-3],
                            True,
                        ]
                    )
                    self.pbar.update()
                else:
                    self.imagelist.append(
                        [
                            "https://%s" % url["responsive_url"],
                            str(url["upload_date"])[:-3],
                            False,
                        ]
                    )
                    self.pbar.update()
            num += 5
            z = self.session.get(
                self.mediaurl,
                params={"size": 100, "page": num},
                headers=media,
            ).json()["media"]
            count = len(z)
        return True

    def download_img_normal(self, lists):
        """
        This function makes sense at least

        The if '%s.whatever' sections are to skip downloading the file again if it's already been downloaded

        At the time I wrote this, I only remember seeing that images and videos were the only things allowed

        So I didn't write an if statement checking for text files, so this would just skip it I believe if it ever came up
        and return True

        :params: lists - My naming sense was beat. lists is just a media item.
        :return: a boolean on whether the media item was downloaded successfully
        """
        if lists[2] is False:
            if "%s.jpg" % lists[1] in os.listdir():
                return True
            with open("%s.jpg" % str(lists[1]), "wb") as f:
                f.write(requests.get(lists[0], stream=True).content)
        else:
            if "%s.mp4" % lists[1] in os.listdir():
                return True
            with open("%s.mp4" % str(lists[1]), "wb") as f:
                for chunk in requests.get(lists[0], stream=True).iter_content(
                        chunk_size=1024
                ):
                    if chunk:
                        f.write(chunk)
        return True

    def run_all(self):
        """
        This runs all of the operations on a user's profile,
        it gets the images, the collection, and the journal entries
        :params: none
        :return: none
        """
        self.getImages()
        self.getCollection()
        self.getJournal()

def parser():
    """Returns the parser arguments
    :params: none
    :return: parser.parse_args() object
    """
    parser = argparse.ArgumentParser(
        description="Scrapes a specified users VSCO Account"
    )
    parser.add_argument(
        "username",
        type=str,
        help="VSCO user to scrape or file name to read usernames off of. Filename feature works with -m, -mc, -mj, and -a only",
    )
    parser.add_argument(
        "-s", "--siteId", action="store_true", help="Grabs VSCO siteID for user"
    )
    parser.add_argument(
        "-i", "--getImages", action="store_true", help="Get the pictures of the user"
    )
    parser.add_argument(
        "-j",
        "--getJournal",
        action="store_true",
        help="Get the journal images of the user",
    )
    parser.add_argument(
        "-c",
        "--getCollection",
        action="store_true",
        help="Get the collection images of the user",
    )
    parser.add_argument(
        "-m", "--multiple", action="store_true", help="Scrape multiple users"
    )
    parser.add_argument(
        "-mj",
        "--multipleJournal",
        action="store_true",
        help="Scrape multiple users journal",
    )
    parser.add_argument(
        "-mc",
        "--multipleCollection",
        action="store_true",
        help="Scrape multiple users collection",
    )
    parser.add_argument(
        "-a",
        "--all",
        action="store_true",
        help="Scrape multiple users journals and images",
    )
    return parser.parse_args()

def main():
    args = parser()

    if args.siteId:
        scraper = Scraper(args.username)
        print(scraper.newSiteId())

    if args.getImages:
        scraper = Scraper(args.username)
        scraper.getImages()

    if args.getJournal:
        scraper = Scraper(args.username)
        scraper.getJournal()

    if args.getCollection:
        scraper = Scraper(args.username)
        scraper.getCollection()

    if args.multiple:
        y = []
        vsco = os.getcwd()
        with open(args.username, "r") as f:
            for x in f:
                y.append(x.replace("\n", ""))
        for z in y:
            try:
                os.chdir(vsco)
                Scraper(z).getImages()
                print()
            except:
                print("%s crashed" % z)
                pass

    if args.multipleJournal:
        y = []
        vsco = os.getcwd()
        with open(args.username, "r") as f:
            for x in f:
                y.append(x.replace("\n", ""))
        for z in y:
            try:
                os.chdir(vsco)
                Scraper(z).getJournal()
                print()
            except:
                print("%s crashed" % z)
                pass

    if args.multipleCollection:
        y = []
        vsco = os.getcwd()
        with open(args.username, "r") as f:
            for x in f:
                y.append(x.replace("\n", ""))
        for z in y:
            try:
                os.chdir(vsco)
                Scraper(z).getCollection()
                print()
            except:
                print("%s crashed" % z)
                pass

    if args.all:
        y = []
        vsco = os.getcwd()
        with open(args.username, "r") as f:
            for x in f:
                y.append(x.replace("\n", ""))
        for z in y:
            try:
                os.chdir(vsco)
                Scraper(z).run_all()
                print()
            except:
                print("%s crashed" % z)
                pass

if __name__ == "__main__":
    main()


import torch

def qrot(q, v):
    """
    Rotate vector(s) v about the rotation described by quaternion(s) q.
    Expects a tensor of shape (*, 4) for q and a tensor of shape (*, 3) for v,
    where * denotes any number of dimensions.
    Returns a tensor of shape (*, 3).
    """
    assert q.shape[-1] == 4
    assert v.shape[-1] == 3
    assert q.shape[:-1] == v.shape[:-1]

    qvec = q[..., 1:]
    uv = torch.cross(qvec, v, dim=len(q.shape)-1)
    uuv = torch.cross(qvec, uv, dim=len(q.shape)-1)
    return (v + 2 * (q[..., :1] * uv + uuv))

def qinverse(q, inplace=False):
    # We assume the quaternion to be normalized
    if inplace:
        q[..., 1:] *= -1
        return q
    else:
        w = q[..., :1]
        xyz = q[..., 1:]
        return torch.cat((w, -xyz), dim=len(q.shape)-1)


from typing import Optional, List, Any
from langchain.llms.base import LLM
import torch
from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig

class HuggingFaceLoraPipeline(LLM):
    model: Any
    tokenizer: Any

    def _llm_type(self) -> str:
        return 'rinna-ppo'

    def _gen_prompt(self, prompt):
         return f"""ユーザー: {prompt}
                システム:
                """

    def _call(self,
              prompt: str,
              stop: Optional[List[str]] = None,
              **kwargs,
              ) -> str:
        prompt = self._gen_prompt(prompt)
        token_ids = self.tokenizer.encode(prompt, add_special_tokens=False, return_tensors="pt")
        with torch.no_grad():
            output_ids = self.model.generate(
                token_ids.to('cuda'),
                do_sample=True,
                max_new_tokens=128,
                temperature=0.7,
                repetition_penalty=1.1,
                pad_token_id=self.tokenizer.pad_token_id,
                bos_token_id=self.tokenizer.bos_token_id,
                eos_token_id=self.tokenizer.eos_token_id,
                **kwargs
            )

        output = self.tokenizer.decode(output_ids.tolist()[0][token_ids.size(1):])
        output = output.replace("<NL>", "\n")
        return output

def get_llm():
    model_name = "rinna/japanese-gpt-neox-3.6b-instruction-ppo"

    tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=False)

    nf4_config = BitsAndBytesConfig(
        load_in_4bit=True,
        bnb_4bit_quant_type="nf4",
        bnb_4bit_use_double_quant=True,
        bnb_4bit_compute_dtype=torch.bfloat16
    )

    model = AutoModelForCausalLM.from_pretrained(model_name, quantization_config=nf4_config, device_map="auto")
    llm = HuggingFaceLoraPipeline(
        model=model,
        tokenizer=tokenizer
    )
    return llm


"""Class definition for kwarg from command line parser"""

__author__ = "Guido Boehm"
__filename__ = "ParseKwargs.py"
__credits__ = [""]
__license__ = ""
__version__ = "0.0.1"
__maintainer__ = "Guido Boehm"
__email__ = "olb@family-boehm.de"
__status__ = "Prototype"
__copyright__ = """
Copyright 2022, Guido Boehm
All Rights Reserved.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.
"""

import argparse

class ParseKwargs(argparse.Action):
    def __call__(self, parser, namespace, values, option_string=None):
        setattr(namespace, self.dest, dict())
        for value in values:
            key, value = value.split('=')
            getattr(namespace, self.dest)[key] = value

def main(args=None):
    pass

if __name__ == "__main__":

    main()


"""

Retrieve images from the web

"""

import shutil
import requests
from tools.system_tools import img_dir
from PIL import Image, ImageTk, ImageGrab

#####################################
#                                   #
#   Saved URLs for updating plots   #
#                                   #
#####################################

# Pivotal Weather daily CONUS weather forecast images
ndfd_loc = "https://x-hv1.pivotalweather.com/maps/ndfd/latest/"
pw_conus_maxtemp_url = ndfd_loc + "ndfd_sfctmax.conus.png"
pw_conus_mintemp_url = ndfd_loc + "ndfd_sfctmin.conus.png"
pw_conus_24hrqpf_url = ndfd_loc + "ndfd_24hqpf.conus.png"
pw_conus_24hsnow_url = ndfd_loc + "ndfd_24hsnow.conus.png"
pw_conus_24hgust_url = ndfd_loc + "ndfd_24hgust.conus.png"
# Alicia Bentley maps (http://www.atmos.albany.edu/student/abentley/realtime.html)
#  CONUS Bentley maps
ab_con_loc = "http://www.atmos.albany.edu/student/abentley/realtime/images/northamer/"
ab_con_6hrprecip_url = ab_con_loc + "6hprecip/6hprecip_57.png"
ab_con_850thetae_url = ab_con_loc + "850_thetae/850_thetae_57.png"
ab_con_capeshear_url = ab_con_loc + "cape_shear/cape_shear_57.png"
ab_con_mslp_jets_url = ab_con_loc + "mslp_jet/mslp_jet_57.png"
#  Pacific Bentley maps
ab_pac_loc = "http://www.atmos.albany.edu/student/abentley/realtime/images/pacific/"
ab_pac_mslp_anom_url = ab_pac_loc + "mslp_anom/mslp_anom_57.png"
ab_pac_6hrprecip_url = ab_pac_loc + "6hprecip/6hprecip_57.png"
ab_pac_850thetae_url = ab_pac_loc + "850_thetae/850_thetae_57.png"
ab_pac_700wnd_pw_url = ab_pac_loc + "700wind_pw/700wind_pw_57.png"
# Coastal Data Information Program swell map
cdip_swell_url = "http://cdip.ucsd.edu/recent/model_images/monterey.png"
# Surfline coastal data
surfline_sst_url = "https://slcharts01.cdn-surfline.com/charts/cencal/monterey/nearshoresst/monterey_large_1_F.png"
surfline_wnd_url = "https://slcharts01.cdn-surfline.com/charts/cencal/monterey/nearshorewinds/monterey_large_1.png"
# NWS fire maps
nws_fire_outlook_url = "https://www.spc.noaa.gov/products/fire_wx/day1otlk_fire.gif"
nifc_fire_month_url  = "https://www.predictiveservices.nifc.gov/outlooks/month1_outlook.png"

# Local directory for downloaded images
# img_dir = config.locs.image_dir
# Each image data (source url, local dir+filename)
pw_conus_maxtemp = ( pw_conus_maxtemp_url, img_dir+"pw_conus_maxtemp.png" )
pw_conus_mintemp = ( pw_conus_mintemp_url, img_dir+"pw_conus_mintemp.png" )
pw_conus_24hrqpf = ( pw_conus_24hrqpf_url, img_dir+"pw_conus_24hrqpf.png" )
pw_conus_24hsnow = ( pw_conus_24hsnow_url, img_dir+"pw_conus_24hsnow.png" )
pw_conus_24hgust = ( pw_conus_24hgust_url, img_dir+"pw_conus_24hgust.png" )
ab_con_6hrprecip = ( ab_con_6hrprecip_url, img_dir+"ab_con_6hrprecip.png" )
ab_con_850thetae = ( ab_con_850thetae_url, img_dir+"ab_con_850thetae.png" )
ab_con_capeshear = ( ab_con_capeshear_url, img_dir+"ab_con_capeshear.png" )
ab_con_mslp_jets = ( ab_con_mslp_jets_url, img_dir+"ab_con_mslp_jets.png" )
ab_pac_mslp_anom = ( ab_pac_mslp_anom_url, img_dir+"ab_pac_mslp_anom.png" )
ab_pac_6hrprecip = ( ab_pac_6hrprecip_url, img_dir+"ab_pac_6hrprecip.png" )
ab_pac_850thetae = ( ab_pac_850thetae_url, img_dir+"ab_pac_850thetae.png" )
ab_pac_700wnd_pw = ( ab_pac_700wnd_pw_url, img_dir+"ab_pac_700wnd_pw.png" )
cdip_swell       = ( cdip_swell_url,       img_dir+"cdip_swell.png"       )
surfline_sst     = ( surfline_sst_url,     img_dir+"surfline_sst.png"     )
surfline_wnd     = ( surfline_wnd_url,     img_dir+"surfline_wnd.png"     )
nws_fire_outlook = ( nws_fire_outlook_url, img_dir+"nws_fire_outlook.png" )
nifc_fire_month  = ( nifc_fire_month_url,  img_dir+"nifc_fire_month.png"  )

# All image data
images = ( pw_conus_maxtemp, pw_conus_mintemp, pw_conus_24hrqpf,
           pw_conus_24hgust, ab_con_6hrprecip, ab_con_850thetae,
           ab_con_capeshear, ab_con_mslp_jets, ab_pac_mslp_anom,
           ab_pac_6hrprecip, ab_pac_850thetae, ab_pac_700wnd_pw,
           cdip_swell, surfline_sst, surfline_wnd, nws_fire_outlook,
           nifc_fire_month )

def get_img( url, filename ):
    try:
        r = requests.get( url, stream = True )
    except:
        return
    # Check if the image was retrieved successfully
    if r.status_code == 200:
        # Set decode_content value to True, otherwise the downloaded image file's size will be zero.
        r.raw.decode_content = True
        with open(filename,'wb') as f:
            shutil.copyfileobj(r.raw, f)
    else:
        print(url)
        raise RuntimeWarning("Couldn't download image!")

def crop_bentleys( path, debug=False ):
    # Opens a image in RGB mode
    im = Image.open(path)
    # Crop specs for removing unnecessary vars from plot
    im1 = im.crop((100, 0, 1060, 791))
    # Debug mode for checking crop specs
    if debug:
        # Shows the image in image viewer
        im1.show()
    else:
        # Overwrite previous image
        im1.save(path)

def get_imgs( ):
    # Retrieves all listed images
    for img in images:
        get_img( img[0], img[1] )
    for img in images[4:12]:
        crop_bentleys( img[1] )

def get_pws( ):
    for img in images[:4]:
        get_img( img[0], img[1] )

def get_bentleys( ):
    for img in images[4:12]:
        get_img( img[0], img[1] )
        crop_bentleys( img[1] )

def get_marine( ):
    for img in images[12:15]:
        get_img( img[0], img[1] )

if __name__ == "__main__":
    #get_img( pw_conus_maxtemp[0], img_dir+"test.png" )
    get_imgs()


#!/usr/bin/python

import os
import requests
import json
import subprocess
from time import sleep

def getextensions(cookie, number):
    headers = {'Cookie': cookie}
    URL = 'https://chrome.google.com/webstore/ajax/item?hl=en-US&gl=US&pv=20170206&mce=atf%2Ceed%2Cpii%2Crtr%2Crlb%2Cgtc%2Chcn%2Csvp%2Cwtd%2Cctm%2Cac%2Chot%2Cmac%2Cfcf%2Crma%2Cigb%2Cpot%2Cevt&count=200&token={0}%401954920&category=extensions&sortBy=0&container=CHROME&features=5&_reqid=3376862&rt=j'
    count = 0

    results = []
    while count < number:
        r = requests.post(URL.format(count), headers=headers)
        parsed = json.loads(r.text[r.text.find('['):])
        results.extend(((x[0], x[1], x[11]) for x in parsed[0][1][1]))
        count += 200
    return results

def getcrx(extension_id):
    CHROME_VERSION = '54.0.2840.99'
    DOWNLOADS = "downloads"

    if (not os.path.exists("%s/%s" %(DOWNLOADS, extension_id))):
        r = requests.get('https://clients2.google.com/service/update2/crx?response=redirect&prodversion=%s&x=id%%3D%s%%26uc' %(CHROME_VERSION, extension_id), timeout=10)
        with open('extension.crx', 'wb') as fd:
            fd.write(r.content)
        subprocess.call(["unzip", "-o", "extension.crx", "-d", "%s/%s" %(DOWNLOADS, extension_id)])
        print("processed %s" %extension_id)
    else:
        print("skipping...")

def fromfile():
    results = []
    with open("list.csv", 'r') as fd:
        for l in fd.readlines():
            results.append(tuple(l.strip('\n').split(',')))
    return results

def createlist(cookie):
    extensions = getextensions(cookie, 5000)
    with open("list.csv", 'w') as fd:
        for l in extensions:
            fd.write(",".join(l).encode('utf-8'))
            fd.write("\n")

def getall():
    extensions = fromfile()
    count = 0
    for x in extensions:
        count += 1
        print("processing extension %d" %count)
        getcrx(x[0])
        # sleep(0.5)

getall()


'''
Created on Jan 14, 2017

@author: User
'''
import string
import itertools
import operator
#word = input("Enter text: ")
def most_common(word):
    keyVal = {}
    if len(word) > 3 and len(word) < 10000:
        for i in word:
            keyVal[i] = word.count(i)
        sortedKeyVal = sorted(keyVal.items(), key = lambda x:x[1], reverse=True)
        #you also do. Make you you import the operator module before you do this
        #sortedKeyVal = sorted(keyVal.items(), key = operator.itemgetter(1), reverse=True) #change the itemgetter to zero to sort with the keys
    for key, value in itertools.islice(sortedKeyVal, 3):
        print(key, value)

#most_common(word)

def most_common2(word):
    diction = {}
    for char in word:
        diction[char] = word.count(char)
    sorted_dict = sorted(diction.items(), key=operator.itemgetter(1), reverse=True)
    keys = list(diction.keys())
    keys.sort()

    sorted_dict = dict(sorted_dict)
    limit = 0
    while len(sorted_dict) > 0:
        highest_count = 0
        highest_word = ""
        position = 0
        for x in range(len(keys)):
            if sorted_dict[keys[x]] > highest_count:
                highest_count = sorted_dict[keys[x]]
                highest_word = keys[x]
                position = x
        del sorted_dict[highest_word]
        del keys[position]
        print(highest_word+" %d" % highest_count)
        limit += 1
        if limit > 2:
            break

most_common(input("Enter a string\n"))


from bs4 import BeautifulSoup as Soup, Tag
import requests

response = requests.get("https://guide.wisc.edu/courses/#text")
soup = Soup(response.content, "html.parser")

index = soup.find("div", {"id": "atozindex"})
f = open("../data/subjectpaths.csv", "w")

for item in index.find_all('li'):
    if isinstance(item, Tag):
        link = item.find('a')
        f.write(link['href'] + ",")

f.close()


import sys
from PyQt5.QtWidgets import QApplication
from PyQt5.QtCore import QTimer, QCoreApplication

from splash import SplashScreen
from wordle import MainWindow

# base_path = os.path.abspath(os.path.dirname(__file__))

try:
  from ctypes import windll
  myappid = "soundwave.wordle.1.0"
  windll.shell32.SetCurrentProcessExplicitAppUserModelID(myappid)
except ImportError:
  pass

def run_app1():
    if not QCoreApplication.instance():
        app = QApplication(sys.argv)
    else:
        app = QCoreApplication.instance()

    splash = SplashScreen()
    splash.show()

    # Schedule the closeSplashScreen function to be called after 5 seconds
    QTimer.singleShot(2300, lambda: closeSplashScreen(app, splash))

    app.exec_()

def closeSplashScreen(app, splash):
    # Close the splash screen and open the main window
    splash.close()
    window = MainWindow()
    window.show()

def run_app2():
    # print("Running app2")
    if not QCoreApplication.instance():
        app = QApplication(sys.argv)
    else:
        app = QCoreApplication.instance()

    window = MainWindow()
    window.show()

    app.exec_()

if __name__ == '__main__':
    # print("Starting app1")
    run_app1()

    # print("Starting app2")
    run_app2()

    # print("Script execution complete")


import logging
import os
import pickle
import collections

from src.storage.AbstractStorage import AbstractStorage

class BinaryFileStorage(AbstractStorage):

    def __init__(self, base_dir):
        self.base_dir = base_dir

    def store(self, source, texts):
        if not os.path.exists(self.base_dir):
            os.makedirs(self.base_dir)

        file_name = os.path.join(self.base_dir, '{}.bin'.format(source))

        with open(file_name, 'wb') as file:
            logging.info("Writing to file: {}".format(file))
            pickle.dump(texts, file)

        logging.info('Successfully saved {} posts to: {}/{}'.format(len(texts),
                                                        self.base_dir, source))

    def get(self, sources):
        texts = []

        if not isinstance(sources, collections.Iterable):
            sources = [sources]

        for source in sources:
            file_name = os.path.join(self.base_dir, '{}.bin'.format(source))

            with open(file_name, 'rb') as file:
                logging.info("Reading file: {}".format(file))
                texts += pickle.load(file)

        return texts


#importando aa bibliotecas e funções
import sqlalchemy
from sqlalchemy.orm import declarative_base
from sqlalchemy.orm import relationship
from sqlalchemy.orm import Session
from sqlalchemy import create_engine
from sqlalchemy import inspect
from sqlalchemy import select
from sqlalchemy import func
from sqlalchemy import Column
from sqlalchemy import Integer
from sqlalchemy import String
from sqlalchemy import ForeignKey

Base = declarative_base()

class User(Base):
    __tablename__ = "user_account"
    # atributos
    id = Column(Integer, primary_key=True)
    name = Column(String)
    full_name = Column(String)

    address = relationship(
        "Address", back_populates="user", cascade="all, delete-orphan"
    )

    def __repr__(self):
        return f"User(id={self.id}, name={self.name}, full_name={self.full_name})"

class Address(Base):
    __tablename__ = "address"
    # atributos
    id = Column(Integer, primary_key=True, autoincrement=True)
    email_address = Column(String(50), nullable=False)
    user_id = Column(Integer, ForeignKey("user_account.id"), nullable=False)

    user = relationship(
        "User", back_populates="address"
    )

    def __repr__(self):
        return f"Address(id={self.id}, email_address={self.email_address})"

# print(User.__tablename__)
# print(Address.__table__)

# conexão com banco de dados

engine = create_engine("sqlite://")

# criando as classes como tabela no BD
Base.metadata.create_all(engine)

# investigando o esquema do BD
insp = inspect(engine)
print(insp.get_table_names())
print(insp.default_schema_name)

with Session(engine) as session:
    guilherme = User(
        name='guilherme',
        full_name='Guilherme Roth',
        address=[Address(email_address='guilhermerth@hotmail.com')]
    )

    fiodor = User(
        name='fiódor',
        full_name='Fiódor Dostoiévsky',
        address=[Address(email_address='fiodor_dost@karamazov.com'),
                 Address(email_address='fdostoievsky@raskolnikov.com')]
    )

    machado = User(
        name='machado',
        full_name='Joaquim Maria Machado de Assis'
        )

    # enviando para o BD (persistência de dados)
    session.add_all([guilherme, fiodor, machado])

    session.commit()

stmt = select(User).where(User.name.in_(["machado"]))
print('\nRecuperando dados a partir de condição de filtragem.')
for user in session.scalars(stmt):
    print(user)

stmt_address = select(Address).where(Address.user_id.in_([2]))
print('\nRecuperando dados a partir de condição de filtragem.')
for address in session.scalars(stmt_address):
    print(address)

print(f'\n{select(User).order_by(User.full_name.desc())}')

stmt_order = select(User).order_by(User.full_name.desc())
print('\nRecuperando dados de maneira ordenada.')
for full_name in session.scalars(stmt_order):
    print(full_name)

stmt_join = select(User.full_name, Address.email_address).join_from(Address, User)
print('\nRecuperando dados a partir de condição de filtragem.')
for result in session.scalars(stmt_join):
    print(result)

# print(select(User.full_name, Address.email_address).join_from(Address, User))

connection = engine.connect()
results = connection.execute(stmt_join).fetchall()
print('\nExecutando statement a partir da connection.')
for result in results:
    print(result)

# print(f"\n{select(func.count('*')).select_from(User)}")

stmt_count = select(func.count('*')).select_from(User)
print('\nRetornando o total de instâncias em User')
for count in session.scalars(stmt_count):
    print(count)