import sys
import os
import re
from fractions import Fraction
from enum import Enum
from collections import namedtuple

Time = namedtuple('Time', ['upper', 'lower', 'hyphen'])

class Note:
    """Member variables

    acc: accidental
        None, -1 (flat), 1 (sharp), 0 (natural)
    name: 0-7 as in numbered musical notation
    octave: int
    duration: Fraction
    line: # dash or underline
        line = 0: duration = 1
        line < 0: duration = 2 ** line
        line > 0: dash = line
    dot: # dots
    tie: 2-tuple of bool
        tie[0] = True: tied with the previous Note
        tie[1] = True: tied with the next Note
    """

    possible_ends = {}

    def __init__(self, acc, name, octave=0, duration=Fraction(1), dashes=0, underlines=0, dots=0, tie=(False, False)):
        self.acc = acc
        self.name = name
        self.octave = octave
        self.duration = duration
        if dashes is None:
            self.lines = None
            self.dots = None
        else:
            if dashes > 0:
                self.lines = dashes
            else:
                self.lines = -underlines
            self.dots = dots
        self.tie = list(tie)

    def copy(self):
        note = Note(self.acc, self.name, self.octave, self.duration)
        note.lines = self.lines
        note.dots = self.dots
        note.tie = self.tie.copy()
        return note

    @classmethod
    def init_possible_ends(cls, n, m):
        """Each beat is divided into (2**n) intervals, and there are m beats.

        Every duration consists of main duration and dots duration.
        The main duration consists of (2**k) intervals.
        """
        if (n, m) in cls.possible_ends:
            return
        num = m << n
        ends = [set() for _ in range(num)]
        # k < n
        for k in range(n):
            for a in range(0, num, 1 << k):
                start = a
                end = a + (1 << k)
                if end > num:
                    break
                ends[start].add(end)
                if (a >> k) % 2 == 0 and k > 0 and end + (1 << (k - 1)) < num:
                    ends[start + (1 << (k - 1))].add(end + (1 << (k - 1)))
                if (a >> k) % 2 == 1:
                    for j in range(k - 1, -1, -1):
                        start -= (1 << j)
                        if start < 0:
                            break
                        ends[start].add(end)
                else:
                    for j in range(k - 1, -1, -1):
                        end += (1 << j)
                        if end > num:
                            break
                        ends[start].add(end)
        # k >= n
        for k in range(n, n + m.bit_length()):
            for a in range(0, num, 1 << n):
                start = a
                end = a + (1 << k)
                if end > num:
                    break
                ends[start].add(end)
                if k > 0 and end + (1 << (k - 1)) < num:
                    ends[start + (1 << (k - 1))].add(end + (1 << (k - 1)))
                for j in range(k - 1, -1, -1):
                    start -= (1 << j)
                    if start < 0:
                        break
                    ends[start].add(end)
                start = a
                for j in range(k - 1, -1, -1):
                    end += (1 << j)
                    if end > num:
                        break
                    ends[start].add(end)
        for i in range(num):
            ends[i] = sorted(ends[i])
        cls.possible_ends[(n, m)] = ends

    @classmethod
    def split_note(cls, time, start_beat, note, _debug=False):
        debug_log = []
        if not time.hyphen:
            raise ValueError('no need to split note for time {}'.format(time))
        if time.upper is None:
            raise ValueError('cannot split note for time ?/{}'.format(time.lower))
        note.lines, note.dots = None, None      # ignore lines and dots
        duration = note.duration
        p = time.hyphen.bit_length() - 3

        if time.lower == 4:
            unit = (1 << p)
            if time.upper == 2:
                n, m = p + 1, 1
            elif time.upper == 3:
                n, m = p, 3
            elif time.upper == 4:
                n, m = p + 2, 1
            else:
                raise ValueError('unknown time.upper {}'.format(time.upper))
            n_unit, m_unit = 0, time.upper
        else:
            unit = int(Fraction(3, 2) * (1 << p))
            n, m = p - 1, 3
            if time.upper == 6:
                n_unit, m_unit = 0, 2
            elif time.upper == 9:
                n_unit, m_unit = 0, 3
            elif time.upper == 12:
                n_unit, m_unit = 2, 1
            else:
                raise ValueError('unknown time.upper {}'.format(time.upper))

        cls.init_possible_ends(n, m)
        ends = cls.possible_ends[(n, m)]
        cls.init_possible_ends(n_unit, m_unit)
        ends_unit = cls.possible_ends[(n_unit, m_unit)]
        end = int((start_beat + duration) * (1 << p))
        subnotes = []
        beat = start_beat
        debug_log.append('note {}'.format(note))

        while beat < start_beat + duration:
            start = int(beat * (1 << p))
            subend = None
            # multiple of units
            if start % unit == 0:
                for e_rel in ends_unit[start // unit]:
                    e = e_rel * unit
                    if e > end:
                        break
                    if time.lower == 8 and e != start + unit:  # only a unit is allowed for <time> X/8
                        continue
                    subend = e
                    debug_log.append('      E {}'.format(Fraction(subend - start, 1 << p)))
            # contains 'dots'
            base = start - start % (1 << n)
            for e_rel in ends[start % (1 << n)]:
                e = base + e_rel
                if e > end:
                    break
                length = e - start
                if (time.lower == 4 and length >= unit * 2) or \
                   (time.lower == 8 and length > unit):
                    break
                debug_log.append('      e {}'.format(Fraction(length, 1 << p)))
                if subend is None or e > subend:
                    subend = e
            if subend is None:
                raise ValueError('cannot find ending point in ({}, {}]'.format(start, end))
            end_beat = Fraction(subend, 1 << p)
            subnote = note.copy()
            subnote.duration = end_beat - beat
            if not subnotes:                            # first
                if end_beat != start_beat + duration:   # not last
                    subnote.tie[1] = True
            else:
                subnote.tie[0] = True
            subnotes.append(subnote)
            debug_log.append('   > {}'.format(subnote))
            beat = end_beat

        for i, subnote in enumerate(subnotes):
            if note.name == 0:
                subnote.tie[0] = False
                subnote.tie[1] = False
            else:
                if i > 0:
                    subnote.tie[0] = True
                if i < len(subnotes) - 1:
                    subnote.tie[1] = True

        if _debug:
            with open('log/split_note.log', 'w') as f:
                f.write('\n'.join(debug_log))
        return subnotes

    def __repr__(self):
        acc_str = {-1: 'b', 0: '%', 1: '#', None: ' '}
        oct_str = {-2: ',,', -1: ',', 0: '', 1: "'", 2: '"'}
        return '{}{}{:2} {:4}'.format(
                acc_str[self.acc], self.name, oct_str[self.octave], str(self.duration), self.tie)

class NodeType(Enum):
    """Type of node."""

    NOTE = 0
    DASH = 1
    DOT = 2

class Node:
    def __init__(self, note):
        if isinstance(note, Note):
            self.type = NodeType.NOTE
            self.value = note
            self.lines = note.lines
            self.dots = note.dots
            # calculate self.lines and self.dots from note.duration
            if self.lines is None or self.dots is None:
                duration = note.duration
                if duration <= 0:
                    raise ValueError('duration <= 0 for note {}'.format(note))
                numerator, denominator = duration.numerator, duration.denominator
                n = denominator.bit_length() - 1
                if (1 << n) != denominator:
                    raise ValueError('duration.denominator is not a power of 2 for note {}'.format(note))
                one_groups = list(filter(None, '{:b}'.format(numerator).split('0')))
                if len(one_groups) != 1:
                    raise ValueError('duration {} cannot be represented as a single note'.format(note.duration))
                m = numerator.bit_length() - 1
                if duration % Fraction(1) == 0:
                    self.lines = int(duration) - 1
                    self.dots = 0
                else:
                    if m > n:   # not allowed in numbered musical notation (e.g., duration = 7/2)
                        raise ValueError('duration {} is not integral, but too long for a note'.format(note.duration))
                    self.lines = m - n                  # self.lines <= 0
                    self.dots = len(one_groups[0]) - 1  # (number of dots) = (number of ones) - 1
        else:
            if note == '-':
                self.type = NodeType.DASH
            elif note == '.':
                self.type = NodeType.DOT
            else:
                raise ValueError('unknown node type for {}'.format(note))
            self.value = note
            self.lines = None
            self.dots = None
        self.text = None

    def __str__(self):
        if self.type == NodeType.NOTE:
            underlines = max(-self.lines, 0)
            if self.value.tie[1]:
                return '{} u{}  {} ~'.format(self.value, underlines, self.text)
            else:
                return '{} u{}  {}'.format(self.value, underlines, self.text)
        else:
            return ' {}'.format(self.value)

def parse_pitch(key, s):
    key_dict = dict(zip('ABCDEFG', [6, 7, 1, 2, 3, 4, 5]))
    extended_upper_name_dict = dict(zip('qwertyu', range(1, 8)))
    extended_lower_name_dict = dict(zip('zxcvbnm', range(1, 8)))
    acc_dict = {'': None, '#': 1, '$': -1, '%': 0}
    match = re.fullmatch(r"([#$%]?)([0-7a-zA-Z])([',]*)", s)
    if match is None:
        raise ValueError('wrong format for pitch {}'.format(s))
    acc, name, octave = match.groups()
    acc = acc_dict[acc]
    octave = octave.count("'") - octave.count(',')
    if name == '0':
        acc, name, octave = None, 0, 0
    elif key == 'solfa':
        if name in '1234567':
            name = int(name)
        elif name in extended_upper_name_dict:
            name = extended_upper_name_dict[name]
            octave += 1
        elif name in extended_lower_name_dict:
            name = extended_lower_name_dict[name]
            octave -= 1
        else:
            raise ValueError('{!r} is not allowed in <key> solfa'.format(s))
    else:
        if name in '1234567':
            name = int(name)
        elif name in 'cdefgabCDEFGAB':
            name = key_dict[name.upper()]
        else:
            raise ValueError('{!r} is not allowed in key {}'.format(name, key))
        if acc is not None:
            acc -= key[2][name]         # relative to key
        if key[0] <= 4:                 # <= #F major
            if name < key[0]:
                octave -= 1
        else:                           # >= G major
            if name >= key[0]:
                octave += 1
        name = (name - key[0]) % 7 + 1  # movable
    assert acc in [None, -1, 0, 1]
    assert name in [0, 1, 2, 3, 4, 5, 6, 7]
    assert octave in [-1, 0, 1]
    return acc, name, octave

class Song:
    """Member variables.

    key: (name, accidental, 8-tuple)
        tuple[i] indicates whether i is flat or sharp.
        For example, key = (2,  0, (?, 1,  0,  0,  1,  0,  0,  0)) stands for D major.
    melody: list of bars
        bar: (time, start_beat, [notes])
    lyrics: list of sections
        section: (tag, [strings])
    """
    def __init__(self):
        self.key = None
        self.melody = []
        self.lyrics = []

    def __str__(self):
        lines = [
            '<key> {}'.format(self.key),
            '<melody> {}'.format(self.melody),
            '<lyrics> {}'.format(self.lyrics),
        ]
        return '\n'.join(lines)

    def print(self):
        print('<key> {}'.format(self.key))
        for time, notes in self.melody:
            print('{}'.format('<time> {}/{}'.format(*time)))
            for note in notes:
                print('    {}'.format(note))
        for tag, lyrics in self.lyrics:
            print('<{}> {}'.format(tag, lyrics))

    def append_time_signature(self, time, s):
        """Append bars to self.melody.

        Split s into bars by '|'.
        If the duration of a bar exceeds time, split it further based on whether it is the first bar.
        """
        if not time:
            raise ValueError('unknown <time>')
        if not s:
            return

        pattern_pitch = r"[#$%]?[0-7a-zA-Z][',]*"
        pattern_pitches = r'\[(?:{})+\]'.format(pattern_pitch)
        pattern_duration = r'(?:[_=]+|-*)\.*(?:/3)?'
        pattern = '(~?)' + '({}|{})({})'.format(pattern_pitch, pattern_pitches, pattern_duration) + '(~?)'

        bars = s.split('|')
        for i, bar in enumerate(bars):
            if not bar:
                continue

            bar_duration = Fraction(0)
            note_list = []

            # parse notes in bar
            notes = re.findall(pattern, bar)
            if ''.join([''.join(note) for note in notes]) != bar:
                raise ValueError('wrong format for {} (notes = {})'.format(bar, notes))
            for tie0, pitches, duration, tie1 in notes:
                tie0, tie1 = (tie0 != ''), (tie1 != '')
                dots = duration.count('.')
                dashes = duration.count('-')
                underlines = duration.count('=') * 2 + duration.count('_')
                if dashes > 0 and underlines > 0:
                    raise ValueError('wrong format for {}'.format(notes))
                triplet = Fraction(1)
                if '/3' in duration:
                    triplet = Fraction(2, 3)
                if time.lower:
                    if pitches.startswith('[') and pitches.endswith(']'):
                        duration = (Fraction(dashes + 1, 1 << underlines)
                                    * (Fraction(2) - Fraction(1, 1 << dots)) * triplet)
                    else:
                        if dots or underlines or (triplet != 1):
                            raise ValueError('dots, underlines and triplets are not allowed'
                                             ' without brackets in <time> {}/{} hyphen={}'
                                             .format(*time))
                        duration = Fraction(dashes + 1, time.hyphen // 4)
                        dashes, underlines, dots = None, None, None
                else:
                    duration = (Fraction(dashes + 1, 1 << underlines)
                                * (Fraction(2) - Fraction(1, 1 << dots)) * triplet)
                pitches = pitches.lstrip('[').rstrip(']')
                pitches = re.findall('({})'.format(pattern_pitch), pitches)

                for k, pitch in enumerate(pitches):
                    acc, name, octave = parse_pitch(self.key, pitch)
                    tie = [tie0, tie1]
                    if k > 0:
                        tie[0] = False
                    if k < len(pitches) - 1:
                        tie[1] = False
                    # new Note
                    note = Note(acc, name, octave, duration, dashes, underlines, dots, tie)
                    note_list.append(note)
                    bar_duration += duration
            assert note_list

            # append to self.melody
            if time.upper is None:
                time_duration = bar_duration
            elif time.lower == 4:
                time_duration = Fraction(time.upper)
            else:
                time_duration = Fraction(time.upper, 2)
            if i == 0 and len(bars) > 1:        # first bar but not last
                beat = (time_duration - (bar_duration % time_duration)) % time_duration
            else:
                beat = Fraction(0)
            if beat > 0:
                self.melody.append((time, beat, []))            # new bar
            for note in note_list:
                assert note.duration > 0
                remaining_duration = note.duration
                first = True
                while remaining_duration > 0:
                    if beat == 0:
                        self.melody.append((time, beat, []))    # new bar
                    sub_duration = min(remaining_duration, time_duration - beat)
                    if not time.hyphen and sub_duration != remaining_duration:
                        raise ValueError('{} goes beyond one bar with time {}/{}'
                                         .format(note, time.upper, time.lower))
                    sub_note = note.copy()
                    sub_note.duration = sub_duration
                    if not first:
                        sub_note.tie[0] = True
                    self.melody[-1][-1].append(sub_note)
                    remaining_duration -= sub_duration
                    beat += sub_duration
                    assert beat <= time_duration
                    if beat == time_duration:
                        beat = 0
                    first = False

    def make_ties_consistent(self):
        """If note0 and note1 are consecutive, make sure that (note0.tie[1] == note1.tie[0])."""
        if not self.melody:
            return
        prev_tie = False
        for time, start_beat, notes in self.melody:
            if not notes:
                raise ValueError('empty bar in self.melody')
            for note in notes:
                if prev_tie:
                    note.tie[0] = True
                if note.name == 0:
                    note.tie = [False, False]
                prev_tie = note.tie[1]
        self.melody[0][-1][0].tie[0] = False    # first note
        self.melody[-1][-1][-1].tie[1] = False  # last note

    def try_split_notes(self):
        """Try to split note, and modify self.melody in place.

        If note.lines or note.dots is None, splitting is necessary.
        For these sub-notes, perform a 2nd-staged grouping.
        """
        for time, start_beat, notes in self.melody:
            if not time.hyphen:
                continue
            subnotes = []
            beat = start_beat
            for note in notes:
                if note.lines is None or note.dots is None:
                    subnotes += Note.split_note(time, beat, note)
                else:
                    subnotes.append(note)
                beat += note.duration
            notes[:] = subnotes

    def merge_melody_lyrics(self, _debug=False):
        """Return a list of sections.

        section: (tag, lines)
        line: [nodes, list of bars, list of ties, list of slurs]
        bar: (time, start_beat, node indices)
        tie: (node_idx1, node_idx2)
        """
        debug_log = []

        # calculate split indices according to self.lyrics
        sum_len = 0
        split_sections = {}
        split_lines = []
        for i, (tag, lyrics) in enumerate(self.lyrics):
            split_sections[sum_len] = tag
            for j, s in enumerate(lyrics):
                if s.startswith('~'):
                    raise ValueError("a line of lyrics cannot start with '~' ({})".format(s))
                split_lines.append(sum_len)
                sum_len += len(s)
        all_lyrics = ''.join([''.join(section[1]) for section in self.lyrics])
        num_words = len(all_lyrics)     # contains '~'

        # calculate the starting and ending index of slurs
        slur_lyrics_idx = {}
        prev_idx = None
        for idx, text in enumerate(all_lyrics):
            if text != '~':
                if prev_idx is not None and idx != prev_idx + 1:
                    slur_lyrics_idx[prev_idx] = idx - 1
                prev_idx = idx
        if prev_idx is not None:
            slur_lyrics_idx[prev_idx] = num_words - 1

        # split melody
        lyrics_idx = 0
        section_added, line_added = False, False    # whether these are added for this lyrics_idx
        line_node_idx_prev = -1
        sections = []
        for time, start_beat, notes in self.melody:
            if time.upper is None:
                time_duration = None
            elif time.lower == 4:
                time_duration = Fraction(time.upper)
            else:
                time_duration = Fraction(time.upper, 2)
            beat = start_beat
            for k, note in enumerate(notes):
                # new section
                if not note.tie[0] and not section_added and lyrics_idx in split_sections:
                    tag = split_sections[lyrics_idx]
                    sections.append((tag, []))
                    section_added = True
                # new line
                if not note.tie[0] and not line_added and lyrics_idx in split_lines:
                    if sections[-1][1] and note.name == 0 and (beat + note.duration) % time_duration == 0:
                        pass
                    else:
                        sections[-1][1].append([[], [], [], []])    # (nodes, bars, ties, slurs)
                        line_added = True
                        line_node_idx_prev = -1
                line = sections[-1][1][-1]
                nodes, bars, ties, slurs = line
                # new bar
                if k == 0 or not bars:
                    bars.append((time, beat, []))
                # append note Node
                node = Node(note)
                line_node_idx = len(nodes)
                bars[-1][-1].append(line_node_idx)
                if note.tie[0]:
                    ties.append((line_node_idx_prev, line_node_idx))
                    node.value.tie[0] = True
                    nodes[line_node_idx_prev].value.tie[1] = True
                else:
                    if note.name:
                        if lyrics_idx >= num_words:
                            raise ValueError('#notes > {} words'.format(num_words))
                        if all_lyrics[lyrics_idx] != '~':
                            node.text = all_lyrics[lyrics_idx]
                        lyrics_idx += 1
                        section_added, line_added = False, False
                nodes.append(node)
                line_node_idx_prev = line_node_idx
                # append dash Node's
                for _ in range(node.lines):
                    bars[-1][-1].append(len(nodes))
                    nodes.append(Node('-'))
                # append dot Node's
                for _ in range(node.dots):
                    bars[-1][-1].append(len(nodes))
                    nodes.append(Node('.'))
                beat += note.duration
        if lyrics_idx != num_words:
            raise ValueError('{} notes != {} words'.format(lyrics_idx, num_words))

        if _debug:
            with open('log/merge.log', 'w') as f:
                f.write('\n'.join(debug_log))

        return sections

    @classmethod
    def group_underlines(cls, sections):
        """Group underlines shared by contiguous notes, and find triplets by modifying sections in place.

        section: (tag, lines)
        line: [nodes, bars, ties, slurs, underlines_list, triplets]
        bar: (time, start_beat, node indices)
        tie: (node_idx1, node_idx2)
        underlines_list: list of underlines
            underlines[k]: list of underlines in depth k
            underline: [node_idx1, node_idx2]
        triplet: [node_idx1, node_idx2, node_idx3]
        """
        for tag, lines in sections:
            for line in lines:
                nodes, bars, ties, slurs = line
                underlines_list = [None]
                triplets = []
                triplet_duration = None
                for time, start_beat, idx_list in bars:
                    beat = start_beat
                    assert beat >= 0
                    idx_prev = -9
                    for idx in idx_list:
                        node = nodes[idx]
                        note = node.value
                        if node.type != NodeType.NOTE:
                            continue
                        # triplet
                        if (time.lower == 4 and beat % Fraction(1) == 0) or \
                           (time.lower == 8 and beat % Fraction(3, 2) == 0):
                            new_group = True
                            triplet_duration = None
                        else:
                            new_group = False
                        if note.duration.denominator % 3 == 0:       # triplet
                            if new_group or not triplets or note.duration != triplet_duration:
                                if triplets and len(triplets[-1]) != 3:
                                    raise ValueError('triplet with less than 3 notes')
                                triplets.append([idx])
                            elif len(triplets[-1]) == 3:
                                triplets.append([idx])
                            else:
                                triplets[-1].append(idx)
                            triplet_duration = note.duration
                        # underline
                        if node.lines < 0:
                            depth = -node.lines        # number of underlines
                            for _ in range(depth - len(underlines_list) + 1):
                                underlines_list.append([])
                            for k in range(1, depth + 1):
                                if new_group or not underlines_list[k]:
                                    underlines_list[k].append([idx, idx])
                                elif underlines_list[k][-1][1] == idx_prev:
                                    underlines_list[k][-1][1] = idx
                                else:
                                    underlines_list[k].append([idx, idx])
                        beat += note.duration
                        idx_prev = idx
                line.append(underlines_list)
                line.append(triplets)

    def to_tex_tikzpicture(self, output_dir=''):
        """Write environment tikzpicture source code to file if provided."""
        slides_file = os.path.join(output_dir, 'slides.tex')
        slides_output = []
        line_file_format = os.path.join(output_dir, 'line-{}{:02d}.tex')
        page_count = 0

        sections = self.merge_melody_lyrics()
        self.group_underlines(sections)

        for i, (tag, lines) in enumerate(sections):
            # new section
            line_count = 0
            for j, (nodes, bars, ties, slurs, underlines_list, triplets) in enumerate(lines):
                # new page
                if line_count % 2 == 0:
                    page_count += 1
                    if page_count > 1:
                        slides_output.append('\n')
                    slides_output.append('%%%%% PAGE {} %%%%%'.format(page_count))
                    slides_output.append(r'\newpage')
                    slides_output.append('')
                    if j == 0:      # first page in section
                        slides_output.append('% <{}>'.format(tag))
                        slides_output.append(r'\begin{nmntag}')
                        slides_output.append(r'\textmd{$<$\hspace{-0pt}' + tag + r'\hspace{-0pt}$>$}')
                        slides_output.append(r'\end{nmntag}')
                    else:
                        slides_output.append(r'\begin{nmnblank}')
                        slides_output.append(r'\end{nmnblank}')

                # new line
                line_lyrics = ''
                line_output = []
                line_output.append(r'\begin{tikzpicture}')
                line_output.append(r"""\tikzstyle{every node}=[inner sep=0pt]
\tikzstyle{dot}=[circle,fill=white,inner sep=0pt,text width=1.5pt]
\tikzstyle{lyrics}=[node distance=15pt]
\tikzstyle{tie}=[line width=0.5pt,bend left=45,min distance=4pt,max distance=5pt]
\tikzstyle{underline}=[line width=0.5pt]
\tikzstyle{tie0}=[line width=0.5pt,out=50,in=180,max distance=20pt]
\tikzstyle{tie1}=[line width=0.5pt,out=130,in=0,max distance=20pt]""")
                line_output.append('\n\n% nodes')
                line_output.append(r'\node at (0pt, 12pt) {}; % for space adjustment')

                pos = 0
                first_text_idx = None
                for k, (time, start_beat, idx_list) in enumerate(bars):
                    # new bar
                    if k > 0:
                        pos -= 2.5
                        line_output.append(r'\node at ({}pt,0) {{|}};'.format(pos))
                        pos += 7.5
                    for idx in idx_list:
                        node = nodes[idx]
                        note = node.value
                        line_output.append('')
                        if node.type != NodeType.NOTE:
                            pos -= 2.5
                            if node.type == NodeType.DASH:
                                line_output.append(r'\node at ({}pt,-1pt) {{-}};'.format(pos))
                            elif node.type == NodeType.DOT:
                                line_output.append(r'\node[dot] at ({}pt,0) {{}};'.format(pos))
                            pos += 7.5
                            continue
                        # name
                        line_output.append(r'\node (a{}) at ({}pt,0) {{{}}};'.format(idx, pos, note.name))
                        # acc
                        acc_dict = {-1: 'flat', 0: 'natural', 1: 'sharp'}
                        if note.acc is not None:
                            line_output.append(r'\node at ($(a{}.north west)+(-1pt,0)$){{\tiny$\{}$}};'
                                               .format(idx, acc_dict[note.acc]))
                        # octave
                        if note.octave > 0:
                            line_output.append(r'\node[dot,above of=a{},node distance=6pt] {{}};'.format(idx))
                        elif note.octave < 0:
                            node_distance = 7
                            if node.lines <= -3:
                                node_distance = 10
                            elif node.lines == -2:
                                node_distance = 9
                            elif node.lines == -1:
                                node_distance = 8
                            line_output.append(r'\node[dot,below of=a{},node distance={}pt] {{}};'
                                               .format(idx, node_distance))
                        # text
                        height = -17
                        if node.text:
                            if node.text in '每悔':
                                height += 1
                            text = '{0}{1}{0}'.format('\phantom{|}', node.text)
                            if first_text_idx is None:
                                first_text_idx = idx
                            line_output.append(r'\node[lyrics] (t{0}) at ($(a{0})+(0,{2}pt)$) {{{1}}};'
                                               .format(idx, text, height))
                            line_lyrics += node.text
                        elif first_text_idx is None:
                            text = r'\phantom{{{}}}'.format('天')
                            line_output.append(r'\node[lyrics] (t{0}) at ($(a{0})+(0,{2}pt)$) {{{1}}};'
                                               .format(idx, text, height))
                            first_text_idx = idx
                        pos += 10

                # ties
                line_output.append('\n\n% ties')
                for idx0, idx1 in ties:
                    dis = 2
                    if nodes[idx0].value.octave >= 1:
                        dis = 5
                    line_output.append(r'\draw[tie] (a{}.north) ++(0,{}pt) coordinate (tmp) to (a{}.north |- tmp);'
                                       .format(idx0, dis, idx1))

                # underlines
                line_output.append('\n\n% underlines')
                for depth, underlines in enumerate(underlines_list):
                    if depth == 0:
                        continue
                    for idx0, idx1 in underlines:
                        line_output.append(r'\draw[underline] (a{}.south west) ++(0,-{}pt)'.format(idx0, depth * 1.5)
                                           + r' coordinate (tmp) to (a{}.south east |- tmp);'.format(idx1))

                # triplets
                line_output.append('\n\n% triplets')
                for triplet in triplets:
                    dis0, dis1 = 2, 9
                    if nodes[triplet[0]].value.octave >= 1 or nodes[triplet[2]].value.octave >= 1:
                        dis0 = 5
                    if nodes[triplet[1]].value.octave >= 1:
                        dis1 = 12
                    line_output.append(r'\node[above of=a{},node distance={}pt] (tri) {{\tiny{{3}}}};'
                                       .format(triplet[1], dis1))
                    line_output.append(r'\draw[tie0] (a{}.north) +(0,{}pt) to ($(tri.west)+(-1pt,0)$);'
                                       .format(triplet[0], dis0))
                    line_output.append(r'\draw[tie1] (a{}.north) +(0,{}pt) to ($(tri.east)+(+1pt,0)$);'
                                       .format(triplet[2], dis0))

                line_output.append('')
                line_output.append(r'\end{tikzpicture}')
                line_output.append('')
                assert line_output[0] == r'\begin{tikzpicture}' and pos > 0
                line_output[0] = line_output[0] + '[xscale={}]'.format(110 / pos)

                line_file = line_file_format.format(chr(ord('a') + i), j)
                with open(line_file, 'w', encoding='utf-8') as f:
                    f.write('\n'.join(line_output))
                line_count += 1

                slides_output.append('\n% {}'.format(line_lyrics))
                slides_output.append(r'\begin{nmnline}')
                slides_output.append(r'\input{{{}}}'.format(line_file.split('/')[-1]))
                slides_output.append(r'\end{nmnline}')

        with open(slides_file, 'w', encoding='utf8') as f:
            f.write('\n'.join(slides_output))

    @classmethod
    def print(cls, sections):
        for tag, lines in sections:
            print('{:=^80}'.format(' ' + tag + ' '))
            for nodes, bars, ties, slurs, underlines_list, triplets in lines:
                print('-' * 50)
                for time, start_beat, a in bars:
                    print('<time> {}/{} beat={}'.format(time.upper, time.lower, start_beat))
                    for idx in a:
                        print('    <node {:02d}> {}'.format(idx, nodes[idx]))
                print('<ties> {}'.format(ties))
                print('<underlines>')
                for k, underlines in enumerate(underlines_list):
                    if k >= 1:
                        print('    depth {}: {}'.format(k, underlines))
                print('<triplets> {}'.format(triplets))

def parse_key(s):
    """Convert key string to ([1-7], 0 or 1 or -1, list of sharps or flats).

    s = [#$]?[a-gA-G]|[0-7][#$]?
    For example,
    s = '$D': D flat major
    s = '3#': A major
    key = (2,  0, (?, 1,  0,  0,  1,  0,  0,  0)): D major
    key = (6, -1, (?, 0, -1, -1,  0,  0, -1, -1)): A flat major
    """
    # construct table
    flat = [0, 7, 3, 6, 2, 5, 1, 4]
    sharp = [0, 4, 1, 5, 2, 6, 3, 7]
    sym2key = {}
    key2sym = {}
    for x in [-1, 1]:       # flat or sharp
        a = [0] * 8
        for n in range(8):  # number of x
            key = (4 * x * n) % 7 + 1
            tmp = 0
            if x == -1 and n >= 2:
                tmp = -1
            elif x == 1 and n >= 6:
                tmp = 1
            if x == -1:
                a[flat[n]] = -1
            else:
                a[sharp[n]] = 1
            sym2key[(n, x)] = (key, tmp, tuple(a))
            key2sym[(key, tmp)] = (n, x, tuple(a))

    # parse
    if not s:
        raise ValueError('empty key')
    elif s == 'solfa':
        return s
    if re.search(r'\d', s):     # s contains digits
        if len(s) == 1:
            if s[0] != '0':
                raise ValueError('wrong format for <key> {}'.format(s))
            return 1, 0
        if len(s) != 2 or s[0] not in '01234567' or s[1] not in '#$':
            raise ValueError('wrong format for <key> {}'.format(s))
        if s[1] == '#':
            x = 1
        else:
            x = -1
        return sym2key[(int(s[0]), x)]
    else:
        key = s[-1].upper()
        key_dict = dict(zip('ABCDEFG', [6, 7, 1, 2, 3, 4, 5]))
        if key not in key_dict:
            raise ValueError('wrong format for <key> {}'.format(s))
        key = key_dict[key]
        tmp = 0
        if len(s) == 2:
            if s[0] == '#':
                tmp = 1
            elif s[0] == '$':
                tmp = -1
            else:
                raise ValueError('wrong format for <key> {}'.format(s))
        return key, tmp, key2sym[(key, tmp)][2]

def parse_time(s):
    """Convert time signature string to a pair of integers."""
    hyphen = 4
    ss = s.split(maxsplit=1)
    if len(ss) > 1:
        hyphen = ss[1].replace(' ', '')
        if not hyphen.startswith('hyphen='):
            raise ValueError('wrong format for <time> {}'.format(s))
        hyphen = hyphen[7:]
        if hyphen not in ['4', '8', '16']:
            raise ValueError('only hyphen=[4,8,16] is allowed')
        hyphen = int(hyphen)
    ss = ss[0].replace(' ', '').split('/')
    if not ss:
        raise ValueError('empty <time>')
    if len(ss) != 2:
        raise ValueError('wrong format for <time> {}'.format(s))
    if ss[0] == '?':
        a, b = None, int(ss[1])
    else:
        a, b = int(ss[0]), int(ss[1])
    if (a, b) not in [(2, 4), (3, 4), (4, 4), (6, 8), (9, 8), (12, 8), (None, 4), (None, 8)]:
        raise ValueError('unrecognizable <time> {}/{}'.format(a, b))
    if hyphen and hyphen < b:
        raise ValueError('hyphen must >= {} for <time> {}/{}'.format(b, a, b))
    return Time(a, b, hyphen)

def load_song(melody_file, lyrics_file=None):
    """Load numbered musical notation and lyrics, and return Song."""
    song = Song()

    # melody
    with open(melody_file) as f:
        time = None
        s = ''
        for line in f:
            line = line.strip()
            if line == 'break':
                break
            elif not line or line.startswith('//'):     # blank or comment
                continue
            elif line.startswith('<key>'):
                if song.key:
                    raise ValueError('only one <key> is allowed')
                song.key = parse_key(line[5:].strip())
            elif line.startswith('<time>'):
                if time:
                    song.append_time_signature(time, s)
                time = parse_time(line[6:].strip())
                s = ''
            else:
                s += line.replace(' ', '')
        song.append_time_signature(time, s)
    song.make_ties_consistent()
    song.try_split_notes()

    # lyrics
    with open(lyrics_file, encoding='utf-8') as f:
        for line in f:
            line = line.strip()
            s = line
            for c in ' ,.!?' + '　。，、！？':
                s = s.replace(c, '')
            if s == 'break':
                break
            elif not s or s.startswith('//'):
                continue
            elif s.startswith('<tag>'):
                tag = s[5:]
                song.lyrics.append((tag, []))
            else:
                if not song.lyrics:
                    raise ValueError('no <tag> specified before {}'.format(line))
                song.lyrics[-1][1].append(s)

    return song


#!/usr/bin/python
# Copyright 2018 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Bootstrapper for running a VM script.

Args:
test-files-gcs-dir: The Cloud Storage location containing the test files.
  This dir of test files must contain a test.py containing the test logic.
"""
import logging
import os
import subprocess
import urllib2

def GetMetadataAttribute(attribute):
  url = 'http://metadata/computeMetadata/v1/instance/attributes/%s' % attribute
  request = urllib2.Request(url)
  request.add_unredirected_header('Metadata-Flavor', 'Google')
  return urllib2.urlopen(request).read()

def Bootstrap():
  """Get test files, run test."""
  try:
    logging.info('Starting bootstrap.py.')
    test_gcs_dir = GetMetadataAttribute('test_files_gcs_dir')
    test_script = GetMetadataAttribute('test_script')
    test_dir = '/test_files'
    full_test_script = os.path.join(test_dir, test_script)
    subprocess.check_call(['mkdir', test_dir])
    subprocess.check_call(
        ['gsutil', '-m', 'cp', '-r', os.path.join(test_gcs_dir, '*'),
         test_dir])
    logging.info('Making test script %s executable.', full_test_script)
    subprocess.check_call(['chmod', '+x', test_script], cwd=test_dir)
    logging.info('Running %s.', full_test_script)
    subprocess.check_call([full_test_script], cwd=test_dir)
  except Exception as e:
    print('TestFailed: error: ' + str(e))

if __name__ == '__main__':
  logging.basicConfig(level=logging.DEBUG)
  Bootstrap()


import contextlib

import torch.nn as nn

@contextlib.contextmanager
def set_mode(net: nn.Module, mode: str):
    """set the model's mode in a context manager"""
    before_train = net.training
    if mode == 'train':
        net.train()
    elif mode == 'eval':
        net.eval()
    else:
        raise NotImplementedError()

    yield net

    if before_train:
        net.train()
    else:
        net.eval()


# Author: Jake Vanderplas -- <vanderplas@astro.washington.edu>

print(__doc__)

from time import time

import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from matplotlib.ticker import NullFormatter
import numpy as np
from sklearn import manifold, datasets

# Next line to silence pyflakes. This import is needed.
Axes3D

n_points = 1000
data = np.genfromtxt('swissRollpoints.txt')
X = data[:,0:3]
color = data[:,3]
n_neighbors = 10
n_components = 2

fig = plt.figure(figsize=(15, 8))
plt.suptitle("Manifold Learning with %i points, %i neighbors"
             % (1000, n_neighbors), fontsize=14)

ax = fig.add_subplot(251, projection='3d')
ax.scatter(X[:, 0], X[:, 1], X[:, 2], c=color, cmap=plt.cm.Spectral)
ax.view_init(4, -72)

methods = ['ltsa']
labels = ['LLE', 'LTSA', 'Hessian LLE', 'Modified LLE']

for i, method in enumerate(methods):
    t0 = time()
    Y = manifold.LocallyLinearEmbedding(n_neighbors, n_components,
                                        eigen_solver='auto',
                                        method=method).fit_transform(X)
    t1 = time()
    print("%s: %.2g sec" % (methods[i], t1 - t0))

    ax = fig.add_subplot(252 + i)
    plt.scatter(Y[:, 0], Y[:, 1], c=color, cmap=plt.cm.Spectral)
    plt.title("%s (%.2g sec)" % (labels[i], t1 - t0))
    ax.xaxis.set_major_formatter(NullFormatter())
    ax.yaxis.set_major_formatter(NullFormatter())
    plt.axis('tight')

plt.show()


#!/usr/bin/python3

# ^ ten dziwny komentarz na górze ma znaczenie w systemach UNIX-owych (np. Linux)
# on wskazuje systemowi operacyjnemu gdzie jest interpreter Pythona
# dzięki temu program można uruchamiać ./waluty5.py zamiast python waluty5.py
# a system traktuje ten plik jak program (taki "exec"), a nie plik tekstowy

import requests

def pobierz_dane(data=None):
    if data: # jeśli data jest podana i jest niepusta
        adres=f'http://api.nbp.pl/api/exchangerates/tables/A/{data}?format=json'
    else: # data nie podana - bierzemy najświeższe dane
        adres=f'http://api.nbp.pl/api/exchangerates/tables/A?format=json'

    dane = requests.get(adres).json()
    return dane[0]

def znajdz_walute(tabela, szukana_waluta):
    for waluta in tabela['rates']:
        if waluta['code'] == szukana_waluta:
            return waluta
    return None

def zajmij_sie_jedna_waluta(waluta):
    kod = waluta['code']
    nazwa = waluta['currency']
    kurs = waluta['mid']
    print(f'Waluta {nazwa} (kod {kod}) ma kurs {kurs}')
    kwota = float(input('Podaj kwotę: '))
    kwotaZlote = kwota * kurs
    print(f'{kwota:.2f} {kod} = {kwotaZlote:.2f} PLN')
    kwotaWaluta = kwota / kurs
    print(f'{kwota:.2f} PLN = {kwotaWaluta:.2f} {kod}')

def main():
    try:
        data = input('Podaj datę dla tabeli, lub pusty napis aby pobrać bieżące: ')
        tabela = pobierz_dane(data)
        print('Tabela numer', tabela['no'], 'z dnia', tabela['effectiveDate'])
        while True:
            szukana_waluta = input('Podaj kod waluty (pusty napis aby zakończyć): ').upper()
            if szukana_waluta == '': break
            waluta = znajdz_walute(tabela, szukana_waluta)
            if waluta is None:
                print('Nie ma takiej waluty')
            else:
                zajmij_sie_jedna_waluta(waluta)
    except Exception as e:
        print('Nie udało się pobrać', e)

# Ten if spowoduje, że main (lub cokolwiek pod ifem) wykona się tylko wtedy,
# gdy ten plik jest uruchamiany jako program
# (np. z konsoli: python waluty5.py)

# Ale if będzie nieprawdziwy (i main się nie wykona) jeśli ten plik będzie importowany jako moduł
if __name__ == '__main__':
    main()


from keras.models import Sequential
from keras import layers
from keras.preprocessing.text import Tokenizer
import pandas as pd
from sklearn import preprocessing
from keras.layers import Flatten
from keras.layers.embeddings import Embedding
from keras.preprocessing.sequence import pad_sequences
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split

df = pd.read_csv('imdb_master.csv',encoding='latin-1')
sentences = df['review'].values
y = df['label'].values
max_review_len= max([len(s.split()) for s in sentences])

#tokenizing data
tokenizer = Tokenizer(num_words=max_review_len)
tokenizer.fit_on_texts(sentences)

vocab_size= len(tokenizer.word_index)+1
sentences = tokenizer.texts_to_sequences(sentences)
padded_docs= pad_sequences(sentences,maxlen=max_review_len)

#getting the vocabulary of data
#sentences = tokenizer.texts_to_matrix(sentences)

le = preprocessing.LabelEncoder()
y = le.fit_transform(y)
X_train, X_test, y_train, y_test = train_test_split(padded_docs, y, test_size=0.25, random_state=1000)

# Number of features
# print(input_dim)
model = Sequential()
model.add(Embedding(vocab_size, 50, input_length=max_review_len))
model.add(layers.Flatten())
model.add(layers.Dense(300,activation='relu'))
model.add(layers.Dense(3, activation='softmax'))
model.compile(loss='sparse_categorical_crossentropy',optimizer='adam',metrics=['acc'])
history=model.fit(X_train,y_train, epochs=5, verbose=True, validation_data=(X_test,y_test), batch_size=256)
#evaluating model on training data and calculating loss and accuracy of training data
[train_loss, train_acc] = model.evaluate(X_test,y_test)
print("Evaluation result on Train Data : Loss = {}, accuracy = {}".format(train_loss, train_acc))
# summarize history for accuracy&loss
plt.plot(history.history['acc'])
plt.plot(history.history['val_acc'])
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['accuray', 'val_accuracy','loss','val_loss'], loc='upper right')
plt.show()

# Predicting the Value for test sample
pred = model.predict_classes(X_test[[2],:])
print("Actual Prediction",y_test[1],"Predicted Prediction", pred)


#!/usr/bin/python3
# Fab file to launch the whole show
from datetime import datetime
from fabric.api import local
import os.path
from fabric.api import put
from fabric.api import run
from fabric.api import env

env.hosts = ["35.237.36.104", "34.74.62.199"]

def do_pack():
    """Create and return a tar.gz obj of web_static"""
    t = datetime.utcnow()
    # concat file name
    ret = "versions/web_static_{}{}{}{}{}{}.tgz".format(t.year,
                                                        t.month,
                                                        t.day,
                                                        t.hour,
                                                        t.minute,
                                                        t.second)
    # if the pathname isn't versions...
    if os.path.isdir("versions") is False:
        # and the mkdir fails...
        if local("mkdir -p versions").failed is True:
            return None
    # simulat. run and conditional
    if local("tar -cvzf {} web_static".format(ret)).failed is True:
        return None
    # come back and build print statement
    return ret

def do_deploy(archive_path):
    """This does deploy"""

    # check that passed file exists
    if os.path.isfile(archive_path) is False:
        return False

    # get file name from file path
    file_name = archive_path.split("/")[-1]
    # remove file extension
    no_ext = file_name.split(".")[0]

    # put the file on the server
    if put(archive_path, "/tmp/{}".format(file_name)).failed is True:
        return False
    # mkdir for new release
    if run("mkdir -p /data/web_static/releases/{}/".
           format(no_ext)).failed is True:
        return False
    # decompress file in tmp but move it to releases folder
    if run("tar -xzf /tmp/{} -C /data/web_static/releases/{}/".
           format(file_name, no_ext)).failed is True:
        return False
    # remove archive file
    if run("rm /tmp/{}".format(file_name)).failed is True:
        return False
    # Move the contents of the folder up one folder
    if run("mv /data/web_static/releases/{}/web_static/*"
           " /data/web_static/releases/{}/"
           .format(no_ext, no_ext)).failed is True:
        return False
    # kill empty folder
    if run("rm -fr /data/web_static/releases/{}/web_static".
           format(no_ext)).failed is True:
        return False
    # delete symbolic link
    if run("rm -fr /data/web_static/current").failed is True:
        return False
    # create new symbolic link
    if run("ln -s /data/web_static/releases/{}/ /data/web_static/current".
           format(no_ext)).failed is True:
        return False
    return True

def deploy():
    """This deploys the whole deal"""
    arch = do_pack()
    if arch is None:
        return False
    return do_deploy(arch)


#!/usr/bin/python

import i3ipc
import json
import os
import pickle
import subprocess
import sys
from functools import reduce

def setup_displays(depth=0, outputs=("DP1", "HDMI2"), preferred_res="1920x1080"):
    if depth > 1: # Adding modes didn't help
        print("ERROR: Tried adding modes, but xrandr still failed")
        sys.exit(1)

    # Get available display outputs, and try turning on the given outputs that exist.
    modes = parse_xrandr()
    outputs_to_use = [output for output in outputs
                      if output in modes and modes[output]["exists"] and output != "eDP1"]

    if len(outputs_to_use) == 0:
        print("ERROR: Given displays not found.")
        sys.exit(1)

    # Build command to turn on displays.
    outputs_to_use.append("eDP1") # Primary output always comes first
    command = ["xrandr"]
    for i, output in enumerate(outputs_to_use[ : -1]):
        command.extend(
            ["--auto", "--output", output, "--left-of", outputs_to_use[i + 1], "--mode", \
             best_res(modes[output]["res"], preferred_res)])
        if output == "DP1":
            command.extend(["--rotate", "left"])

    with open(os.devnull, "w") as devnull: # Suppress command output
        proc = subprocess.run(command, stdout=devnull, stderr=devnull)

    # Handle xrandr failure.
    if proc.returncode == 1: # Maybe modes weren't present; add them and retry
        add_modes(modes, outputs_to_use, preferred_res)
        setup_displays(depth + 1, outputs, preferred_res)
    elif proc.returncode != 0:
        print("ERROR: xrandr exited with return code %i" % proc.returncode)
        sys.exit(1)

def parse_xrandr():
    xrandr_output = subprocess \
        .check_output("xrandr") \
        .decode("utf-8") \
        .split("\n")[1 : ] # Omit first line, which refers to the screen
    xrandr_output = [line for line in xrandr_output if len(line.strip()) > 0] # Remove empty lines

    outputs_info = {}

    i = 0
    while i < len(xrandr_output):
        line = xrandr_output[i]

        if not line.startswith((" ", "\t")):
            output, status = line.split()[ : 2]
            outputs_info[output] = {"exists": status != "disconnected", "res": []}

            i += 1
            while i < len(xrandr_output) and xrandr_output[i].startswith((" ", "\t")):
                line = xrandr_output[i]

                res = line.split()[0] # Extract resolution
                outputs_info[output]["res"].append(res)

                i += 1

    return outputs_info

def best_res(resolutions, preferred_res=None):
    if preferred_res is not None and preferred_res in resolutions:
        return preferred_res

    best_res, max_res_val = None, - float("inf")
    for res in resolutions:
        if "_" in res:
            res = res[ : res.index("_")]

        res_val = reduce(lambda x, y: x * y, (int(s) for s in res.split("x")))
        if res_val > max_res_val:
            best_res, max_res_val = res, res_val

    return best_res

def add_modes(modes, outputs, mode):
    missing_mode = [output for output in outputs if output in modes and mode not in modes[output]["res"]]

    # Add missing modes, using cvt to obtain the necessary Modeline
    horizontal, vertical = mode.split("x")
    modeline = subprocess \
        .check_output(["cvt", horizontal, vertical]). \
        decode("utf-8") \
        .split("\n")[1][len("Modeline ") :]

    if len(missing_mode) > 0:
        subprocess.run(["xrandr", "--newmode"] + modeline.split())

    for output in missing_mode:
        subprocess.run(["xrandr", "--addmode", output, "%s_60.00" % mode])

# Takes in a dict of mappings for which windows go to which output. Unlisted workspaces default
# to eDP1.
# xrandr output names:
#   eDP1: laptop screen
#   DP1: left monitor
#   DP2: right monitor
def move_workspaces(display_to_ws=None, cache_file=None):
    # Get currently focused workspace.
    i3 = i3ipc.Connection()
    focused_ws = None
    for ws in i3.get_workspaces():
        if ws["focused"]:
            focused_ws = ws["name"]
    assert focused_ws is not None

    assert display_to_ws is not None or cache_file is not None
    if display_to_ws is None:
        with open(cache_file, "rb") as f:
            ws_list = pickle.load(f)

        display_to_ws = {}
        for ws in ws_list:
            display_to_ws[ws["output"]] = ws["name"]

    with open(os.devnull, "w") as devnull:
        command = "i3 workspace {name} && i3 move workspace to output {display}"
        for display, ws_name in display_to_ws.items():
            subprocess.run(command.format(name=ws_name, display=display),
                           shell=True, stdout=devnull, stderr=devnull)

        # Switch focus back to original workspace
        subprocess.run(["i3", "workspace", focused_ws],
                       stdout=devnull, stderr=devnull)

def available_displays():
    return set(subprocess
        .check_output("xrandr | grep -w connected | cut -d \" \" -f 1", shell=True)
        .decode("utf-8")
        .splitlines())

def parse_arguments():
    if len(sys.argv) <= 1:
        return None

    available = available_displays()
    workspace_config = {}
    for i in range(1, len(sys.argv), 2):
        workspace, output = sys.argv[i : i + 2]
        try:
            workspace = int(workspace)
        except:
            print("Non-integer workspace index \"{}\" found".format(workspace))
            return None

        if output not in available:
            print("Invalid display \"{}\" found".format(output))
            return None

        if workspace in workspace_config:
            print("Duplicate workspace {} found".format(workspace))
            return None

        print("Putting workspace {} on display {}".format(workspace, output))
        workspace_config[workspace] = output

    return workspace_config

def main():
    setup_displays()

    cache_file = os.path.expanduser("~/.cache/piyush_i3_cache.pkl")
    workspace_config = parse_arguments()
    if workspace_config is not None:
        print("Moving workspaces according to command line options")
        move_workspaces(workspace_config=workspace_config)
    elif os.path.exists(cache_file):
        print("Restoring from cache file {}".format(cache_file))
        move_workspaces(cache_file=cache_file)

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-
# author: houzhiwei
# https://grass.osgeo.org/grass74/manuals/r.fill.stats.html

from scrapy import Spider, Request
from GrassModules.items import GrassModulesItem

class GrassSpider(Spider):
	name = 'grass'
	# allowed_domains = ['grass.osgeo.org']
	# start_urls = ['https://grass.osgeo.org/grass74/manuals/full_index.html']
	# base_url = 'https://grass.osgeo.org/grass74/manuals/'

	start_urls = ['file://127.0.0.1/H:/gisdemo/grass-7.5_html_manual/manuals/full_index.html']
	base_url = 'file://127.0.0.1/H:/gisdemo/grass-7.5_html_manual/manuals/'

	def parse(self, response):
		"""处理模块命令索引页"""
		# process requests
		tables = response.xpath("//div[@id='container']//table")
		# print(tables)
		for table in tables:
			trs = table.xpath("//tr")
			for i, tr in enumerate(trs):
				# if i > 1:
				# 	return
				href = tr.xpath("./td[1]/a/@href").extract_first()
				# meta: 将此处获取的数据传递给回调函数（callback）
				yield Request(url=self.base_url + href, callback=self.parse_detail)

	def parse_detail(self, response):
		"""处理模块命令详情页"""
		item = GrassModulesItem()
		item['manual_url'] = response.url
		content = response.css("#container")
		item['name'] = content.css("#name>b::text").extract_first()
		print('name %s' % item['name'])
		item['definition'] = content.xpath('./h2[1][contains(text(),"NAME")]/following-sibling::text()[2]').extract()
		item['keywords'] = self.parse_keywords(content)
		item['synopsis'] = self.parse_synopsis(content)
		item['parameters'] = self.parse_params(content)
		item['description'] = self.parse_des(content)
		item['notes'] = self.parse_notes(content)
		item['see_also'] = self.parse_see_also(content)
		item['authors'] = self.parse_authors(content)
		item['source_code'] = self.parse_source(content)
		return item

	def parse_keywords(self, selector):
		"""解析命令的关键词"""
		# 文本为KEYWORDS的h2之后和文本为SYNOPSIS的h2之前的a标签?
		a_path = './a[preceding-sibling::h2[contains(text(),"KEYWORDS")] and following-sibling::h2[contains(text(),"SYNOPSIS")]]'
		keywords_list = selector.xpath(a_path)
		keywords = []
		for keyword in keywords_list:
			word = keyword.xpath('normalize-space(.//text())').extract_first()
			keywords.append(word)
		# print('keywords %s' % keywords)
		return keywords

	def parse_synopsis(self, selector):
		# help = selector.xpath("./h2[text()='SYNOPSIS']/following::b[1]/text()").extract_first()
		synopsis = selector.css("#synopsis *::text").extract()
		# print('synopsis: %s' % ''.join(synopsis))
		return ''.join(synopsis).strip()

	def parse_params(self, selector):
		dl = selector.xpath('./div[@id="flags"]/dl')
		flags_dt = dl.xpath('//dt')
		parameters = []
		for i in range(0, len(flags_dt)):
			flag = flags_dt[i].xpath('./b/text()').extract_first()
			# print('flag %s' % dt)
			dds = flags_dt[i].xpath('./following-sibling::dd[following-sibling::dt[1]]')
			flag_exp = dds.xpath('.//text()').extract()
			parameters.append(
				{'parameter': str(flag).replace("-", ""), 'flag': flag, 'explanation': ' '.join(flag_exp),
				 'optional': True, })
		dts = selector.css("#parameters>dl>dt")
		input_file = False
		out_file = False
		optional = True
		data_type = 'String'
		for i in range(0, len(dts)):
			dt = dts[i]
			# key
			dt_id = dt.xpath('./b[1]/text()').extract_first().strip()
			if dt_id == 'input':
				input_file = True
			elif dt_id == 'output':
				out_file = True
			# value
			param_type = dt.xpath('string(./em//text())').extract()
			if param_type == 'name':
				data_type = 'String'
			elif param_type == 'value':
				data_type = 'Double'
			elif param_type == 'character':
				data_type = 'String'
			elif param_type == 'float':
				data_type = 'Float'
			# required
			require = dt.xpath('./b[2]/text()').extract_first()
			if require == '[required]':
				optional = False
			# 后面的所有非空节点
			dds_test = dt.xpath('./following-sibling::node()[not(string-length(text())=0)]')
			dd_list = []
			# 遍历所有节点，直到遇到 dt
			for i, item in enumerate(dds_test):
				if len(item.css('dt')) > 0:
					break
				dd_list.append(item)
			# print("dd_list %s" % dd_list)
			# dds = dt.xpath('./following-sibling::dd[following-sibling::dt[1] or not(following-sibling::dt)]')
			# print("dds %s" % dds)
			explanation = None
			alternatives = None
			default = None
			# length = len(dd_list)
			for j, item in enumerate(dd_list):
				if j == 0 and item:
					explanation = item.css('dd::text').extract_first().strip()
					# print("explanation %s" % explanation)
					continue
				alternatives = item.xpath('./em[parent::dd[contains(text(),"Options")]][1]/text()').extract_first()
				if not alternatives:
					alternatives = item.xpath(
						'./text()[parent::dd[contains(text(),"Special characters")]]').extract_first()
				if alternatives:
					alternatives = str(alternatives).replace('Special characters:', '')
					print("alternatives %s" % alternatives)
					continue
				default = item.xpath('./em[parent::dd[contains(text(),"Default")]][1]/text()').extract_first()
				if not default:
					continue
				print("default %s" % default)
			print("explanation %s" % explanation)
			parameters.append(
				{'parameter': dt_id, 'flag': dt_id, 'optional': optional, 'explanation': explanation,
				 'dataType': data_type, 'defaultValue': default, 'isInputFile': input_file, 'isOutputFile': out_file,
				 'alternatives': alternatives})
			print("parameters %s" % parameters)
		return parameters

	def parse_des(self, selector):
		desc_selector = selector.xpath(".//a[@name='description']/parent::h2")
		em = desc_selector.xpath("normalize-space(./following-sibling::em/text())").extract_first()
		# print('em %s' % em)
		des = desc_selector.xpath(
			"normalize-space(./following-sibling::em/following-sibling::text()[1])").extract_first()
		# print('des %s' % des)
		return str(em) + str(des)

	def parse_notes(self, selector):
		notes = selector.xpath(
			"normalize-space(.//text()[preceding-sibling::h2[child::a[@name='notes']] and following-sibling::h2[child::a[@name='see-also']]])").extract()
		# print('notes %s' % notes)
		return ''.join(notes)

	def parse_see_also(self, selector):
		also_selector = selector.xpath(".//a[@name='see-also']/parent::h2")
		alist = also_selector.xpath("./following-sibling::em//a")
		alsos = []
		for a in alist:
			also = a.xpath("normalize-space(.//text())").extract_first()
			alsos.append(also)
		# print('alsos %s' % alsos)
		return alsos

	def parse_authors(self, selector):
		authors_selector = selector.xpath(".//a[@name='authors' or @name='author']/parent::h2")
		authors = authors_selector.xpath("normalize-space(./following-sibling::text()[position()<3])").extract()
		# print('authors %s' % authors)
		return authors

	def parse_source(self, selector):
		source = selector.xpath(
			"./h2[contains(text(),'SOURCE CODE')]/following-sibling::p/a/@href").extract_first()
		return source


'''
update-ratestat-tags-from-vote-playlists.py

This script uses the votes playlists (1-10) to add the new votes to and update the ratestat tags of
audio files in the music library.

'''

# import external Python modules
from prettytable import PrettyTable

# Do setup processing so that this script can import all the needed modules from the "mlu" package.
# This is necessary because these scripts are not located in the root directory of the project, but
# instead in the 'scripts' folder.
import envsetup
envsetup.PreparePythonProjectEnvironment()

from mlu.settings import MLUSettings

from com.nwrobel import mypycommons
import com.nwrobel.mypycommons.logger
import com.nwrobel.mypycommons.file
import com.nwrobel.mypycommons.time

mypycommons.logger.initSharedLogger(logDir=MLUSettings.logDir)
logger = mypycommons.logger.getSharedLogger()

# import project-related modules
import mlu.tags.backup
import mlu.ratestats

# --------------------------------------- Helper functions -----------------------------------------
#
def _getRatestatTagUpdatesSummaryFilepath():
    '''
    Returns the filepath for the ratestat tags updates summary log file.
    '''
    backupFilename = "[{}] update-ratestat-tags-from-vote-playlists_tag-updates-summary.txt".format(
        mypycommons.time.getCurrentTimestampForFilename()
    )
    filepath = mypycommons.file.JoinPaths(MLUSettings.logDir, backupFilename)

    return filepath

def _writeRatestatTagUpdatesSummaryFile(audioFileVoteDataList, summaryFilepath):
    '''
    Writes out a log file containing a table in pretty format with the ratestat tags updates.

    Params:
        audioFileVoteDataList: list of AudioFileVoteData objects, the new votes data
        summaryFilepath: path of the log output file to be written
    '''
    tagUpdatesTable = PrettyTable()
    tagUpdatesTable.field_names = ["Title", "Artist", "Votes Added", "New Rating", "New Votes List"]
    tagUpdatesTable.align["Title"] = "l"
    tagUpdatesTable.align["Artist"] = "l"
    tagUpdatesTable.align["Votes Added"] = "r"
    tagUpdatesTable.align["New Rating"] = "r"
    tagUpdatesTable.align["New Votes List"] = "r"

    for audioFileVotesData in audioFileVoteDataList:
        tagHandler = mlu.tags.io.AudioFileTagIOHandler(audioFileVotesData.filepath)
        currentTags = tagHandler.getTags()

        votesAdded = mlu.tags.common.formatValuesListToAudioTag(audioFileVotesData.votes)

        tagUpdatesTable.add_row([
            currentTags.title,
            currentTags.artist,
            votesAdded,
            currentTags.rating,
            currentTags.votes
        ])

    mypycommons.file.writeToFile(filepath=summaryFilepath, content=tagUpdatesTable.get_string())

# ------------------------------- Main script procedure --------------------------------------------
#
if __name__ == "__main__":

    logger.info("Performing full backup (checkpoint) of all music library audio files tags")
    tagsBackupFilepath = mlu.tags.backup.backupMusicLibraryAudioTags()

    logger.info("Copying vote playlist files to temp location in ~cache")
    mlu.ratestats.copyVotePlaylistsToTemp(votePlaylistsSourceDir=MLUSettings.votePlaylistsDir, votePlaylistsTempDir=MLUSettings.tempDir)

    logger.info("Loading audio file votes data from all vote playlists")
    audioFileVoteDataList = mlu.ratestats.getAudioFileVoteDataFromRatePlaylists(votePlaylistsDir=MLUSettings.tempDir)

    logger.info("Vote playlists data loaded successfully")
    logger.info("Writing new ratestats tag data to audio files")

    erroredAudioFilepaths = []
    for audioFileVoteData in audioFileVoteDataList:
        try:
            mlu.ratestats.updateRatestatTagsFromVoteData(audioFileVoteData)
        except:
            logger.exception("updateRatestatTagsFromVoteData operation failed: File='{}'".format(audioFileVoteData.filepath))
            erroredAudioFilepaths.append(audioFileVoteData.filepath)

    logger.info("Votes processing complete")
    logger.info("{} audio files were processed".format(len(audioFileVoteDataList)))
    logger.info("{} audio files failed update".format(len(erroredAudioFilepaths)))

    if (not erroredAudioFilepaths):
        logger.info("Process completed successfully: all ratestat tags updated with new votes")
        logger.info("Writing ratestat tag updates summary file")

        summaryFilepath = _getRatestatTagUpdatesSummaryFilepath()
        _writeRatestatTagUpdatesSummaryFile(audioFileVoteDataList, summaryFilepath)
        logger.info("Summary file written successfully: File='{}'".format(summaryFilepath))

        logger.info('Archiving the old vote playlists which were just processed')
        mlu.ratestats.archiveVotePlaylists(playlistsDir=MLUSettings.tempDir, archiveDir=MLUSettings.votePlaylistsArchiveDir)

        logger.info('Emptying already counted votes from vote playlists')
        mlu.ratestats.resetVotePlaylists(votePlaylistsSourceDir=MLUSettings.votePlaylistsDir, votePlaylistsTempDir=MLUSettings.tempDir)

    else:
        erroredAudioFilepathsFmt = "\n".join(erroredAudioFilepaths)
        logger.info("Failed to update ratestat tag values with new votes for the following files:\n{}".format(erroredAudioFilepathsFmt))

        logger.info("Process completed with failures: undoing all tag changes to the music library (reverting to checkpoint)")
        mlu.tags.backup.restoreMusicLibraryAudioTagsFromBackup(tagsBackupFilepath)

        logger.info("Tags backup restored successfully: all changes were undone - run this script again to retry")

    mypycommons.file.DeleteDirectory(MLUSettings.tempDir)
    logger.info('Script complete')


import json
import logging

from aiohttp.web import Request, Response

import dataclasses
import ray
import aiohttp.web
import ray.dashboard.optional_utils as optional_utils
import ray.dashboard.utils as dashboard_utils
from ray.dashboard.modules.version import (
    CURRENT_VERSION,
    VersionResponse,
)

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

routes = optional_utils.ClassMethodRouteTable

# NOTE (shrekris-anyscale): This class uses delayed imports for all
# Ray Serve-related modules. That way, users can use the Ray dashboard agent for
# non-Serve purposes without downloading Serve dependencies.
class ServeAgent(dashboard_utils.DashboardAgentModule):
    def __init__(self, dashboard_agent):
        super().__init__(dashboard_agent)

    # TODO: It's better to use `/api/version`.
    # It requires a refactor of ClassMethodRouteTable to differentiate the server.
    @routes.get("/api/ray/version")
    async def get_version(self, req: Request) -> Response:
        # NOTE(edoakes): CURRENT_VERSION should be bumped and checked on the
        # client when we have backwards-incompatible changes.
        resp = VersionResponse(
            version=CURRENT_VERSION,
            ray_version=ray.__version__,
            ray_commit=ray.__commit__,
        )
        return Response(
            text=json.dumps(dataclasses.asdict(resp)),
            content_type="application/json",
            status=aiohttp.web.HTTPOk.status_code,
        )

    @routes.get("/api/serve/deployments/")
    @optional_utils.init_ray_and_catch_exceptions()
    async def get_all_deployments(self, req: Request) -> Response:
        from ray.serve.schema import ServeApplicationSchema

        client = self.get_serve_client()

        if client is None:
            config = ServeApplicationSchema.get_empty_schema_dict()
        else:
            config = client.get_app_config()

        return Response(
            text=json.dumps(config),
            content_type="application/json",
        )

    @routes.get("/api/serve/deployments/status")
    @optional_utils.init_ray_and_catch_exceptions()
    async def get_all_deployment_statuses(self, req: Request) -> Response:
        from ray.serve.schema import serve_status_to_schema, ServeStatusSchema

        client = self.get_serve_client()

        if client is None:
            status_json = ServeStatusSchema.get_empty_schema_dict()
            status_json_str = json.dumps(status_json)
        else:
            status = client.get_serve_status()
            status_json_str = serve_status_to_schema(status).json()

        return Response(
            text=status_json_str,
            content_type="application/json",
        )

    @routes.delete("/api/serve/deployments/")
    @optional_utils.init_ray_and_catch_exceptions()
    async def delete_serve_application(self, req: Request) -> Response:
        from ray import serve

        if self.get_serve_client() is not None:
            serve.shutdown()

        return Response()

    @routes.put("/api/serve/deployments/")
    @optional_utils.init_ray_and_catch_exceptions()
    async def put_all_deployments(self, req: Request) -> Response:
        from ray.serve.schema import ServeApplicationSchema
        from ray.serve._private.api import serve_start

        config = ServeApplicationSchema.parse_obj(await req.json())

        client = serve_start(
            detached=True,
            http_options={
                "host": config.host,
                "port": config.port,
                "location": "EveryNode",
            },
        )

        if client.http_config.host != config.host:
            return Response(
                status=400,
                text=(
                    "Serve is already running on this Ray cluster. Its "
                    f'HTTP host is set to "{client.http_config.host}". '
                    f'However, the requested host is "{config.host}". '
                    f"The requested host must match the running Serve "
                    "application's host. To change the Serve application "
                    "host, shut down Serve on this Ray cluster using the "
                    "`serve shutdown` CLI command or by sending a DELETE "
                    "request to this Ray cluster's "
                    '"/api/serve/deployments/" endpoint. CAUTION: shutting '
                    "down Serve will also shut down all Serve deployments."
                ),
            )

        if client.http_config.port != config.port:
            return Response(
                status=400,
                text=(
                    "Serve is already running on this Ray cluster. Its "
                    f'HTTP port is set to "{client.http_config.port}". '
                    f'However, the requested port is "{config.port}". '
                    f"The requested port must match the running Serve "
                    "application's port. To change the Serve application "
                    "port, shut down Serve on this Ray cluster using the "
                    "`serve shutdown` CLI command or by sending a DELETE "
                    "request to this Ray cluster's "
                    '"/api/serve/deployments/" endpoint. CAUTION: shutting '
                    "down Serve will also shut down all Serve deployments."
                ),
            )

        client.deploy_app(config)

        return Response()

    def get_serve_client(self):
        """Gets the ServeControllerClient to the this cluster's Serve app.

        return: If Serve is running on this Ray cluster, returns a client to
            the Serve controller. If Serve is not running, returns None.
        """

        from ray.serve.context import get_global_client
        from ray.serve.exceptions import RayServeException

        try:
            return get_global_client(_health_check_controller=True)
        except RayServeException:
            logger.debug("There's no Serve app running on this Ray cluster.")
            return None

    async def run(self, server):
        pass

    @staticmethod
    def is_minimal_module():
        return False


"""Loader that loads YouTube transcript."""
from __future__ import annotations

from pathlib import Path
from typing import Any, Dict, List, Optional

from pydantic import root_validator
from pydantic.dataclasses import dataclass

from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader

SCOPES = ["https://www.googleapis.com/auth/drive.readonly"]

@dataclass
class GoogleApiClient:
    """A Generic Google Api Client.

    To use, you should have the ``google_auth_oauthlib,youtube_transcript_api,google``
    python package installed.
    As the google api expects credentials you need to set up a google account and
    register your Service. "https://developers.google.com/docs/api/quickstart/python"

    Example:
        .. code-block:: python

            from langchain.document_loaders import GoogleApiClient
            google_api_client = GoogleApiClient(
                service_account_path=Path("path_to_your_sec_file.json")
            )

    """

    credentials_path: Path = Path.home() / ".credentials" / "credentials.json"
    service_account_path: Path = Path.home() / ".credentials" / "credentials.json"
    token_path: Path = Path.home() / ".credentials" / "token.json"

    def __post_init__(self) -> None:
        self.creds = self._load_credentials()

    @root_validator
    def validate_channel_or_videoIds_is_set(
        cls, values: Dict[str, Any]
    ) -> Dict[str, Any]:
        """Validate that either folder_id or document_ids is set, but not both."""

        if not values.get("credentials_path") and not values.get(
            "service_account_path"
        ):
            raise ValueError("Must specify either channel_name or video_ids")
        return values

    def _load_credentials(self) -> Any:
        """Load credentials."""
        # Adapted from https://developers.google.com/drive/api/v3/quickstart/python
        try:
            from google.auth.transport.requests import Request
            from google.oauth2 import service_account
            from google.oauth2.credentials import Credentials
            from google_auth_oauthlib.flow import InstalledAppFlow
            from youtube_transcript_api import YouTubeTranscriptApi  # noqa: F401
        except ImportError:
            raise ImportError(
                "You must run"
                "`pip install --upgrade "
                "google-api-python-client google-auth-httplib2 "
                "google-auth-oauthlib"
                "youtube-transcript-api`"
                "to use the Google Drive loader"
            )

        creds = None
        if self.service_account_path.exists():
            return service_account.Credentials.from_service_account_file(
                str(self.service_account_path)
            )
        if self.token_path.exists():
            creds = Credentials.from_authorized_user_file(str(self.token_path), SCOPES)

        if not creds or not creds.valid:
            if creds and creds.expired and creds.refresh_token:
                creds.refresh(Request())
            else:
                flow = InstalledAppFlow.from_client_secrets_file(
                    str(self.credentials_path), SCOPES
                )
                creds = flow.run_local_server(port=0)
            with open(self.token_path, "w") as token:
                token.write(creds.to_json())

        return creds

class YoutubeLoader(BaseLoader):
    """Loader that loads Youtube transcripts."""

    def __init__(
        self, video_id: str, add_video_info: bool = False, language: str = "en"
    ):
        """Initialize with YouTube video ID."""
        self.video_id = video_id
        self.add_video_info = add_video_info
        self.language = language

    @classmethod
    def from_youtube_url(cls, youtube_url: str, **kwargs: Any) -> YoutubeLoader:
        """Given youtube URL, load video."""
        video_id = youtube_url.split("youtube.com/watch?v=")[-1]
        return cls(video_id, **kwargs)

    def load(self) -> List[Document]:
        """Load documents."""
        try:
            from youtube_transcript_api import NoTranscriptFound, YouTubeTranscriptApi
        except ImportError:
            raise ImportError(
                "Could not import youtube_transcript_api python package. "
                "Please install it with `pip install youtube-transcript-api`."
            )

        metadata = {"source": self.video_id}

        if self.add_video_info:
            # Get more video meta info
            # Such as title, description, thumbnail url, publish_date
            video_info = self._get_video_info()
            metadata.update(video_info)

        transcript_list = YouTubeTranscriptApi.list_transcripts(self.video_id)
        try:
            transcript = transcript_list.find_transcript([self.language])
        except NoTranscriptFound:
            en_transcript = transcript_list.find_transcript(["en"])
            transcript = en_transcript.translate(self.language)

        transcript_pieces = transcript.fetch()

        transcript = " ".join([t["text"].strip(" ") for t in transcript_pieces])

        return [Document(page_content=transcript, metadata=metadata)]

    def _get_video_info(self) -> dict:
        """Get important video information.

        Components are:
            - title
            - description
            - thumbnail url,
            - publish_date
            - channel_author
            - and more.
        """
        try:
            from pytube import YouTube

        except ImportError:
            raise ImportError(
                "Could not import pytube python package. "
                "Please install it with `pip install pytube`."
            )
        yt = YouTube(f"https://www.youtube.com/watch?v={self.video_id}")
        video_info = {
            "title": yt.title,
            "description": yt.description,
            "view_count": yt.views,
            "thumbnail_url": yt.thumbnail_url,
            "publish_date": yt.publish_date,
            "length": yt.length,
            "author": yt.author,
        }
        return video_info

@dataclass
class GoogleApiYoutubeLoader(BaseLoader):
    """Loader that loads all Videos from a Channel

    To use, you should have the ``googleapiclient,youtube_transcript_api``
    python package installed.
    As the service needs a google_api_client, you first have to initialize
    the GoogleApiClient.

    Additionally you have to either provide a channel name or a list of videoids
    "https://developers.google.com/docs/api/quickstart/python"

    Example:
        .. code-block:: python

            from langchain.document_loaders import GoogleApiClient
            from langchain.document_loaders import GoogleApiYoutubeLoader
            google_api_client = GoogleApiClient(
                service_account_path=Path("path_to_your_sec_file.json")
            )
            loader = GoogleApiYoutubeLoader(
                google_api_client=google_api_client,
                channel_name = "CodeAesthetic"
            )
            load.load()

    """

    google_api_client: GoogleApiClient
    channel_name: Optional[str] = None
    video_ids: Optional[List[str]] = None
    add_video_info: bool = True
    captions_language: str = "en"

    def __post_init__(self) -> None:
        self.youtube_client = self._build_youtube_client(self.google_api_client.creds)

    def _build_youtube_client(self, creds: Any) -> Any:
        try:
            from googleapiclient.discovery import build
            from youtube_transcript_api import YouTubeTranscriptApi  # noqa: F401
        except ImportError:
            raise ImportError(
                "You must run"
                "`pip install --upgrade "
                "google-api-python-client google-auth-httplib2 "
                "google-auth-oauthlib"
                "youtube-transcript-api`"
                "to use the Google Drive loader"
            )

        return build("youtube", "v3", credentials=creds)

    @root_validator
    def validate_channel_or_videoIds_is_set(
        cls, values: Dict[str, Any]
    ) -> Dict[str, Any]:
        """Validate that either folder_id or document_ids is set, but not both."""
        if not values.get("channel_name") and not values.get("video_ids"):
            raise ValueError("Must specify either channel_name or video_ids")
        return values

    def _get_transcripe_for_video_id(self, video_id: str) -> str:
        from youtube_transcript_api import NoTranscriptFound, YouTubeTranscriptApi

        transcript_list = YouTubeTranscriptApi.list_transcripts(self.video_ids)
        try:
            transcript = transcript_list.find_transcript([self.captions_language])
        except NoTranscriptFound:
            en_transcript = transcript_list.find_transcript(["en"])
            transcript = en_transcript.translate(self.captions_language)

        transcript_pieces = transcript.fetch()
        return " ".join([t["text"].strip(" ") for t in transcript_pieces])

    def _get_document_for_video_id(self, video_id: str, **kwargs: Any) -> Document:
        captions = self._get_transcripe_for_video_id(video_id)
        video_response = (
            self.youtube_client.videos()
            .list(
                part="id,snippet",
                id=video_id,
            )
            .execute()
        )
        return Document(
            page_content=captions,
            metadata=video_response.get("items")[0],
        )

    def _get_channel_id(self, channel_name: str) -> str:
        request = self.youtube_client.search().list(
            part="id",
            q=channel_name,
            type="channel",
            maxResults=1,  # we only need one result since channel names are unique
        )
        response = request.execute()
        channel_id = response["items"][0]["id"]["channelId"]
        return channel_id

    def _get_document_for_channel(self, channel: str, **kwargs: Any) -> List[Document]:
        channel_id = self._get_channel_id(channel)
        request = self.youtube_client.search().list(
            part="id,snippet",
            channelId=channel_id,
            maxResults=50,  # adjust this value to retrieve more or fewer videos
        )
        video_ids = []
        while request is not None:
            response = request.execute()

            # Add each video ID to the list
            for item in response["items"]:
                if not item["id"].get("videoId"):
                    continue
                meta_data = {"videoId": item["id"]["videoId"]}
                if self.add_video_info:
                    item["snippet"].pop("thumbnails")
                    meta_data.update(item["snippet"])
                video_ids.append(
                    Document(
                        page_content=self._get_transcripe_for_video_id(
                            item["id"]["videoId"]
                        ),
                        metadata=meta_data,
                    )
                )
            request = self.youtube_client.search().list_next(request, response)

        return video_ids

    def load(self) -> List[Document]:
        """Load documents."""
        document_list = []
        if self.channel_name:
            document_list.extend(self._get_document_for_channel(self.channel_name))
        elif self.video_ids:
            document_list.extend(
                [
                    self._get_document_for_video_id(video_id)
                    for video_id in self.video_ids
                ]
            )
        else:
            raise ValueError("Must specify either channel_name or video_ids")
        return document_list


# ========================================================================
# Copyright 2020 Emory University
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========================================================================

from pathlib import Path
from typing import List, Tuple, Dict

import requests

def download_data(path):
    def download(remote_addr: str, local_addr: str):
        r = requests.get(remote_addr)
        with open(local_addr, 'wb') as fin:
            fin.write(r.content)

    url = 'https://raw.githubusercontent.com/emory-courses/cs329/master/dat/pos/wsj-pos.{}.gold.tsv'

    remote = url.format('trn')
    download(remote, path / Path(remote).name)

    remote = url.format('dev')
    download(remote, path / Path(remote).name)

def read_data(filename: str):
    data, sentence = [], []
    fin = open(filename)

    for line in fin:
        l = line.split()
        if l:
            sentence.append((l[0], l[1]))
        else:
            data.append(sentence)
            sentence = []

    return data

def word_count(data: List[List[Tuple[str, str]]]) -> int:
    """
    :param data: a list of tuple list where each inner list represents a sentence and every tuple is a (word, pos) pair.
    :return: the total number of words in the data
    """
    return sum([len(sentence) for sentence in data])

def create_uni_pos_dict(data: List[List[Tuple[str, str]]]) -> Dict[str, List[Tuple[str, float]]]:
    """
    :param data: a list of tuple list where each inner list represents a sentence and every tuple is a (word, pos) pair.
    :return: a dictionary where the key is a word and the value is the list of possible POS tags with probabilities in descending order.
    """
    model = dict()
    # To be updated
    return model

if __name__ == '__main__':
    # create a directory to download pos data
    path = Path.cwd()

    while path.name != 'cs329':
        path = path.parent

    path /= 'dat/pos'
    path.mkdir(parents=True, exist_ok=True)

    # download data
    download_data(path)

    # read data
    trn_data = read_data(path / 'wsj-pos.trn.gold.tsv')
    print(len(trn_data))
    print(trn_data[0])

    # word count
    print(word_count(trn_data))