import copy
import json
import logging
import random
import re
import shutil
from argparse import ArgumentParser
from pathlib import Path
from typing import Dict, List, Set

from jinf import Jinf
from rhoknp import Jumanpp, Morpheme, Sentence
from rhoknp.utils.reader import chunk_by_sentence
from tqdm import tqdm

from kwja.utils.logging_util import filter_logs

filter_logs(environment="production")
logging.basicConfig(format="")

logger = logging.getLogger("kwja_cli")
logger.setLevel(logging.INFO)

jinf = Jinf()

STOP_SURFS: Set[str] = {'"', "\u3000"}

def is_hiragana(value):
    return re.match(r"^[\u3040-\u309F\u30FC]+$", value) is not None

def get_canons(morpheme: Morpheme) -> List[str]:
    canons: List[str] = []
    if morpheme.canon is not None:
        canons.append(morpheme.canon)
    for feature in morpheme.features:
        if feature[:4] == "ALT-":
            canons.append(feature.split('"')[1].split(" ")[0].replace("代表表記:", ""))
    return canons

def set_other_canons(sentence: Sentence, target_morpheme_indexes: Set[int]) -> None:
    for morpheme in sentence.morphemes:
        if morpheme.index in target_morpheme_indexes:
            continue
        if "代表表記" in morpheme.semantics:
            continue
        if morpheme.conjtype == "*" or morpheme.pos == "特殊":
            canon: str = f"{morpheme.lemma}/{morpheme.reading}"
        elif is_hiragana(morpheme.lemma):
            canon = f"{morpheme.lemma}/{morpheme.lemma}"
        else:
            canon_right: str = jinf(morpheme.reading, morpheme.conjtype, morpheme.conjform, "基本形")
            canon = f"{morpheme.lemma}/{canon_right}"
        morpheme.semantics["代表表記"] = canon

def get_is_excluded(sentence: Sentence) -> bool:
    is_excluded: bool = False
    for morpheme in sentence.morphemes:
        for stop_surf in STOP_SURFS:
            if stop_surf in morpheme.surf:
                is_excluded = True
                break
        if (morpheme.pos != "特殊") and (not is_hiragana(morpheme.reading)):
            is_excluded = True
            break
    return is_excluded

def main():
    parser = ArgumentParser()
    parser.add_argument("-i", "--input-dirs", nargs="*", required=True)
    parser.add_argument("-o", "--output-dir", type=str, required=True)
    parser.add_argument("-max", "--max-samples", type=int, default=3)
    args = parser.parse_args()

    random.seed(42)

    jumanpp = Jumanpp()

    input_paths: List[Path] = []
    for input_dir in args.input_dirs:
        input_paths.extend(list(Path(input_dir).glob("**/*.txt")))

    sentences: List[Sentence] = []
    canon2freq: Dict[str, int] = {}
    excluded_nums: Dict[str, int] = {}
    for input_path in tqdm(input_paths):
        with input_path.open() as f:
            for knp in chunk_by_sentence(f):
                try:
                    sentence: Sentence = Sentence.from_knp(knp)
                except ValueError:
                    excluded_nums["sent_from_knp"] = excluded_nums.get("sent_from_knp", 0) + 1
                    continue

                juman_sentence: Sentence = jumanpp.apply_to_sentence(sentence.text)

                kwja_words: List[str] = [morpheme.text for morpheme in sentence.morphemes]
                juman_words: List[str] = [morpheme.text for morpheme in juman_sentence.morphemes]
                if kwja_words != juman_words:
                    excluded_nums["word_segmentation"] = excluded_nums.get("word_segmentation", 0) + 1
                    continue

                for morpheme in sentence.morphemes:
                    if morpheme.pos not in {"名詞", "動詞", "形容詞", "接頭辞", "接尾辞", "副詞", "連体詞"}:
                        continue
                    canons: List[str] = get_canons(morpheme)
                    if len(canons) > 1:
                        for canon in canons:
                            canon2freq[canon] = canon2freq.get(canon, 0) + 1
                sentences.append(sentence)
    print(f"num_sentences: {len(sentences)}")

    sid2knp_str: Dict[str, str] = {}
    sid2info: Dict[str, Dict[int, Dict[str, str]]] = {}
    sampled_canon2freq: Dict[str, int] = {}
    for sentence in sentences:
        if get_is_excluded(sentence):
            continue
        info: Dict[int, Dict[str, str]] = {}
        target_canons: Set[str] = set()
        target_morpheme_indexes: Set[int] = set()
        for morpheme in sentence.morphemes:
            canons: List[str] = get_canons(morpheme)
            if len(canons) > 1 or sampled_canon2freq.get(morpheme.canon, 0) >= args.max_samples:
                continue

            if canon2freq.get(morpheme.canon, 0) >= 2:
                if morpheme.conjtype == "*":
                    surf: str = morpheme.reading
                    lemma: str = morpheme.reading
                else:
                    try:
                        lemma = jinf(morpheme.reading, morpheme.conjtype, morpheme.conjform, "基本形")
                        surf = jinf(lemma, morpheme.conjtype, "基本形", morpheme.conjform)
                    except (ValueError, NotImplementedError):
                        excluded_nums["jinf"] = excluded_nums.get("jinf", 0) + 1
                        continue
                if surf == morpheme.text and lemma == morpheme.lemma:
                    continue
                surf_before: str = copy.deepcopy(morpheme.text)
                lemma_before: str = copy.deepcopy(morpheme.lemma)
                morpheme.text = surf
                morpheme._text_escaped = surf
                morpheme.lemma = lemma
                info[morpheme.index] = {
                    "partial_annotation_type": "canon",
                    "surf_before": surf_before,
                    "lemma_before": lemma_before,
                    "surf_after": surf,
                    "lemma_after": lemma,
                    "conjtype": morpheme.conjtype,
                    "conjform": morpheme.conjform,
                }
                target_canons.add(morpheme.canon)
                target_morpheme_indexes.add(morpheme.index)
        if len(info) == 0:
            continue

        try:
            set_other_canons(sentence, target_morpheme_indexes)
        except (ValueError, NotImplementedError):
            excluded_nums["set_canon"] = excluded_nums.get("set_canon", 0) + 1
            continue
        try:
            sid2knp_str[sentence.sid] = sentence.to_knp()
        except AttributeError:
            excluded_nums["to_knp"] = excluded_nums.get("to_knp", 0) + 1
            continue
        sid2info[sentence.sid] = info
        for target_canon in target_canons:
            sampled_canon2freq[target_canon] = sampled_canon2freq.get(target_canon, 0) + 1

        if len(sid2knp_str) >= 5000:
            break

    sid2knp_str_list: List[tuple[str, str]] = list(sid2knp_str.items())
    random.shuffle(sid2knp_str_list)
    train_size: int = int(len(sid2knp_str_list) * 0.9)
    valid_size: int = int(len(sid2knp_str_list) * 0.05)
    train_list: List[tuple[str, str]] = sid2knp_str_list[:train_size]
    valid_list: List[tuple[str, str]] = sid2knp_str_list[train_size : train_size + valid_size]
    test_list: List[tuple[str, str]] = sid2knp_str_list[train_size + valid_size :]

    output_dir: Path = Path(args.output_dir)
    output_dir.mkdir(parents=True, exist_ok=True)

    with open(output_dir / "canon2freq.txt", "w") as f:
        for canon, freq in canon2freq.items():
            f.write(f"{canon}\t{freq}\n")
    with open(output_dir / "sampled2freq.txt", "w") as f:
        for canon, freq in sampled_canon2freq.items():
            f.write(f"{canon}\t{freq}\n")
    with open(output_dir / "info.json", "w") as f:
        json.dump(sid2info, f, indent=2, ensure_ascii=False)

    train_dir: Path = output_dir / "train"
    if train_dir.exists():
        shutil.rmtree(str(train_dir))
    train_dir.mkdir(exist_ok=True)
    for name, knp_str in train_list:
        with (train_dir / f"{name}.knp").open("w") as f:
            f.write(f"{knp_str}\n")

    valid_dir: Path = output_dir / "valid"
    if valid_dir.exists():
        shutil.rmtree(str(valid_dir))
    valid_dir.mkdir(exist_ok=True)
    for name, knp_str in valid_list:
        with (valid_dir / f"{name}.knp").open("w") as f:
            f.write(f"{knp_str}\n")

    test_dir: Path = output_dir / "test"
    if test_dir.exists():
        shutil.rmtree(str(test_dir))
    test_dir.mkdir(exist_ok=True)
    for name, knp_str in test_list:
        with (test_dir / f"{name}.knp").open("w") as f:
            f.write(f"{knp_str}\n")

    print(f"train: {len(train_list)}")
    print(f"valid: {len(valid_list)}")
    print(f"test: {len(test_list)}")
    print(f"total: {len(train_list) + len(valid_list) + len(test_list)}")
    print(f"excluded_nums: {excluded_nums}")

if __name__ == "__main__":
    main()


from tkinter import Tk, filedialog
from datetime import datetime
import json

import os
import subprocess
import webbrowser

def runReport(Folders):
    #create folder for new report
    now = datetime.now()
    current_time = now.strftime("%Y%m%d_%H%M%S")
    f = open ('PyRep.json')
    data = json.load (f)
    f.close()

    driveName = data['appDrive']
    folderName = driveName + data['repFolder'] + data['repPrefix'] + current_time
    os.mkdir(folderName)

    #create report parameters
    repExe = driveName + data['repExe']
    reports = '-reports:"'
    for folder in Folders:
        reports += folder + "\\coverage.xml;"
    reports +='" '
    targetdir = '-targetdir:"{PATH}" '.format(PATH=folderName)
    sourcedirs = '-sourcedirs:"' + driveName + data['sourcedirs'] + '"'

    #run the report
    cmd = repExe + reports + targetdir + sourcedirs
    print (cmd)
    subprocess.call(cmd)

    #show the report
    url = "file://" + folderName + '\\index.html'
    webbrowser.open(url)

# MAIN
root = Tk() # pointing root to Tk() to use it as Tk() in program.
root.withdraw() # Hides small tkinter window.
root.attributes('-topmost', True) # Opened windows will be active. above all windows despite of selection.
folders = []
while True:
    path1 = filedialog.askdirectory(title="Select first folder") # Returns opened path as str
    if not path1:
        break
    folders.append(path1)

runReport(folders)


#!/usr/bin/python
# -*- coding:utf-8 -*-
# 评分/评论页面
import json
import math

from django.http.response import HttpResponse, HttpResponseRedirect
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.shortcuts import render
from GameModel.models import MagzineScores, Shelf, Magazines
from .util import *

active = "comment"
theme = "weui"

# 首页
def home(request):
    # 每个杂志一块，每块显示评分Top4的游戏
    magazines = Magazines.objects.filter(enable=True)
    data = {}
    review_ids = []
    for m in magazines:
        top4 = MagzineScores.objects.filter(magazine=m.title, gameId__gt=0).order_by("-score")[:4]
        data[m.title] = top4
        review_ids += map(lambda x: x.gameId, top4)

    # 获取评测的游戏标题、封图等
    games = Shelf.objects.filter(gameId__in=review_ids).only("gameId", "cover", "titleCh")

    # 游戏信息和杂志块发生关联
    game_cover = {}
    for game in games:
        game_cover[game.gameId] = json.loads(game.cover.replace('\'', '\"'), encoding="utf-8")[0]
    print(logo)
    # 评测信息和游戏信息不必捏合到一起，通过gameId可以访问game_data里的数据
    render_data = {
        "active": active, "magazines": data, "game_cover": game_cover,
        "logo": logo
    }
    return render(request, "weui/review/home.html", render_data)
    # return HttpResponse("test")

# 媒体评分
def magazine(request, magazine=""):
    # request数据
    get_data = request.GET.copy()
    page = get_data.setdefault("page", 1)

    # 根据游戏名查找评分 @TODO
    game_name = get_data.setdefault("g", "")
    # 根据得分范围查找评分 @TODO
    score_low = get_data.setdefault("low", 1)
    score_high = get_data.setdefault("high", 10)

    # 获取数据
    size = 10
    filters = {"gameId__gt": 0}
    if len(magazine):
        filters["magazine"] = magazine
    if len(game_name):
        #@TODO 根据游戏名查找ID
        pass
    if score_low > 0:
        # filter["score"]
        pass

    mag_obj = MagzineScores()
    # magzines = MagzineScores.objects.filter(**filters)[offset:size]
    data_list = MagzineScores.objects.filter(**filters).order_by("-score")
    # 分页
    paginator = Paginator(data_list, size)
    try:
        current_page_data = paginator.page(page)
    except PageNotAnInteger:
        current_page_data = paginator.page(1)
    except EmptyPage:
        current_page_data = paginator.page(paginator.num_pages)

    template = "weui/review/magazine.html"
    if theme == "bootstrap":
        template = "bootstrap/comment/magazine.html"

    # 获取评测所属游戏
    for item in current_page_data.object_list:
        shelf = Shelf.objects.filter(gameId=item.gameId).first()
        if shelf is not None:
            shelf.cover = shelf.cover.replace("\'", "\"")
            cover = json.loads(shelf.cover, encoding="utf-8")[0]
            item.__setattr__("shelf_cover", cover)
        else:
            item.__setattr__("shelf_cover", "/static/image/ns.png")

    page_range = range(1, paginator.num_pages+1)
    if paginator.num_pages > 6:
        page_range = range(1, 6+1)
    if current_page_data.number > 3:
        page_range = range(current_page_data.number - 3, current_page_data.number + 3)

    render_data = {
        "active": active,
        "list": current_page_data.object_list,
        "page_count": paginator.num_pages,
        "page_range": page_range,
        "current_page": current_page_data,
        "magzines": mag_obj.getMagzineNames(),
        "logo": logo,
        "magazine_title": magazine
    }

    return render(request, template, render_data)

# 根据游戏id获取评测
def review(request, pk):
    current_action = "review"

    if pk != 0:
        review = MagzineScores.objects.filter(id=pk).first()
        game = None
        if review.gameId != 0:
            game = Shelf.objects.filter(gameId=review.gameId).first()
            # 如果数据库里没有关联或gameId查不到，跳转到原url吧。。。
            if game is None:
                return HttpResponseRedirect(review.url)

            game.cover = json.loads(game.cover.replace("\'", "\""))[0]

        render_data = {
            "active": active,
            "review": review,
            "game": game,
            "breadcrumb": breadcrumb(current_action, gameId=review.gameId),
            "logo": logo,
        }

        if theme == "weui":
            template = "weui/review/game.html"
        else:
            template = "comment/review.html"

        return render(request, template, render_data)
    else:
        return HttpResponse("请求的游戏ID不存在！")

# 用户评分（@TODO）
def member(request):
    pass


import tkinter as tk
from tkinter import Label, StringVar
from datetime import datetime
from tkcalendar import Calendar  # pip install tkcalendar
import tkinter.font as font

okno = tk.Tk()
# tytuł, rozmiar, blokada wielkości
okno.title("Zegar i kalendarz")
okno.geometry("400x400")
okno.minsize(400, 400)
okno.maxsize(400, 400)
# utwórz StringVar()
text = StringVar()
myFont = font.Font(family='Arial', size=38, weight='bold')
date_time = Label(okno, textvariable=text, font=myFont, background="black", foreground="white")
def update_date_time():
	# dzien = i tak dalej... miesiac, rok, czas, dzien
	# czytamy datetime.today().strftime('%A')
	# kody https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes
	dt = datetime.today().strftime("%d %B %Y\n%A %H:%M:%S")
	# dt = dzien + ... + "\n" + ...
	# ustaw za pomocą .set dt dla StringVar zrobinego powyżej
	# ważne: rekurencyjne odświeżanie etykiety - patrz poniżej
	text.set(dt)
	date_time.after(1000, update_date_time)

# widget Label ustawiony na StringVar zrobiony na początku, rozmiar, czcionki, tło - wg uznania
# date_time = Label(...
date_time.pack(anchor="center")

current_time = datetime.now()
# przyda się do kalendarza
# day = odczytaj przez .strftime('%d')
# month =
# year =

# utwórz cal = Calendar(...
# wstaw przez .pack poniżej zegara
cal = Calendar(okno, year = datetime.today().year, month = datetime.today().month, day = datetime.today().day)
cal.pack(anchor="center", pady=20, padx=50)
update_date_time()

okno.mainloop()


import librosa
import matplotlib.pyplot as plt

# Video dosyasını yükle
filename = 'video.mp4'
y, sr = librosa.load(filename)

# Ses düzeylerini analiz et
melspec = librosa.feature.melspectrogram(y, sr=sr, n_mels=128)

# Konuşma olmadığı yerleri tespit et
silent_frames = librosa.util.find_silent_frames(melspec,threshold=30)

# Sessiz kareleri görüntüle
plt.figure(figsize=(12,4))
librosa.display.specshow(librosa.power_to_db(melspec,ref=np.max),x_axis='time',sr=sr,hop_length=512)
plt.vlines(silent_frames/sr,0,sr, color='r', linestyle='dotted')
plt.show()


# from Post.decorators import unauthenticated_user
from copy import error

from django import db, http
from django.contrib.auth import login
from django.core.checks.messages import ERROR
from django.db.models import query
from django.db.models.query import QuerySet
from django.dispatch.dispatcher import NONE_ID
from django.http import Http404, HttpResponse, JsonResponse
from django.http.response import HttpResponseGone
from django.shortcuts import get_object_or_404, redirect, render
from django.views.decorators.csrf import requires_csrf_token
from django.views.generic.base import View

# allows us to authenticate tokens from knox auth
from knox.auth import TokenAuthentication
from knox.models import User  # ****
from knox.models import AuthToken
from knox.views import LoginView as KnoxLoginView
from Post.models import Article
from Post.serializers import ArticleSerializer
from Post.views import ArticleViewSet
from rest_framework import (
    generics,
    mixins,
    permissions,
    relations,
    response,
    serializers,
    status,
    views,
    viewsets,
)
from rest_framework.authtoken.serializers import AuthTokenSerializer
from rest_framework.decorators import (
    APIView,
    api_view,
    authentication_classes,
    permission_classes,
)
from rest_framework.fields import to_choices_dict
from rest_framework.parsers import JSONParser
from rest_framework.permissions import IsAuthenticated, IsAuthenticatedOrReadOnly
from rest_framework.relations import PrimaryKeyRelatedField
from rest_framework.response import Response
from rest_framework.serializers import Serializer

# from .decorators import promoter_only
from .models import FollowerRelation, User, UserProfile
from .serializers import (
    ChangePasswordSerializer,
    ProfileEditSerializer,
    RegisterSerializer,
    UserProfileSerializer,
    UserSerializer,
    ViewUserProfileSerializer,
    followSerializer,
)

@api_view(["POST"])
@permission_classes([IsAuthenticated])
@authentication_classes([TokenAuthentication])
def followUser(request, *args, **kwargs):
    # a class to follow a another user
    is_following = False
    prof = request.data.get("profile")

    profile_ = UserProfile.objects.get(user_id=prof)
    profile_username = profile_.user

    user_ = User.objects.get(username=request.user)

    # check follower count of focused profile
    follower_count = profile_.followers.count()
    # check if user is in the list of followers of the profile.followers
    follower_list = profile_.followers.all()

    if user_ in follower_list:
        is_following = True

    if is_following == False:
        relation = FollowerRelation.objects.create(user=user_, profile=profile_)
        profile_.followers.add(user_)

        return Response(
            {
                "isFollowing": True,
                "followCount": follower_count,
                "response": f"{user_.username} followed {profile_username}",
            }
        )

    elif is_following == True:
        relation = FollowerRelation.objects.filter(
            user=user_, profile=profile_
        ).delete()
        profile_.followers.remove(user_)

        return Response(
            {
                "isFollowing": False,
                "followCount": follower_count,
                "response": f"{user_.username} unfollowed {profile_username}",
            }
        )

class ProfileAPI(viewsets.ViewSet):
    permission_classes = (IsAuthenticated,)
    authentication_classes = (TokenAuthentication,)

    def retrieve(self, request, pk=None):
        # print(pk)
        try:
            profile = UserProfile.objects.get(user=pk)
            user = request.user

            user_username = user.username
            str(user_username)
            if profile.user.username == user_username:

                UserSerializers = UserProfileSerializer(profile)
            else:
                UserSerializers = ViewUserProfileSerializer(
                    profile, context={"request": request, "pk": pk}
                )
            return Response(UserSerializers.data)
        except error:
            print(error)

    def update(self, request, pk=None):
        """
        update() is function of the Profile API, that allows users to edit their profile
        """
        try:
            user = request.user
            profile = UserProfile.objects.get(user=pk)
            # check if request is by by profile's user
            if user != profile.user:
                return HttpResponse("invalid credentials")
            else:
                serializers = ProfileEditSerializer(
                    profile, data=request.data, context={"request": request}
                )
                if serializers.is_valid():
                    serializers.save()
                # return Response(serializers.data, status=status.HTTP_201_CREATED)
            return Response(serializers.data)
        except (error):
            return HttpResponse(error)

class ProfilePostsAPI(viewsets.ViewSet):
    permission_classes = (IsAuthenticated,)
    authentication_classes = (TokenAuthentication,)

    def retrieve(self, request, pk=None):
        try:
            userProfile = UserProfile.objects.get(user=pk)
            userArticles = Article.objects.filter(author=userProfile)
            # userArticles = Article.objects.filter(author=user_)
            articleSerializer = ArticleSerializer(
                userArticles, many=True, context={"request": request}
            )

            return Response(articleSerializer.data)
        except error:
            print(error)

# Register API
#
###
class RegisterAPI(generics.GenericAPIView):

    serializer_class = RegisterSerializer

    def post(self, request, *args, **kwargs):
        user = None
        data = request.data

        serializer = self.get_serializer(data=data)

        if serializer.is_valid(raise_exception=True):

            user = serializer.create(data)
        else:
            print("invalid data")
        return Response(
            {
                # returns the given fields of UserSerializer
                "user": UserSerializer(
                    user, context=self.get_serializer_context()
                ).data,
                "token": AuthToken.objects.create(user)[1],
            }
        )

#
# LoginAPI
#
# #
class LoginAPI(KnoxLoginView):
    permission_classes = (permissions.AllowAny,)

    def post(self, request, format=None):
        serializer = AuthTokenSerializer(data=request.data)
        serializer.is_valid(raise_exception=True)
        user = serializer.validated_data["user"]
        login(request, user)

        return super(LoginAPI, self).post(request, format=None)

#
# ChangePasswordView
#
# #
class ChangePasswordView(generics.UpdateAPIView):
    """
    An endpoint for changing password.
    """

    serializer_class = ChangePasswordSerializer
    model = User
    permission_classes = (IsAuthenticated,)
    authentication_classes = (TokenAuthentication,)

    def get_object(self, queryset=None):
        obj = self.request.user
        return obj

    def update(self, request, *args, **kwargs):
        self.object = self.get_object()
        serializer = self.get_serializer(data=request.data)

        if serializer.is_valid():
            # Check old password
            if not self.object.check_password(serializer.data.get("old_password")):
                return Response(
                    {"old_password": ["Wrong password."]},
                    status=status.HTTP_400_BAD_REQUEST,
                )
            # set_password also hashes the password that the user will get
            self.object.set_password(serializer.data.get("new_password"))
            self.object.save()
            response = {
                "status": "success",
                "code": status.HTTP_200_OK,
                "message": "Password updated successfully",
                "data": [],
            }

            return Response(response)

        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)


import logging
import os
import re

from cros.factory.probe import function
from cros.factory.probe.lib import cached_probe_function
from cros.factory.test.utils import v4l2_utils

class GenericVideoFunction(cached_probe_function.GlobPathCachedProbeFunction):
  """Probe the generic video information."""

  GLOB_PATH = '/sys/class/video4linux/video*'

  _probed_dev_paths = {}

  @classmethod
  def ProbeDevice(cls, dir_path):
    logging.debug('Find the node: %s', dir_path)

    dev_path = os.path.abspath(os.path.realpath(os.path.join(dir_path,
                                                             'device')))
    if dev_path not in cls._probed_dev_paths:
      # We don't know if this is an USB device or not, let's check.
      results = function.InterpretFunction(
          {'usb': os.path.join(dev_path, '..')})()
      assert len(results) <= 1

      if len(results) == 0:
        # This is not an USB component, therefore, not a USB webcam.
        # We might need to deal with this case when we have discrete GPU in the
        # future.
        cls._probed_dev_paths[dev_path] = (None, None)
      else:
        cls._probed_dev_paths[dev_path] = (results[0], [])

    result, comp_types = cls._probed_dev_paths[dev_path]
    if result is None:
      # This is not an USB component, skip.
      return None

    # Get video4linux2 (v4l2) result.
    video_idx = re.search(r'video(\d+)$', dir_path).group(1)
    comp_type = v4l2_utils.GuessComponentType(int(video_idx))
    if comp_type is not None and comp_type not in comp_types:
      assert len(comp_types) == 0, (f'A component cannot be both {comp_types} '
                                    'at the same time.')
      comp_types.append(comp_type)
      result['type'] = comp_type
      return result
    # Either comp_type is None, or we already returned probe result with this
    # type before, don't return anything in this round.
    return None


# Databricks notebook source
usvisa = sqlContext.read.format("com.databricks.spark.csv").options(header='true').load("/FileStore/tables/us_perm_visas.csv")
usvisa.registerTempTable("usvisa")
usvisa.printSchema()

# COMMAND ----------

#Case Status
usvisa.select("case_status").distinct().show()

# COMMAND ----------

#Maximum Wage per Acceptance
max_wage=sqlContext.sql("Select MAX(INT(wage_offer_from_9089)) from usvisa where case_status='Certified'")
max_wage.show()

# COMMAND ----------

#Minimum Wage per Acceptance
min_wage=sqlContext.sql("Select MIN(INT(wage_offer_from_9089)) from usvisa where case_status='Certified'")
min_wage.show()

# COMMAND ----------

#Acceptance per Country
acceptance=sqlContext.sql("Select Count(case_status) as acceptance_count ,country_of_citzenship from usvisa where case_status='Certified' group by country_of_citzenship order by acceptance_count DESC")
acceptance.show(10)

# COMMAND ----------

#Rejection per Country
rejection=sqlContext.sql("Select Count(case_status) as rejection_count ,country_of_citzenship from usvisa where case_status='Denied' group by country_of_citzenship order by rejection_count DESC")
rejection.show(10)

# COMMAND ----------

#Application per Country
application=sqlContext.sql("Select Count(case_status) as application_count ,country_of_citzenship from usvisa  group by country_of_citzenship order by application_count DESC")
application.show(10)

# COMMAND ----------

#Acceptance Ratio per Country
import pyspark.sql.functions as F

acceptance_ratio = application\
    .join(acceptance, "country_of_citzenship")\
    .withColumn("ratio", (F.col("acceptance_count") / F.col("application_count")))

acceptance_ratio.sort('ratio', ascending=False)
acceptance_ratio.sort('ratio', ascending=False).filter(F.col("application_count") > 100).show(10)

# COMMAND ----------

#Rejection Ratio per country
import pyspark.sql.functions as F

rejection_ratio = application\
    .join(rejection, "country_of_citzenship")\
    .withColumn("ratio", (F.col("rejection_count") / F.col("application_count")))

rejection_ratio.sort('ratio', ascending=False)
rejection_ratio.sort('ratio', ascending=False).filter(F.col("application_count") > 100).show(10)

# COMMAND ----------

#Applications Count per company
top_10_applications=sqlContext.sql("Select COUNT(case_number) as app_count,employer_name from usvisa group by employer_name order by app_count DESC")
top_10_applications.show(10)

# COMMAND ----------

#Acceptance count per company
top_10_acceptances=sqlContext.sql("Select COUNT(case_number) as acc_count,employer_name from usvisa where case_status='Certified' group by employer_name order by acc_count DESC ")
top_10_acceptances.show(10)

# COMMAND ----------

#Rejection count per company
top_10_rejections=sqlContext.sql("Select COUNT(case_number) as rej_count,employer_name from usvisa where case_status='Denied' group by employer_name order by rej_count DESC ")
top_10_rejections.show(10)

# COMMAND ----------

#Company Acceptance Ratio
Company_acceptance_ratio = top_10_applications\
    .join(top_10_acceptances, "employer_name")\
    .withColumn("ratio", (F.col("acc_count") / F.col("app_count")))

Company_acceptance_ratio.sort('ratio', ascending=False)
Company_acceptance_ratio.sort('ratio', ascending=False).filter(F.col("app_count") > 100).show(20)

# COMMAND ----------

#Year of Application
date= sqlContext.sql("select distinct(YEAR ( decision_date ) )from usvisa")

# COMMAND ----------

#Accepted rate over years
rateaccpt= sqlContext.sql("Select YEAR( decision_date) as Year, count(case_status) as Acceptance_Count  from usvisa where case_status='Certified' group by Year order by  Acceptance_Count").show(10)

# COMMAND ----------

#Rejected rate over years
ratereject= sqlContext.sql("Select  YEAR( decision_date) as Year, count(case_status) as Rejection_Count from usvisa where case_status='Denied' group by Year order by  Rejection_Count").show(10)

# COMMAND ----------

#Training months
exp= sqlContext.sql("select distinct(job_info_training_num_months) from usvisa where job_info_training_num_months !=''")

# COMMAND ----------

#Acceptance
WorkExpAccept= sqlContext.sql("Select job_info_training_num_months, count(case_status) as Acceptance_Count  from usvisa where case_status='Certified' and job_info_training_num_months!='' group by job_info_training_num_months order by  Acceptance_Count DESC").show(10)

# COMMAND ----------

#Rejected
WorkExpReject= sqlContext.sql("Select job_info_training_num_months, count(case_status) as Rejection_Count  from usvisa where case_status='Denied' and job_info_training_num_months!='' group by job_info_training_num_months order by  Rejection_Count DESC ").show(10)

# COMMAND ----------

#Applications
career_application_rate = sqlContext.sql("Select pw_soc_title as career, count(case_status) as app_Count  from usvisa  group by career order by  app_Count DESC")
career_application_rate.show(10)

# COMMAND ----------

#Acceptance rate
career_acceptance_rate= sqlContext.sql("Select pw_soc_title as career, count(case_status) as acc_count  from usvisa where case_status='Certified' group by career order by  acc_count DESC")
career_acceptance_rate.show(10)

# COMMAND ----------

#Acceptance ratio
career_acceptance_ratio = career_application_rate\
    .join(career_acceptance_rate, "career")\
    .withColumn("ratio", (F.col("acc_count") / F.col("app_count")))

career_acceptance_ratio.sort('ratio', ascending=False)
career_acceptance_ratio.sort('ratio', ascending=False).filter(F.col("app_count") > 100).show(10)

# COMMAND ----------

#Applications
City= sqlContext.sql("select distinct(job_info_work_city) from usvisa")

# COMMAND ----------

#Maximum number of applications recieved.
maxappcity= sqlContext.sql("Select job_info_work_city, count(case_status) as application_count  from usvisa where job_info_work_city!='' group by job_info_work_city order by  application_count DESC ")
maxappcity.show()

# COMMAND ----------

#Maximum nuber of acceptances are in the following cities,
maxacceptances=sqlContext.sql("Select Count(case_status) as acceptance_count ,job_info_work_city from usvisa where case_status='Certified' and job_info_work_city!=''  group by job_info_work_city order by acceptance_count DESC")
maxacceptances.show()

# COMMAND ----------

#Maximum number of rejections,
maxrejections=sqlContext.sql("Select Count(case_status) as rejection_count ,job_info_work_city from usvisa where case_status='Denied' and job_info_work_city!=''  group by job_info_work_city order by rejection_count DESC")
maxrejections.show()

# COMMAND ----------

#City Acceptance Ratio
city_acceptance_ratio = maxappcity\
    .join(maxacceptances, "job_info_work_city")\
    .withColumn("ratio", (F.col("acceptance_count") / F.col("application_count")))

city_acceptance_ratio.sort('ratio', ascending=False)
city_acceptance_ratio.sort('ratio', ascending=False).filter(F.col("application_count") > 100).show()
#city_acceptance_ratio.sort('ratio', ascending=False).filter(F.col("job_info_work_city") =='New York').show()

# COMMAND ----------

#City Rejection Ratio
city_rejection_ratio = maxappcity\
    .join(maxrejections, "job_info_work_city")\
    .withColumn("ratio", (F.col("rejection_count") / F.col("application_count")))

city_rejection_ratio.sort('ratio', ascending=False)
city_rejection_ratio.sort('ratio', ascending=False).filter(F.col("application_count") > 100).show()
city_rejection_ratio.sort('ratio', ascending=False).filter(F.col("job_info_work_city") =='New York').show()

# COMMAND ----------

#Application Rate
sector_application_rate = sqlContext.sql("Select naics_2007_us_title as sector, count(case_status) as app_Count  from usvisa where naics_2007_us_title <> 'null' group by sector order by  app_Count DESC")
sector_application_rate.show(10)

# COMMAND ----------

#Acceptance Rate
sector_acceptance_rate= sqlContext.sql("Select naics_2007_us_title as sector, count(case_status) as acc_count  from usvisa where case_status='Certified' AND  naics_2007_us_title <> 'null' group by sector order by  acc_count DESC")
sector_acceptance_rate.show(10)

# COMMAND ----------

#Acceptance Ratio
sector_acceptance_ratio = sector_application_rate\
    .join(sector_acceptance_rate, "sector")\
    .withColumn("ratio", (F.col("acc_count") / F.col("app_count")))

sector_acceptance_ratio.sort('ratio', ascending=False)
sector_acceptance_ratio.sort('ratio', ascending=False).filter(F.col("app_count") > 100).show(10)


#================How to Use==================#
"""
1 main Class
Indeed will do the necessary updates for indeed.
Import into shell and create an instance of Indeed to run.
This code definitely needs refactoring.

"""
class Indeed():

    def __init__(self):
        self.jobtitles,self.links,self.company,self.unique_ids= self.indeed_pull()
        self.indeed_update_database(self.jobtitles,self.links,self.company,self.unique_ids)

    def indeed_pull(self):
        from urllib.request import urlopen as uReq
        from bs4 import BeautifulSoup as soup
        base_url = "https://www.indeed.com/jobs?q=Python+developer&l=San+Antonio%2C+TX"
        #helper function
        def scrape_soup_2(all_job_boxes,jobtitles,links,company_names,unique_ids):
            #Run this only when you are feeding empty strings.
            link_base = "https://www.indeed.com"
            for i,job in enumerate(all_job_boxes):
                anchors = job.select('a.jobtitle')
                #this is the case without h2 tags
                if len(anchors)==1:
                    current_title= anchors[0].get('title').strip()
                    current_link = link_base+anchors[0].get('href')
                #this is the case with h2 tags. h2 > a.title
                else:
                    h2s= job.select("h2.jobtitle")
                    link = h2s[0].select("a.turnstileLink")
                    if len(h2s) == 1:
                        current_title = h2s[0].a.get('title').strip()
                        current_link = link_base+link[0].get('href')
                #========Company Names================#
                company_span = job.select('span.company')
                company_anchor = company_span[0].select('a')
                    #==========1.No company links ==========#
                if len(company_anchor) == 0:
                    current_company = company_span[0].text.strip()
                    #==========2.company links ==========#
                else:
                    current_company = company_anchor[0].text.strip()
                unique_id = current_company+current_title
                #Only addition to scrape soup 1 since we are not starting with empty unique ids.
                #realistically i can JUST use this.
                if unique_id not in unique_ids:
                    jobtitles.append(current_title)
                    links.append(current_link)
                    company_names.append(current_company)
                    unique_ids.append(unique_id)
            return jobtitles,links,company_names,unique_ids

        starts_with = 0
        jobtitles,links,company_names,unique_ids = [],[],[],[]
        while starts_with < 100:
            #should do 10,20,30,40
            if starts_with == 0:
                url = base_url
            else:
                url = base_url + "&start="+str(starts_with)
            print(len(jobtitles))
            print("crawling: \n")
            print(url)
            print(len(jobtitles))
            uClient = uReq(url)
            page_html= uClient.read()
            uClient.close()
            #parse soup
            page_soup = soup(page_html, 'html.parser')
            all_job_boxes = page_soup.findAll("div",{"class":"jobsearch-SerpJobCard"})
            jobtitles,links,company_names,unique_ids = scrape_soup_2(
                all_job_boxes,jobtitles,links,company_names,unique_ids
                )
            starts_with += 10

        return jobtitles,links,company_names,unique_ids

    def indeed_update_database(self,jobtitles,links,company_names,unique_ids):
        from jobs.models import Job
        from datetime import date
        current_jobs = Job.objects.all()
        current_links =[]
        for job in current_jobs:
            current_links.append(job.job_link)
        for i in range(len(jobtitles)):
            if links[i] not in current_links:
                Job.objects.create(
                    job_title= jobtitles[i],
                    job_company = company_names[i],
                    job_link = links[i],
                    job_last_action=date.today(),
                )


# -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import unicode_literals

import sys
from numbers import Number

import six
import tldextract

from osxcollector.output_filters.summary_filters.summary import SummaryFilter

class HtmlSummaryFilter(SummaryFilter):
    """Prints the analysis summary (AKA "Very Readable Output") in HTML format."""

    def __init__(self, html_output_file=None, **kwargs):
        super(HtmlSummaryFilter, self).__init__(summary_output_file=html_output_file, **kwargs)

    def filter_line(self, blob):
        """Each Line of OSXCollector output will be passed to filter_line.

        The OutputFilter should return the line, either modified or unmodified.
        The OutputFilter can also choose to return nothing, effectively swallowing the line.

        Args:
            blob: A dict

        Returns:
            A dict or None
        """
        isIOC = False

        if 'osxcollector_vthash' in blob:
            self._vthash.append(blob)
            isIOC = True

        if 'osxcollector_vtdomain' in blob:
            self._vtdomain.append(blob)
            isIOC = True

        if 'osxcollector_alexa_rank' in blob:
            self._alexarank.append(blob)

        if 'osxcollector_opendns' in blob:
            self._opendns.append(blob)
            isIOC = True

        if 'osxcollector_blacklist' in blob:
            self._blacklist.append(blob)
            isIOC = True

        if 'osxcollector_related' in blob:
            self._related.append(blob)
            isIOC = True

        if self._show_signature_chain:
            if 'signature_chain' in blob and blob['osxcollector_section'] in ['startup', 'kext']:
                signature_chain = blob['signature_chain']
                if not len(signature_chain) or 'Apple Root CA' != signature_chain[-1]:
                    self._signature_chain.append(blob)
                    isIOC = True

        if self._show_browser_ext:
            if blob['osxcollector_section'] in ['firefox', 'chrome'] and blob.get('osxcollector_subsection') == 'extensions':
                self._extensions.append(blob)
                isIOC = True

        if isIOC:
            if self._group_key in blob:
                key = blob[self._group_key]
                if self._group_key == 'osxcollector_domains':
                    key = tldextract.extract(key[0]).domain + '.' + tldextract.extract(key[0]).suffix
                self._iocs_by_key[key].append(blob)
            else:
                self._iocs.append(blob)

        return blob

    def _write(self, text):
        try:
            self._output_stream.write(text.encode('utf-8', errors='replace'))
        except UnicodeDecodeError as err:
            self._output_stream.write(text)
            sys.stderr.write(
                'Unicode decode error when encoding text:\n{0}\nError:\n{1}\n'
                .format(text, err),
            )

    def end_of_lines(self):
        """Called after all lines have been fed to filter_output_line.

        The OutputFilter can do any batch processing on that requires the complete input.

        Returns:
            An array of dicts (empty array if no lines remain)
        """
        self._write('''''')
        self._print_header('Very Readable Output Bot')
        self._print_para('Let\'s see what\'s up with this machine.')

        if self._group_by_iocs:
            self.summarize_by_ioc()
        else:
            self.summarize_by_threat_indicator()

        self._print_para('Very Readable Output Bot')
        self._print_para('#kaythanksbye')

        self._write('')

        return []

    def summarize_by_ioc(self):

        if self._group_key:
            self._print_header('Table of contents (sorted by {0})'.format(self._group_key), level=2)
            self._write('
')
            for ioc_key, ioc_value in six.iteritems(self._iocs_by_key):
                self._print_section_link(
                    ioc_key, ioc_key, len(ioc_value),
                )
            self._print_section_link('iocs', 'Remaining IOCs not tagged by key', len(self._iocs))
            self._write('
')

            for ioc_key, ioc_value in six.iteritems(self._iocs_by_key):
                self._write('
'.format(ioc_key))
                self._print_header('{0}'.format(ioc_key), level=2)
                self._print_para('Here is the analysis for the IOC {0}.'.format(ioc_key))
                self._summarize_blobs(ioc_value)
                self._print_para('Hopefully that was helpful.')
                self._write('
')

        self._write('
')
        if (self._group_key):
            self._print_header('Remaining IOCs ({0})'.format(self._iocs), level=2)
        else:
            self._print_header('All IOCs ({0})'.format(len(self._iocs)), level=2)
        if (self._group_key):
            self._print_para('Here is a list of remaining IOCs that were not tagged by your key, just in case!')
        else:
            self._print_para('Get ready to analyze a bunch of IOCs!')
        self._summarize_blobs(self._iocs)
        self._print_para('Hopefully that was helpful.')
        self._write('
')

    def summarize_by_threat_indicator(self):

        self._print_header('Table of contents', level=2)
        self._write('
')

        if len(self._vthash):
            self._print_section_link(
                'vthash', 'VirusTotal bad hash hits', len(self._vthash),
            )

        if len(self._vtdomain):
            self._print_section_link(
                'vtdomain', 'VirusTotal bad domain hits', len(self._vtdomain),
            )

        if len(self._opendns):
            self._print_section_link(
                'opendns', 'OpenDNS Investigate hits', len(self._opendns),
            )

        if len(self._blacklist):
            self._print_section_link(
                'blacklist', 'Blacklist hits', len(self._blacklist),
            )

        if len(self._related):
            self._print_section_link(
                'related', 'Related hits', len(self._related),
            )

        if len(self._signature_chain):
            self._print_section_link(
                'signature_chain', 'Signature chain',
                len(self._signature_chain),
            )

        if len(self._extensions):
            self._print_section_link(
                'extensions', 'Extensions', len(self._extensions),
            )

        if len(self._add_to_blacklist):
            self._print_section_link(
                'add_to_blacklist', 'Blacklist update suggestions',
                len(self._add_to_blacklist),
            )

        self._write('
')

        if len(self._vthash):
            self._write('
')
            self._print_header('VirusTotal bad hash hits', level=2)
            self._print_para('Dang! You\'ve got known malware on this machine. Hope it\'s commodity stuff')
            self._summarize_blobs(self._vthash)
            self._print_para('Sheesh! This is why we can\'t have nice things!')
            self._write('
')

        if len(self._vtdomain):
            self._write('
')
            self._print_header('VirusTotal bad domain hits', level=2)
            self._print_para('I see you\'ve been visiting some \'questionable\' sites. If you trust VirusTotal that is.')
            self._summarize_blobs(self._vtdomain)
            self._print_para('I hope it was worth it!')
            self._write('
')

        if len(self._opendns):
            self._write('
')
            self._print_header('OpenDNS Investigate hits', level=2)
            self._print_para('Well, here\'s some domains OpenDNS wouldn\'t recommend.')
            self._summarize_blobs(self._opendns)
            self._print_para('You know you shouldn\'t just click every link you see? #truth')
            self._write('
')

        if len(self._blacklist):
            self._write('
')
            self._print_header('Blacklist hits', level=2)
            self._print_para('We put stuff on a blacklist for a reason. Mostly so you don\'t do this.')
            self._summarize_blobs(self._blacklist)
            self._print_para('SMH')
            self._write('
')

        if len(self._related):
            self._write('
')
            self._print_header('Related hits', level=2)
            self._print_para('This whole things started with just a few clues. Now look what I found.')
            self._summarize_blobs(self._related)
            self._print_para('Nothing hides from Very Readable Output Bot')
            self._write('
')

        if len(self._signature_chain):
            self._write('
')
            self._print_header('Signature chain', level=2)
            self._print_para('If these binaries were signed by \'Apple Root CA\' I\'d trust them more.')
            self._summarize_blobs(self._signature_chain)
            self._print_para('Let\'s just try and stick with some safe software')
            self._write('
')

        if len(self._extensions):
            self._write('
')
            self._print_header('Extensions', level=2)
            self._print_para('Let\'s see what\'s hiding in the browser, shall we.')
            self._summarize_blobs(self._extensions)
            self._print_para('You know these things have privileges galore.')
            self._write('
')

        if len(self._add_to_blacklist):
            self._write('
')
            self._add_to_blacklist = list(set(self._add_to_blacklist))
            self._print_header('Blacklist update suggestions', level=2)
            self._print_para('If I were you, I\'d probably update my blacklists to include:')
            for key, val in self._add_to_blacklist:
                self._summarize_val(key, val)
            self._print_para('That might just help things, Skippy!')
            self._write('
')

    def _print_section_link(self, section, title, size):
        self._write(
            '
{1} ({2})
'.format(
                section, title, size,
            ),
        )

    def _summarize_blobs(self, blobs):
        self._write('
')
        for blob in blobs:
            self._write('
')
            section = blob.get('osxcollector_section')
            subsection = blob.get('osxcollector_subsection', '')
            self._print_header(u'{0} {1}'.format(section, subsection), level=3)
            self._write('
')
            self._summarize_general(blob)

            add_to_blacklist = False

            if 'osxcollector_vthash' in blob:
                self._summarize_vthash(blob)
                add_to_blacklist = True

            if 'osxcollector_vtdomain' in blob:
                self._summarize_vtdomain(blob)

            if 'osxcollector_alexa_rank' in blob:
                self._summarize_alexa_rank(blob)

            if 'osxcollector_opendns' in blob:
                self._summarize_opendns(blob)

            if 'osxcollector_blacklist' in blob:
                self._summarize_blacklist(blob)

            if 'osxcollector_related' in blob:
                self._summarize_related(blob)

            if 'md5' in blob and '' == blob['md5']:
                add_to_blacklist = True

            if add_to_blacklist:
                blacklists = blob.get('osxcollector_blacklist', {})
                values_on_blacklist = blacklists.get('hashes', [])
                for key in ['md5', 'sha1', 'sha2']:
                    val = blob.get(key, '')
                    if len(val) and val not in values_on_blacklist:
                        self._add_to_blacklist.append((key, val))

                values_on_blacklist = blacklists.get('domains', [])
                for domain in blob.get('osxcollector_domains', []):
                    if domain not in values_on_blacklist:
                        self._add_to_blacklist.append(('domain', domain))
            self._write('
')  # this is the end of the list started by "_summarize_line"
            self._write('
')
        self._write('
')

    def _summarize_general(self, blob):
        self._write(u'
General
')
        for key in sorted(blob):
            if not key.startswith('osxcollector') and blob.get(key):
                val = blob.get(key)
                self._summarize_val(key, val, etype='dd')

    def _summarize_vthash(self, blob):
        self._write(u'
Virustotal Hash
')
        for blob in blob['osxcollector_vthash']:
            for key in ['positives', 'total', 'scan_date']:
                val = blob.get(key)
                self._summarize_val(key, val, 'vthash', etype='dd')
            permalink = blob.get('permalink')
            self._write(u'
{0}
'.format(permalink))

    def _summarize_vtdomain(self, blob):
        self._write(u'
Virustotal Domain
')
        for blob in blob['osxcollector_vtdomain']:
            for key in ['domain', 'detections']:
                val = blob.get(key)
                self._summarize_val(key, val, 'vtdomain', etype='dd')

    def _summarize_alexa_rank(self, blob):
        for blob in blob['osxcollector_alexa_rank']:
            for key in ['attributes']:
                val = blob.get(key)
                self._summarize_val(key, val, 'alexarank')

    def _summarize_opendns(self, blob):
        self._write(u'
OpenDNS
')
        for blob in blob['osxcollector_opendns']:
            for key in ['domain', 'categorization', 'security']:
                val = blob.get(key)
                self._summarize_val(key, val, 'opendns', etype='dd')
            link = blob.get('link')
            self._write(u'
{0}
'.format(link))

    def _summarize_blacklist(self, blob):
        self._write(u'
Blacklist
')
        for key in blob['osxcollector_blacklist']:
            self._summarize_val(u'blacklist-{0}'.format(key), blob['osxcollector_blacklist'][key])

    def _summarize_related(self, blob):
        self._write(u'
Related
')
        for key in blob['osxcollector_related']:
            self._summarize_val(u'related-{0}'.format(key), blob['osxcollector_related'][key])

    def _summarize_val(self, key, val, prefix=None, etype='li'):
        self._write('<{0}>'.format(etype))
        self._print_key(key, prefix)
        self._print_val(val)
        self._write('</{0}>'.format(etype))

    def _print_header(self, text, level=1):
        self._write(u'<h{0}>{1}</h{0}>'.format(level, text))

    def _print_para(self, text):
        self._write(u'
{0}
'.format(text))

    def _print_list_item(self, item):
        self._write('
')
        self._print_val(item)
        self._write('
')

    def _print_key(self, key, prefix=None):
        if not prefix:
            prefix = ''
        else:
            prefix += '-'
        self._write(u'{0}{1}: '.format(prefix, key))

    def _print_val(self, val):
        if isinstance(val, list):
            self._write('
')
            for v in val:
                self._print_list_item(v)
            self._write('
')
        elif isinstance(val, dict):
            self._write('
')
            for key in val:
                self._summarize_val(key, val[key])
            self._write('
')
        elif isinstance(val, Number):
            val = str(val)
            self._encode_val(val)
        elif isinstance(val, six.string_types):
            self._encode_val(val)

    def _encode_val(self, val):
        if not isinstance(val, six.text_type):
            val = six.text_type(val, errors='replace')
        self._write(u'{0}'.format(val))


import rospy
from mrs_msgs.srv import *
from mrs_msgs.msg import *
from geometry_msgs.msg import *
from sensor_msgs.msg import *
from std_msgs.msg import *

from utils.functions.move_uav import go_to
from utils.classes.ReferencePublisher import ReferencePublisher
from utils.classes.Position_Command import Position_Info
from utils.classes.BlueFoxImage import *

import time
import numpy as np

#def in_position(goal):
#    pos = Position_Info()
#    delta = 0.01
#
#    if abs(pos.pos.heading - goal) >= delta:
#        return False
#    return True

#sobe o drone e volta para indicar que encontrou algo
def indicacao():
    publisher = ReferencePublisher()
    pos_obj = Position_Info()

    while pos_obj.pos.position.x == 0.0:
        pass

    current_x, current_y, current_z = [
        pos_obj.pos.position.x,
        pos_obj.pos.position.y,
        pos_obj.pos.position.z,
    ]

    for i in range(
        1, 3, 1
    ):  # aumentar o segundo argumento apenas de 2 em 2 para restaurar posicao inicial
        if i % 2 == 0:
            go_to(publisher.pub_reference, (current_x, current_y, current_z))
            time.sleep(2)
        else:
            go_to(publisher.pub_reference, (current_x, current_y, current_z + 1.2))
            time.sleep(2)

#comparacao utilizando delta x e y, nao utilizamos mas resultados podem ser melhores
def compara_locs(distances):
    unique_sens = list()
    idx_visto = list()
    num_sens = len(distances)
    temp = False
    for i in range(num_sens):
        for j in range(i, num_sens):
            dx = distances[i][0] - distances[j][0]
            dy = distances[i][1] - distances[j][1]
            dist = np.sqrt(dx * dx + dy * dy)
            if dist < 0.3:
                x = int((distances[i][0] + distances[j][0]) / 2)
                y = int((distances[i][1] + distances[j][1]) / 2)
                unique_sens.append((x, y))
                idx_visto.append(i)
                idx_visto.append(j)
                break
    for i in range(distances):
        temp = False
        for j in idx_visto:
            if i == j:
                temp = True
                break
        if temp == False:
            unique_sens.append(distances[i])
    return unique_sens

#compara um sinal com uma lista de sinais (sinal eh uma cor vermelha)
def red_already_exists(signal, all_signals):
    for known_signal in all_signals:
        if (
            (abs(signal[1] - known_signal[1]) < 0.4)
            and (abs(signal[0] - known_signal[0]) < 0.5)
            or (signal[0] == 0.0 and signal[1] == 0.0)
        ):
            print("\nSensor detectado anteriormente")
            return True
    return False

#tira a foto e procura se existe sinal
def search_red_in_photo(n_signals, all_signals):
    image = BlueFoxImage()
    found_red = image.get_distances_red_signals()
    new_points = []
    for red_signal in found_red:
        if not red_already_exists(red_signal, all_signals):
            n_signals += 1
            new_points.append(red_signal)
            print(
                "\n\nSinal numero: {}\nCooordenadas: {:.2f}, {:.2f}".format(
                    n_signals, red_signal[0], red_signal[1]
                )
            )
            print("Executando indicacao para o sinal {}".format(n_signals))
            indicacao()
            time.sleep(3)

    for red_signal in new_points:
        all_signals.append(red_signal)

    return n_signals, all_signals

#cria varios pontos para percorrer o tubo devagar
def divide_line(tube_start_end, k, z):
    # definindo cada um apenas para ficar mais entendivel
    # divide a reta em k vezes e retorna os pontos
    x_s = tube_start_end[0][0]
    y_s = tube_start_end[0][1]
    x_e = tube_start_end[1][0]
    y_e = tube_start_end[1][1]
    positions_list = []
    i = 1.0
    while i <= k:
        t = float(i) / float(k)
        x_destin = x_s * (1 - t) + x_e * t
        y_destin = y_s * (1 - t) + y_e * t
        positions_list.append((x_destin, y_destin, z))
        i += 1

    return positions_list

#retorna dois pontos existentes no tubo
def get_tube_points():
    return [[-3.55, 4.599], [-2.898, 1.625]]

#utilizando os dois pontos, calcula um ponto final e inicial
#tomando em consideracao o tamanho da arena
def get_tube_start_end(points):
    is_x = True
    x_1 = points[0][0]
    y_1 = points[0][1]
    x_2 = points[1][0]
    y_2 = points[1][1]
    x_test = -6
    delta_x_test = x_1 - x_2
    delta_y_test = y_1 - y_2
    inc = float(delta_y_test) / float(delta_x_test)
    my_delta_x = x_test - x_1
    y_test = y_1 + my_delta_x * inc
    if y_test > 6.3:
        is_x = False
        y_test = 6
        inc = float(1) / float(inc)
        my_delta_y = y_test - y_1
        x_test = x_1 + my_delta_y * inc

    if is_x:
        x_end = 0
        y_end = y_test + ic * 5
    else:
        y_end = 0
        x_end = x_test + inc * (-5)

    return [[x_test, y_test], [x_end, y_end]]


'''
Created on 13.08.2018

@author: larissa.hoefling@uni-tuebingen.de
-*- coding: utf-8 -*-
'''
import numpy as np
import matplotlib.pyplot as plt
from analysis.utils import smooth
from scipy.signal import periodogram
import pickle as pkl
import matplotlib as mpl

class Stimulus():
    '''
    #########################################################################
    When an object of the class stimulus is instantiated, a stimulus file (.txt)
    is opened and the timeline and voltage commands are read, and the expected
    current is computed. For now, a general capacitance value is assumed for all
    chips of the same type (typically no oxide), however in a future version, a
    chip name should be passed and the corresponding exact capacitance should be
    retrieved from a .csv file and saved as an instance variable
    '''
    temp_res = 10000
    conversion_factor = 10**(-3) #times 10**-6 to convert from uF to F,
                                 #times 10**-6 to convert from uV to V,
                                 #times 10**4 to convert from 1/0.0001 s to 1/s
                                 #times 10**3 to convert from A to mA
                                 #times 10**2 to convert from 1/mm**2 to 1/cm**2

    def __init__(self, path = None, len_snippets = 200, stepwidth_snippets=20, \
                 rcond=10**-5, smoothing_window_len = 20):
        '''
        set the path to the stimulus file, set the capacitance that should be
        used to convert from voltage command to expected current density;

        load the stimulus time_line and the voltage_command from the .txt file
        found under path

        compute the expected current density

        Inputs:
        -----------------------------------------------------------------------
        path    path to stimulus file
        len_snippets    int
        stepwidth_snippets    int
        rcond                 float
        smoothing_window_len    int
                parameters to reconstruct_stimulus

        Outputs:
        -----------------------------------------------------------------------
        None;
        sets
        self.path to path
        self.capacitance to fixed value
        self.time_line to the time line read from file
        self.voltage_command to the voltage command read from file
        self.current_density to the output of self.expected_current()
        self.white_current_density to the output of self.reconstruct_stimulus()
        '''
        if path == None:
            #fill this later with a gui
            raise ValueError("no path for the stimulus file was passed")
        else:
            self.path = path

        self.capacitance = 0.04227
        temp = np.genfromtxt(path)

        self.time_line = temp[:-1,0]
        self.voltage_command = temp[:,1]
        self.current_density = self.expected_current()
        self.white_current_density = \
            self.reconstruct_stimulus(len_snippets, stepwidth_snippets, rcond, smoothing_window_len)
#         self.white_current_density = \
#         np.concatenate(np.transpose(self.whiten_stimulus(len_snippets, stepwidth_snippets, rcond)))

    def expected_current(self):
        '''
        using I = C*dV/dt and i = I/A, compute the expected current density
        '''
        current_density = \
        np.diff(self.voltage_command)*self.capacitance*self.conversion_factor
        return current_density

    def create_stimulus_ensemble(self, len_sta, stepwidth = 1, white = False):
        '''
        from the continuous stimulus loaded from the file, create a stimulus
        ensemble, where each element is a len_sta long snippet, and snippets are
        taken stepwidth ms apart

        input:
        len_sta     int
                    length of the snippets in samples
        stepwidth    int
                    stepwidth between the snippets in ms
        white        boolean
            if white is true, then the stimulus ensemble is created from the
            reconstructed whitened stimulus

        output:
        stim_ensemble    array
                        the created stimulus ensemble, dimensions
                        len_sta x # of snippets (i.e. 1 column is 1 snippet)

        '''
        stim_ensemble = []
        sample_stepwidth = int((self.temp_res/1000)*stepwidth)
        if white:

            len_stim = len(self.white_current_density)
            dim = len_stim-len_sta
            [stim_ensemble.append(self.white_current_density[i:i+len_sta]) \
                                    for i in range(0, dim+sample_stepwidth, sample_stepwidth)]
        else:

            len_stim = len(self.current_density)
            dim = len_stim-len_sta
            [stim_ensemble.append(self.current_density[i:i+len_sta]) \
                                    for i in range(0, dim+sample_stepwidth, sample_stepwidth)]

        # if len(stim_ensemble[-1]) != len(stim_ensemble[-2]):
        #     stim_ensemble.pop(-1)
        #     print("Last element of stimulus ensemble had to be removed because it was of a different dimension ")
        if stepwidth==1:
            stim_ensemble.pop(-1)
        stim_ensemble = np.asarray(stim_ensemble)
        stim_ensemble = np.transpose(stim_ensemble)
        return stim_ensemble

    def compute_stimulus_cov(self, len_sta, mode = 'original'):
        '''
        computes the surrogate stimulus covariance, based on the stimulus
        ensemble with snippets of length len_sta (for more details, see
        description of create_stimulus_ensemble)

        '''
        if mode == 'original':
            stim_ensemble = self.create_stimulus_ensemble(len_sta)
        elif mode == 'white_smooth':
            stim_ensemble = self.create_stimulus_ensemble(len_sta, white=True)
        elif mode == 'white_raw':
            stim_ensemble = self.whiten_stimulus(len_sta, stepwidth_stim_snippets=1)

        stim_cov = np.cov(stim_ensemble)

        return stim_cov

    def compute_stim_cov_inverse_sqrt(self, len_sta, rcond=10**-5):
        '''
        computes sqrt pseudoinverse of the stimulus covariance matrix. For
        details, see Aljadeff et al. (2016), Neuron
        '''
        stim_cov = self.compute_stimulus_cov(len_sta)
        u,s,v = np.linalg.svd(stim_cov)
        singular_values = s
        rcond = rcond * np.max(singular_values)
        ind = np.where(singular_values<=rcond)
        singular_values = 1/np.sqrt(singular_values)
        singular_values[ind] = 0
        sqrt_inv = np.dot(np.dot(u, np.diag(singular_values)), v)
        return sqrt_inv

    def compute_stim_cov_inverse(self, len_sta, rcond=10**-5):
        '''
        computes pseudoinverse of the stimulus covariance matrix. For
        details, see Aljadeff et al. (2016), Neuron
        '''
        stim_cov = self.compute_stimulus_cov(len_sta)
        u,s,v = np.linalg.svd(stim_cov)
        singular_values = s
        rcond = rcond * np.max(singular_values)
        ind = np.where(singular_values<=rcond)
        singular_values = 1/singular_values
        singular_values[ind] = 0
        inv = np.dot(np.dot(u, np.diag(singular_values)), v)
        return inv

    def whiten_stimulus(self, len_sta, stepwidth_stim_snippets, rcond = 10**-5):
        '''
        creates a stimulus ensemble, and whitens it, i.e. removes autocorrelations
        from this ensemble by multiplying with the sqrt of the pseudo inverse
        of the stimulus covariance matrix
        '''

        stim_cov_sqrt_inv = self.compute_stim_cov_inverse_sqrt(len_sta, rcond)
        stimulus_ensemble = self.create_stimulus_ensemble(len_sta, stepwidth_stim_snippets)
        whitened_stimulus_ensemble = \
        np.matmul(stim_cov_sqrt_inv, stimulus_ensemble)

        return whitened_stimulus_ensemble

    def reconstruct_stimulus(self, len_snippets, stepwidth_snippets, rcond, \
                             smoothing_window_len):
        '''
        This function expects a stimulus ensemble of shape snippet_length x
        snippet_number, where the snippets just have to be appended to yield
        the full stimulus (i.e. snippet_length * snippet_number = stimulus
        length), but the transitions need to be smoothed. The smoothing is
        performed here.

        Inputs:
        -----------------------------------------------------------------------
            len_snippets    int
                            length of the stimulus snippets in samples
        stepwidth_snippets    int
                                stepwidth between the snippets in ms;
        rcond            float
                         rounding condition to compute_stim_cov_inverse_sqrt
        smoothing_window_len    int
                                length of the smoothing window to smooth, in samples

        Outputs:
        -----------------------------------------------------------------------
        the whitened stimulus, reconstructed by appending and smoothing the
        snippets from the whitened ensemble
        '''

#         stim_ensemble = []
#         sample_stepwidth = int((self.temp_res/1000)*stepwidth_snippets)
#
#
#         dim = len(self.current_density)
#
#         [stim_ensemble.append(self.current_density[i:i+len_snippets]) \
#                                         for i in range(0, dim, sample_stepwidth)]
#
#
#
#         stim_ensemble = np.asarray(stim_ensemble)
#         stim_ensemble = np.transpose(stim_ensemble)

        stim_ensemble = self.create_stimulus_ensemble(len_snippets, stepwidth_snippets)
        stim_cov_inv_sqrt = self.compute_stim_cov_inverse_sqrt(len_snippets, rcond)
        white_ensemble = np.matmul(stim_cov_inv_sqrt, stim_ensemble)
        shift = int(smoothing_window_len/2)
        recon = np.concatenate(np.transpose(white_ensemble))

        smooth_recon = smooth(recon, window_len=smoothing_window_len)
        return smooth_recon

    def check_and_plot(self, len_snippets = 200, stepwidth_snippets = 20,\
                        rcond = 10**-5, path = None, load_pickled_version = False):

        '''
        creates a 4x3 plot of
                    (1) an excerpt from x
                    (2) the periodogram of x
                    (3) the covariance of x
                    (4) the autocorrelation of x
        where x
                    (1) the original stimulus
                    (2) the whitened and smoothed version of the stimulus
                    (3) the whitened raw version of the stimulus
        '''

        def format_func(value, tick_number):
            return value/self.temp_res

        fname = 'stimulus'
        if load_pickled_version:
            f = open(r'D:/Lara Analysis/paper pickles/stim_versions.pkl', 'rb')
            stim_versions = pkl.load(f)
            f = open(r'D:/Lara Analysis/paper pickles/cov_versions.pkl', 'rb')
            cov_versions = pkl.load(f)

        else:
            original = self.current_density
            original_cov = self.compute_stimulus_cov(len_snippets, mode = 'original')
            whitened_smoothed = self.white_current_density
            whitened_smoothed_cov = self.compute_stimulus_cov(len_snippets, mode='white_smooth')
            whitened_raw = np.concatenate(np.transpose(self.whiten_stimulus(len_snippets, stepwidth_snippets, rcond)))
            whitened_raw_cov = self.compute_stimulus_cov(len_snippets, mode='white_raw')
    #         stim_versions = dict(zip(['original', 'whitened and smoothed', 'whitened raw'], \
    #                                  [original, whitened_smoothed, whitened_raw]))
    #
    #         cov_versions = dict(zip(['original', 'whitened and smoothed', 'whitened raw'], \
    #                                  [original_cov, whitened_smoothed_cov, whitened_raw_cov]))

            stim_versions = dict(zip(['original', 'whitened and smoothed'], \
                                     [original, whitened_smoothed]))

            cov_versions = dict(zip(['original', 'whitened and smoothed'], \
                                     [original_cov, whitened_smoothed_cov]))

            f = open(r'D:/Lara Analysis/paper pickles/stim_versions.pkl', 'wb')
            pkl.dump(stim_versions, f)
            f = open(r'D:/Lara Analysis/paper pickles/cov_versions.pkl', 'wb')
            pkl.dump(cov_versions, f)

        fs = self.temp_res
        n_cols = 1
        n_rows = 3
        counter = 0

        sample_start = 1000
        sample_stop = 2500
        t_start = sample_start/fs
        t_stop = sample_stop/fs

        time_line = np.linspace(t_start, t_stop, sample_stop-sample_start)

        '''
        plot original stimulus
        '''

#         plt.figure(figsize=[6,8])
#         plt.figure(figsize=[6.4,8])
        plt.figure(figsize=[8,6])
        plt.style.use('seaborn-paper')
        s=13
        mpl.rcParams.update({'font.size':s, 'axes.labelsize':s,\
                          'xtick.labelsize':s, 'ytick.labelsize':s})
        axes_orig = []

#         axes_orig.append(plt.subplot2grid([n_rows, 3], [1, 0]))
#         ax = plt.imshow(cov_versions['original'], cmap = 'Greys')
#         plt.gca().set_axis_off()
# #         plt.colorbar(ticks = [-0.1, 0, 0.3])
#
#
#
#         axes_orig.append(plt.subplot2grid([n_rows, 3], [1, 1]))
#         plt.imshow(self.compute_stim_cov_inverse_sqrt(len_sta=len_snippets, \
#                                                  rcond = rcond), cmap = 'Greys')
#         plt.gca().set_axis_off()
# #         plt.colorbar(ticks = [-2, 0, 5])
#
#         axes_orig.append(plt.subplot2grid([n_rows, 3], [1, 2]))
#         plt.imshow(cov_versions['whitened and smoothed'], cmap = 'Greys')
#         plt.gca().set_axis_off()
#         plt.colorbar(ticks = [0, 0.02, 0.05])

        axes_orig.append(plt.subplot2grid([n_rows, n_cols], [0, counter]))
        plt.plot(time_line, stim_versions['original'][sample_start:sample_stop],\
                 color = '0.5')

        plt.plot(time_line, stim_versions['whitened and smoothed'][sample_start:sample_stop],\
                 color = 'k')
        plt.xlabel('Time [s]')
        plt.ylabel('Current \n'  r'density [$\frac{mA}{cm^2}$]')
        plt.gca().xaxis.set_major_locator(plt.MaxNLocator(4))
        plt.gca().yaxis.tick_right()

        ax = plt.subplot2grid([n_rows, n_cols], [1, counter])

        start = 20

        axes_orig.append(ax)
        '''
        compute the absolute current in Amperes
        '''
        area = 3*((5*0.003)**2)                #active stimulation area, assuming 3 x 5x5 active electrodes, one
                                              #electrode has 0.003 cm length

        orig_stim = stim_versions['original'] * area * 10**-3   #last factor to convert from mA to A
        whitened_smoothed_stim = stim_versions['whitened and smoothed'] * area * 10**-3

        frequencies, psd = periodogram(orig_stim, fs, return_onesided = True)
#         bax = brokenaxes(ylims = ((psd[0], psd[22]),(psd[23], psd[2000])), subplot_spec=psd_ax)
        plt.semilogy(frequencies[start:2000], psd[start:2000], color = '0.5')
#         plt.yscale(value='log')
        frequencies, psd = periodogram(whitened_smoothed_stim, fs, return_onesided = True)
#         bax = brokenaxes(ylims = ((psd[0], psd[20]),(psd[21], psd[2000])), subplot_spec=psd_ax)
        plt.semilogy(frequencies[start:2000], psd[start:2000], color = 'k')
#         plt.yscale(value='log')
#         plt.gca().xaxis.set_major_locator(plt.MaxNLocator(5))
        ax.set_xlim((frequencies[start], frequencies[2000]))
        current_ticks = list(plt.xticks()[0])
        current_ticks.append(4)
        current_ticks.pop(0)
        plt.xticks(current_ticks)
        plt.gca().yaxis.tick_right()
        print(frequencies[20])
#         plt.gca().yaxis.tick_right()
#         plt.gca().xaxis.set_major_locator(plt.MaxNLocator(6))
        plt.xlabel('Frequency [Hz]')
#         plt.gca().yaxis.set_label_position('right')
        plt.ylabel('Power spectral' +'\n' +r'density [$\frac{A}{\sqrt{Hz}}$]', labelpad = 14)

        axes_orig[1].spines["top"].set_visible(False)
#         axes_orig[1].spines["bottom"].set_visible(False)
        axes_orig[1].spines["left"].set_visible(False)
        axes_orig[1].spines["right"].set_visible(False)
#         plt.semilogy(frequencies[20:2000], psd[20:2000])
#         axes_orig.append(plt.subplot2grid([n_rows, n_cols+1], [2, counter]))
#
#         plt.imshow(cov_versions['original'],cmap = 'hot')
#
#         axes_orig.append(plt.subplot2grid([n_rows, n_cols+1], [2, 1]))
#
#         plt.imshow(cov_versions['whitened and smoothed'], cmap='hot')
#
#         axes_orig[2].spines["top"].set_visible(False)
#         axes_orig[2].spines["bottom"].set_visible(False)
#         axes_orig[2].spines["left"].set_visible(False)
#         axes_orig[2].spines["right"].set_visible(False)
#         axes_orig[2].xaxis.set_major_locator(plt.NullLocator())
#         axes_orig[2].yaxis.set_major_locator(plt.NullLocator())
#
#         axes_orig[3].spines["top"].set_visible(False)
#         axes_orig[3].spines["bottom"].set_visible(False)
#         axes_orig[3].spines["left"].set_visible(False)
#         axes_orig[3].spines["right"].set_visible(False)
#         axes_orig[3].xaxis.set_major_locator(plt.NullLocator())
#         axes_orig[3].yaxis.set_major_locator(plt.NullLocator())

        axes_orig.append(plt.subplot2grid([n_rows, n_cols], [2, counter]))
        plt.acorr(stim_versions['original'], maxlags = 200, color = '0.5', \
                  usevlines= False, linewidth = 1, linestyle = '-')

        plt.acorr(stim_versions['whitened and smoothed'], maxlags = 200, \
                  color = 'k', usevlines = False, linestyle = '-')

        plt.xlabel('Time [s]')
        plt.ylabel('Auto-\n correlation')
        axes_orig[-1].xaxis.set_major_formatter(plt.FuncFormatter(format_func))
        plt.gca().xaxis.set_major_locator(plt.MaxNLocator(4))
        plt.gca().yaxis.tick_right()
        for ax in axes_orig:
            ax.spines["top"].set_visible(False)
            ax.spines["left"].set_visible(False)
            ax.spines["right"].set_visible(False)

        '''
        plot whitened and smoothed stimulus
        '''
#         counter = 1
#         axes_ws = []
#         axes_ws.append(plt.subplot2grid([n_rows, n_cols], [0, counter], sharey= axes_orig[0]))
#         plt.plot(time_line, stim_versions['whitened and smoothed'][sample_start:sample_stop])
#
#         axes_ws.append(plt.subplot2grid([n_rows, n_cols], [1, counter]))
#         frequencies, psd = periodogram(stim_versions['whitened and smoothed'], fs, return_onesided = True)
#         plt.plot(frequencies[:2000], psd[:2000])
#         axes_ws[1].spines["top"].set_visible(False)
#         axes_ws[1].spines["bottom"].set_visible(False)
#         axes_ws[1].spines["left"].set_visible(False)
#         axes_ws[1].spines["right"].set_visible(False)
# #         plt.semilogy(frequencies[20:2000], psd[20:2000])
#         axes_ws.append(plt.subplot2grid([n_rows, n_cols], [2, counter]))
#
#         plt.imshow(cov_versions['whitened and smoothed'])
#         axes_ws[2].spines["top"].set_visible(False)
#         axes_ws[2].spines["bottom"].set_visible(False)
#         axes_ws[2].spines["left"].set_visible(False)
#         axes_ws[2].spines["right"].set_visible(False)
#         axes_ws[2].xaxis.set_major_locator(plt.NullLocator())
#         axes_ws[2].yaxis.set_major_locator(plt.NullLocator())
#
#         axes_ws.append(plt.subplot2grid([n_rows, n_cols], [3, counter]))
#         plt.acorr(stim_versions['whitened and smoothed'], maxlags = 200)
#         axes_ws[-1].xaxis.set_major_formatter(plt.FuncFormatter(format_func))

#         plt.tight_layout()

        plt.subplots_adjust(bottom = 0.2,wspace = 0.3, hspace = 1)
        if path is not None:
            plt.savefig(path+fname+'.png', dpi = 300, fmt='png')
            plt.savefig(path+fname+'.pdf', dpi = 300, fmt='pdf')
            plt.savefig(path + fname + '.svg', dpi=300, fmt='svg')
        else:
            plt.show()


import akshare as ak
import matplotlib.pyplot as plt
from pylab import mpl
import pandas as pd
from jqdatasdk  import *
import datetime
import time

def a():
    print('aaa')

def b(fun):
    fun

def timerFun(sched_Timer):
    flag = 0
    while True:
        now = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
        if now == sched_Timer:
            # run_Task()
            print("我在规定时间运行了！！")
            print(now)
            flag-1
            return 0

        else:
            print("时间未到")
            time.sleep(0.5)
            if flag == 1:
                sched_Timer = sched_Timer + datetime.timedelta(day=1)
                flag = 0

def market_open():
    # log.info('函数运行时间(market_open):'+str(context.current_dt.time()))
    #1、读取将要购入的个股代码
    df = pd.read_csv('D:\office\Github\爬虫\自动买卖股票数据采集\data\willbuy_code_list.csv')

    #2、创建名为own_stock的列表，用于存储以购买的个股代码
    own_stock = []

    #3、取得当前的现金并平均分成n份（n为即将购买的个股数）
    # cash = context.portfolio.available_cash/len(df)

    #4、遍历数据
    for code in df['code']:
        # print(code)
        # g.security = f'{code}.XSHE'
        # security = g.security

        #5、调用high_open()函数判断高开是否在0-2%之间
        # if high_open(f'{code}.XSHE'):

            #6、若满足要求用cash买入股票
            # order_value(code, cash)

            #7、将已经买入的股票代码保存入own_stock列表
            own_stock.append(code)
        # else:
            # 若不满足则打印出来
            print("不买入",code)

    #8、将已经买入的股票代码以CSV格式保存入本地文件
    pf = pd.DataFrame(own_stock)
    pf.columns=['code']
    pf.to_csv('D:\office\Github\爬虫\自动买卖股票数据采集\data\own_stock.csv',index=False)

def sell_stock():

    #1、读取已拥有的个股代码
    df = pd.read_csv('D:\office\Github\爬虫\自动买卖股票数据采集\data\own_stock.csv')

    #2、创建Limit_of_drop列表用于存储跌停的个股
    Limit_of_drop = []

    #3、判断数据是否为空,不为空则遍历数据
    if len(df) != 0:
        # for code in df['code']:
            # print(code)
            # g.security = f'{code}.XSHE'
            # security = g.security

            #4、调用change_pct()函数判断其涨跌幅(返回'0'可卖出,返回'-1'为跌停,涨停返回'1')
            state = change_pct('-1')
            if state == '0':

                #5、若满足要求卖出股票
                # order_target(code, 0)

                #6、将已经卖出的股票代码从own_stock文件移除
                df = pd.read_csv('D:\office\Github\爬虫\自动买卖股票数据采集\data\own_stock.csv')
                df01 = df.drop(df[df['code']==688350].index)
                df01.to_csv('D:\office\Github\爬虫\自动买卖股票数据采集\data\own_stock.csv',index=False)

            #7、处理跌停的个股
            elif state == '-1':

                #8、将代码添加进列表
                Limit_of_drop.append(603859)

                #9、将跌停的股票代码从own_stock文件移除
                df = pd.read_csv('D:\office\Github\爬虫\自动买卖股票数据采集\data\own_stock.csv')
                df01 = df.drop(df[df['code']==603859].index)
                df01.to_csv('D:\office\Github\爬虫\自动买卖股票数据采集\data\own_stock.csv',index=False)
                # print("无法卖出",code)
    if  len(Limit_of_drop) != 0:
        #10、将跌停的股票代码以CSV格式保存为Limit_of_drop_stock文件
        drop = pd.DataFrame(Limit_of_drop)
        drop.columns=['code']
        drop.to_csv('D:\office\Github\爬虫\自动买卖股票数据采集\data\Limit_of_drop_stock.csv',index=False)

def change_pct(code):
    #1、获取今天日期
    # g.today = context.current_dt.strftime('%Y-%m-%d')

    # #2、获取上个交易日日期
    # g.start = context.current_dt - datetime.timedelta(day = 1)

    # #3、获取涨跌幅
    # money_flow = get_money_flow(f'{code}.XSHG',start_date=g.start, end_date=g.today, fields="change_pct")

    #4、判断是否满足高开在0-2%之间
    if code == '0':
        return '0'
    elif code == '-1':
        return '-1'
    else:
        return '1'

if __name__ == "__main__":

    # df = ak.stock_zh_a_hist(symbol='688039',start_date="20230301", end_date='20230310', adjust="")
    # print(df)
    # path = '/Users/pengbin/Downloads/mkt_index.xlsx'
    # df.to_excel(path, index=False)
    # date = df['date']
    # close = df['close']
    # print(close)
    # start_date = (datetime.datetime.now() - datetime.timedelta(days = 30))
    # start_date = start_date.strftime('%Y%m%d')
    # end_date = time.strftime('%Y%m%d')

    # df = ak.stock_zh_a_hist(symbol='600095',start_date=start_date, end_date=end_date, adjust="")
    # print(df)
    # tomorrow = (datetime.datetime.now() + datetime.timedelta(days = 1))
    # sell_Timer = datetime.datetime(tomorrow.year,tomorrow.month,tomorrow.day,14,55,0)
    # print(sell_Timer.strftime('%Y%m%d%H%M%S'))
    # timerFun(sell_Timer.strftime('%m%d%H%M%S'))

    # df = pd.read_csv('D:\office\Github\爬虫\自动买卖股票数据采集\data\willbuy_code_list.csv')
    # df01 = df.drop(df[df['code']==688004].index)
    # df01.to_csv('D:\office\Github\爬虫\自动买卖股票数据采集\data\willbuy_code_list.csv',index=False)
    # b(a())
    # market_open()
    # print(change_pct('-1'))
    sell_stock()


import requests
from bs4 import BeautifulSoup

def get_win_data():

    url = 'http://www.cwl.gov.cn/kjxx/ssq/kjgg/' # 您提供的网页 URL
    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'}
    page = requests.get(url, headers=headers)
    soup = BeautifulSoup(page.content, 'html.parser')

    table = soup.select_one('div.table_ssq > table.ssq_table > tbody') # 找到指定位置的表格
    print(table)
    rows = table.find_all('tr')

    for row in rows:
        cols = row.select('td > div.qiu') # 找到每一行中所有的 div.qiu 元素
        cols = [col.text.strip() for col in cols]
        print(cols) # 打印每一行的数据


from sqlalchemy import create_engine
from sqlalchemy import Table,Column,String,MetaData

db_string = "postgresql+psycopg2://postgres:12345@localhost:5432/batch1db"
db = create_engine(db_string)

meta = MetaData(db)

film_table = Table(
    'films_exp_d26',
    meta,
    Column('title',String),
    Column('director',String),
    Column('year',String)
)

with db.connect() as conn:
    # create table
    film_table.create()
    insert_statement = film_table.insert().values(
        title ="Doctor Strange 50",
        director = "Rohit Shetty",
        year = "2050"
    )
    conn.execute(insert_statement)

    # read from table
    print("Reading from table")
    select_statement = film_table.select()
    select_result = conn.execute(select_statement)
    for r in select_result:
        print(r)

    # update table
    print("Updating")
    print(film_table.c)
    update_statement = film_table.update().where(
        film_table.c.year == '2016'
    ).values(title="Amazing Spiderman")
    conn.execute(update_statement)

    # delete
    print("Delete operation")
    delete_statement = film_table.delete().where(
        film_table.c.year == '2016'
    )
    conn.execute(delete_statement)


# small script to visualize a random heat map from raw json data in .txt files

import json, sys, matplotlib.pyplot as plt

if len(sys.argv) != 5:
    exit(
        "[visualize_json_image] Error - missing arguments\n-> input file's path\n-> min index of image to save to output dir\n-> max index of image to save to output dir\n -> output dir"
    )
else:
    INPUT_FILE_PATH: str = sys.argv[1]
    PICTURE_RANGE_MIN: int = int(sys.argv[2])
    PICTURE_RANGE_MAX: int = int(sys.argv[3])
    OUTPUT_DIR: str = sys.argv[4]

with open(INPUT_FILE_PATH, "r") as f:

    for _ in range(PICTURE_RANGE_MIN):
        raw_line = f.readline()

    for i in range(PICTURE_RANGE_MAX - PICTURE_RANGE_MIN + 1):
        raw_line = f.readline()

        # fetching data into the raw input file
        parsed_json: dict = json.loads(raw_line)
        thermal_data_matrix: list = parsed_json["data"]
        theral_meta_data: dict = {k: v for k, v in parsed_json.items() if k != "data"}

        # creating the heatmap plot
        fig, ax = plt.subplots()
        im = ax.imshow(thermal_data_matrix)

        # adding legend
        title: str = f"Random thermal sensor frame heatmap from file {INPUT_FILE_PATH}\nMetadata : {json.dumps(theral_meta_data, indent=4)}"
        cbar = ax.figure.colorbar(im, ax=ax)
        cbar.ax.set_ylabel("Room temperature", rotation=-90, va="bottom")
        ax.set_title(title)

        # final plotting
        fig.tight_layout()
        print(
            f"\r[+] Saving heatmap n°{i + PICTURE_RANGE_MIN} ... [{i + 1}/{PICTURE_RANGE_MAX - PICTURE_RANGE_MIN + 1}]"
        )
        plt.savefig(f"{OUTPUT_DIR}/heatmap_{i+ PICTURE_RANGE_MIN}.png")


""" Organize data"""

import numpy as np

from torch.utils.data import Dataset

class AbsDataset(Dataset):
    """Face Landmarks dataset."""

    def __init__(self, flux_file, lbl_file, mode='train', transform=None, testing=False):
        """
        Args:
            flux_file (string): Path to the csv file with annotations.
            lbl_file (string):
            transform (callable, optional): Optional transform to be applied
                on a sample.
        """
        all_flux = np.load(flux_file)
        all_lbls = np.load(lbl_file)
        # Train, validate or test?
        if mode == 'train':
            if testing:
                self.all_flux = all_flux[:,0:1000]
                self.all_lbls = all_lbls[:,0:1000]
            else:
                self.all_flux = all_flux[:,0:9500]
                self.all_lbls = all_lbls[:,0:9500]
        elif mode == 'validate':
            self.all_flux = all_flux[:,-500:]
            self.all_lbls = all_lbls[:,-500:]

        self.transform = transform

    def __len__(self):
        return self.all_flux.shape[1]

    def __getitem__(self, idx):
        spec = self.all_flux[:, idx]
        isz = int(np.sqrt(spec.size))
        image = spec.reshape((1,isz,isz)).astype(np.float32)
        # Labels
        lbls = self.all_lbls[:,idx]
        lbls_image = lbls.reshape((1,isz,isz)).astype(np.float32)
        sample = {'image': image, 'labels': lbls_image}

        #if self.transform:
        #    sample = self.transform(sample)

        return sample