import re
import torch
from transformers import AutoTokenizer

from typing import List, Tuple, Optional

BOS = "<|endoftext|>"
EOM = "<|endofmask|>"

_END_OF_MASK_OR_NEWLINE_RE = re.compile(r"<\|endofmask\|>|<\|endoftext\|>|\n", re.M)
INFILL_MARKER = "_hole_"

def _extract_maybe_type(text: str) -> str:
    """
    Extracts the text before the end-of-mask, end-of-test, or newline tokens.
    Note that the result may be the empty string.
    """
    return _END_OF_MASK_OR_NEWLINE_RE.split(text, maxsplit=1)[0].strip()

def _prefix_ending_with_newline(str, max_length):
    """
    Produces a prefix of str is at most max_length, but does not split a line.
    """
    return str[:max_length].rsplit("\n", 1)[0]

def _suffix_starting_with_newline(str, max_length):
    """
    Produces a suffix of str is at most max_length, but does not split a line.
    """
    return str[-max_length:].split("\n", 1)[0]

def _clip_text(prefix, suffix, max_length):
    """
    Clip the prefix and suffix to be at most `max_length` characters long.
    The start of the prefix should be clipped, and the end of the suffix
    should be clipped. If both already fit within `max_length`, then do
    nothing.
    """
    # we need at least 2 characters to show something
    assert max_length >= 2

    prefix_len = len(prefix)
    suffix_len = len(suffix)

    if prefix_len + suffix_len <= max_length:
        return prefix, suffix  # nothing to do

    # distribute 3/4 of the max length to the prefix and 1/4 to the suffix
    prefix_max = int(max_length * 0.75)
    suffix_max = max_length - prefix_max

    # remember: we want to clip the start of the prefix and the end of the suffix
    # also, if we have leftover of the prefix, we want to give it to the suffix
    prefix_len = min(prefix_max, prefix_len)
    suffix_len = min(suffix_max, suffix_len)

    # if we have leftover of the prefix, we want to give it to the suffix
    leftover = max_length - prefix_len - suffix_len
    suffix_len += leftover

    return prefix[-prefix_len:], suffix[:suffix_len]

class TypeInference:
    def __init__(self, model, tokenizer, temperature: float = 0.0, type_length_limit: int = 5, max_length: int = 70, device = 0, num_comps: int = 1):
        self.model = model
        self.tokenizer = tokenizer
        self.temperature = temperature
        self.type_length_limit = type_length_limit
        self.max_length = max_length
        self.do_sample = False if temperature == 0 else True
        self.device = device
        self.num_comps = num_comps

    def _generate(self, prefix_suffix_tuples) -> List[str]:
        """
        A canonical function to generate text from a prompt. The length_limit
        limits the maximum length of the generated text (beyond the prompt).
        """
        if type(prefix_suffix_tuples) == tuple:
            prefix_suffix_tuples = [prefix_suffix_tuples]

        print(f"prefix_suffix_tuples: {prefix_suffix_tuples}")

        prompts= [f"{p}: <|mask:0|>{s}<|mask:1|><|mask:0|>" for p, s in prefix_suffix_tuples]

        inputs = self.tokenizer(
            prompts,
            return_tensors="pt",
            return_token_type_ids=False
        ).to(self.device)

        max_length = inputs.input_ids[0].size(0) + self.max_length
        print(max_length)

        with torch.no_grad():
            outputs = self.model.generate(
                **inputs,
                do_sample=True,
                top_p=0.95,
                temperature=self.temperature,
                max_length=max_length,
                pad_token_id=self.tokenizer.pad_token_id
            )

        detok_hypo_strs = [self.tokenizer.decode(output.flatten()) for output in outputs]
        detok_hypo_strs = [string[len(BOS) :] for string in detok_hypo_strs if string.startswith(BOS)]
        return detok_hypo_strs

    def _generate_valid_types(self, prefix: str, suffix: str, retries: int) -> List[str]:
        """
        Given an InCoder-style prompt for infilling, tries to fill <|mask:0|> with a valid
        TypeScript type.
        """
        for _ in range(retries):
            # generated = self._generate([prompt] * self.num_samples)
            generated = self._generate([(prefix, suffix)] * self.num_comps)

            checked_not_empty = []
            for g in generated:
                if g.strip() != "":
                    checked_not_empty.append(g.strip())

            if len(checked_not_empty) == 0:
                continue

            # filled_type = _extract_maybe_type(generated[len(prompt) :])
            # if filled_type == "":
                # filled_type = "any"
                # continue
            return checked_not_empty
        return ["any"]

    # def _get_type_annotation(self, prefix: str, suffix: str) -> List[str]:
        # clipped_prefix, clipped_suffix = _clip_text(
            # prefix, suffix, self.max_length
        # )
        # prompt = f"{clipped_prefix}: <|mask:0|>{clipped_suffix}<|mask:1|><|mask:0|>"
        # filled_types = self._generate_valid_types(
            # clipped_prefix, clipped_suffix, retries=3
        # )
        # return filled_types

    def _infill_one(self, template: str) -> List[str]:
        parts = template.split(INFILL_MARKER, 1)
        print(parts)
        if len(parts) < 2:
            raise ValueError(
                f"Expected at least one {INFILL_MARKER} in template. Got {template}"
            )
        infilled_prefix = parts[0]
        suffix = parts[1].replace(": " + INFILL_MARKER, "")

        print(f"\tleft:\n {infilled_prefix}\n\tright:\n {suffix}")

        clipped_prefix, clipped_suffix = _clip_text(
            infilled_prefix, suffix, self.max_length
        )

        print(
            f"\tclipped left:\n {clipped_prefix}\n\tclipped right:\n {clipped_suffix}")

        # type_annotations = self._get_type_annotation(clipped_prefix, clipped_suffix)
        # return type_annotations
        filled_types = self._generate_valid_types(clipped_prefix, clipped_suffix, retries=3)
        return filled_types

    # returns union now, must be fixed later
    def infer(self, code: str) -> List[str]:
        if INFILL_MARKER not in code:
            return []
        return self._infill_one(code)

def split_string(string: str, max_length: int) -> List[str]:
    return [string[i : i + max_length] for i in range(0, len(string), max_length)]

tokenizer = AutoTokenizer.from_pretrained("facebook/incoder-6B")

def infer(model_dict: dict, code: str, num_comps: int, max_length: int = 2048, temperature: float = 1.0) -> List[str]:
    assert num_comps > 0
    type_inf = TypeInference(
        model=model_dict["model"],
        tokenizer=model_dict["tokenizer"],
        temperature=temperature,
        device=model_dict["device"],
        max_length=max_length,
    )
    return type_inf.infer(code)


from scrapy import Item, Field
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor

class AllRecipesCrawler(CrawlSpider):
    name = 'allrecipes'
    allowed_domains = ['www.allrecipes.com']
    start_urls = ['https://www.allrecipes.com']
    rules = (Rule(LinkExtractor(), callback='parse_item', follow=True),)
    filename = 'allrecipes.txt'
    accept_url = "https://www.allrecipes.com/recipe/"
    reject_urls = ["printview", "page"]

    def parse_item(self, response):
        if response.status == 200:
            url =  response.url

            item = { }
            item['url'] = response.url
            item['referer'] = response.request.headers.get('Referer')
            item['status'] = response.status

            if url.startswith(self.accept_url):
                matches = True
                for prefix in self.reject_urls:
                    if prefix in url:
                        matches = False

                if matches:
                    with open(self.filename, 'at') as f:
                        f.write(url +"\n")

            return item


"""pabo URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/3.1/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path, include, path
from django.conf.urls import url
from allauth.account import views
from rest_framework.schemas import get_schema_view
from django.views.generic import TemplateView
from pegawai.views import GoogleLogin

urlpatterns = [
    path('admin/', admin.site.urls),
    path('', include('pegawai.urls')),

    #kebutuhan dokumentasi
    path('openapi', get_schema_view(
        title="Sistem Kepegawaian",
        description="API untuk melakukan share data kepegawaian",
        # url="http://localhost:8000",
        version="1.0.0"
    ), name="openapi-schema"),
    path('swagger-ui/', TemplateView.as_view(
        template_name="swagger-ui.html",
    ), name="swagger-ui"),

    #kebutuhan auth
    path('rest-auth/google/', GoogleLogin.as_view(), name='google_login'),
    path('accounts/', include('allauth.urls')),
    path('api-auth/login/', views.login, name='account_login'),
    path('api-auth/logout/', views.logout, name='account_logout'),
    path('api-auth/', include('rest_framework.urls', namespace='rest_framework')),

]


# download all gcs file
import os
import datetime

from google.cloud import storage

os.environ["GOOGLE_APPLICATION_CREDENTIALS"]="/PATH/TO/KEY.json"

if not os.path.exists('./DOWNLOAD'):
    os.makedirs('./DOWNLOAD')

def list_blobs_with_prefix(bucket_name, prefix, delimiter=None):
    file_list = []
    storage_client = storage.Client()

    blobs = storage_client.list_blobs(
        bucket_name, prefix=prefix, delimiter=delimiter
    )

    print("Blobs:")
    i = 0
    today = datetime.date.today()
    # check today's folder
    if not os.path.exists(f'./DOWNLOAD/{today}'):
        os.makedirs(f'./DOWNLOAD/{today}')

    yesterday = datetime.date.today() - datetime.date.resolution
    list_name = f'DOWNLOAD/download_list_{today}.txt'

    for blob in blobs:
        update_day = blob.updated.date()
        if update_day >= yesterday and update_day < today:
            print(blob.name)
            filename = blob.name.split('/')[-1]
            updated = blob.updated.strftime("%Y-%m-%d %H:%M:%S")
            print(f'{filename} | {updated}')
            print(f'file size: {blob.size}')

            # f.write(f'{i:04d} | {filename:16} | {updated}\n')
            file_list.append(f'{i:04d} | {filename:16} | {updated}\n')

            # blob download

            destination_file_name = f'DOWNLOAD/{today}/{filename}'
            blob.download_to_filename(destination_file_name)

            print(
                "Blob [{}] downloaded to [{}]".format(
                    filename, destination_file_name
                )
            )
            i += 1

    with open(list_name, 'w') as f:
        for file_info in file_list:
            f.write(file_info)

list_blobs_with_prefix('BUCKET','PREFIX')


from jobs.payment_reminder import payment_reminder
from utils.utils import extract_match_time, extract_match_day, compute_seconds_from_now

def trigger_payment_reminder(update, context, day, time):
    chat_id = update.effective_message.chat_id
    match_time = extract_match_time(time)
    reminder_time_from_now = -1

    if match_time == -1:
        match_date = extract_match_day(day)
        waiting_time_in_seconds = 36 * 3600
    else:
        match_date = extract_match_day(f"{day} {time}")
        waiting_time_in_seconds = 2 * 3600

    if match_date != -1:
        reminder_time_from_now = compute_seconds_from_now(match_date) + waiting_time_in_seconds

    if reminder_time_from_now > 0:
        context.job_queue.run_once(payment_reminder, reminder_time_from_now, context=context, name=str(chat_id))


from __future__ import absolute_import
import argparse
import os
import sys

from hudsucker.industries import log
from hudsucker.proxy import HulaHoop

parser = argparse.ArgumentParser(description='Hudsucker Proxy Generator - generate MtG proxy sheets')
parser.add_argument('input', metavar='FILE', nargs='+',
                    help='each line of FILE should be a MtG card name, or a url')
parser.add_argument('-v', '--verbose', dest='verbose', action='store_true',
                    help='print verbose details')

output_group = parser.add_argument_group('proxy sheet options')
output_group.add_argument('-b', '--border', dest='border', metavar='N', default=0.072, type=float,
                          help='border width in inches, defaults to 0.072')
output_group.add_argument('-o', '--output', dest='output_dir', metavar='DIR', default='.',
                          help='output dir, defaults to current dir')
output_group.add_argument('-p', '--resolution', dest='resolution', metavar='N', default=600.0, type=float,
                          help='print resolution of output PDF, defaults to 600')
output_group.add_argument('-q', '--quality', dest='quality', metavar='N', default=95, type=int,
                          help='quality to use for JPEG encoding, defaults to 95')
output_group.add_argument('-s', '--site', dest='site', metavar='URL', default='mythicspoiler.com',
                          help='site to search for card images, defaults to mythicspoiler.com')

cache_group = parser.add_argument_group('caching options', description='NOTE: '
                                        'Careful turning off cache, search engines may ban your IP')
cache_group.add_argument('-c', '--cache', dest='cache_dir', metavar='DIR', default='hudsucker_cache',
                         help='cache dir, defaults to hudsucker_cache')
cache_group.add_argument('-n', '--no-cache', dest='no_cache', action='store_true',
                         help='don\'t cache any downloaded files')
cache_group.add_argument('-r', '--refresh', dest='refresh_cache', action='store_true',
                         help='force refresh of any cached downloads')

def main(argv=None):
    if argv is None:
        argv = sys.argv

    args = parser.parse_args(argv[1:])

    log.verbose = args.verbose

    hula_hoop = HulaHoop(
        border=args.border,
        quality=min(95, max(1, args.quality)),
        resolution=args.resolution,
        site=args.site,
        output_dir=os.path.expanduser(args.output_dir),
        cache_dir=os.path.expanduser(args.cache_dir),
        no_cache=args.no_cache,
        refresh_cache=args.refresh_cache)

    hula_hoop.run(args.input)

if __name__ == '__main__':
    sys.exit(main())


import os
import numpy as np
import cv2

# Directory Config initialization
inputhdatasetpath="COLOR_DATASET/train"
skindataset="SKIN_DATASET/train"
if not os.path.exists(skindataset):
    os.makedirs(skindataset)

dir_list = os.listdir(inputhdatasetpath)
# Reads all the images from the directory and converts it to gray scale for training
for dirname in dir_list:
    newdirpath=skindataset+"//"+dirname
    if not os.path.exists(newdirpath):
        os.makedirs(newdirpath)

    olddirpath=inputhdatasetpath+"//"+dirname
    images_path = os.listdir(olddirpath)
    imageno=1
    for n, image in enumerate(images_path):

        path=(os.path.join(olddirpath, image))

        image = cv2.imread(path)
        try:
          grayscale = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
        except:
          print(olddirpath, images_path, path)
          exit(0)

        filename=str(imageno)
        imagename=filename+".jpg"
        newimagepath=newdirpath+"//"+imagename
        img_YCrCb = cv2.cvtColor(image, cv2.COLOR_BGR2YCrCb)
        #skin color range for hsv color space
        YCrCb_mask = cv2.inRange(img_YCrCb, (0, 135, 85), (255,180,135))
        YCrCb_mask = cv2.morphologyEx(YCrCb_mask, cv2.MORPH_OPEN, np.ones((3,3), np.uint8))

        YCrCb_result = cv2.bitwise_not(YCrCb_mask)
        cv2.imwrite(newimagepath, YCrCb_result)
        imageno=imageno+1

    print(olddirpath+" all images are converted in Skin Color \n\n")


from django.shortcuts import render, get_object_or_404, redirect
from .models import Event, Venue
from .forms import VenueForm, EventForm, EventFormAdmin

from django.http import HttpResponse
import csv

#for pdf
from django.http import FileResponse
import io
from reportlab.pdfgen import canvas
from reportlab.lib.units import inch
from reportlab.lib.pagesizes import A4

#for pagination
from django.core.paginator import Paginator

from django.contrib import messages
from django.contrib.auth.models import User

from django.db.models import Q
import calendar
from datetime import datetime

# Create your views here.
def home(request):
    year = datetime.now().year
    month=datetime.now().strftime('%B')
    month = month.capitalize()

    # Convert month from name to number
    month_number = list(calendar.month_name).index(month)
    month_number = int(month_number)

    event_list = Event.objects.filter(
        event_date__year = year,
        event_date__month = month_number
        )
    return render(request, 'events/home.html', {"event_list": event_list})

def all_events(request):
    events = Event.objects.all().order_by("-event_date")
    return render(request, 'events/events_list.html', {'events':events})

def add_venue(request):
    submitted = False
    if request.method == "POST":
        form = VenueForm(request.POST, request.FILES)
        if form.is_valid():
            venue = form.save(commit=False)
            venue.owner = request.user.id  #logged in user-id
            venue.save()
            #form.save()
            submitted = True
    else:
        form = VenueForm()

    return render(request, 'events/add_venue.html', {'form' : form, 'submitted':submitted})

def all_venues(request):
    venue_lists= Venue.objects.all().order_by("name")
    #setup pagination , 2 venues at a time to display
    p = Paginator(venue_lists, 2)
    page = request.GET.get('page')
    venues_pg = p.get_page(page)

    return render(request, 'events/venues.html', {'venues_pg': venues_pg})

def show_venue(request, pk):
    venue= Venue.objects.get(pk=pk)
    venue_owner = User.objects.get(pk=venue.owner)
    return render(request, 'events/show_venue.html', {'venue':venue, 'venue_owner': venue_owner})

def search_venues(request):
    if request.method == "POST":
        searched=request.POST["searched"]
        #icontains case insensitive
        venues =Venue.objects.filter(name__icontains = searched)
        return render(request, 'events/search_venues.html', {'searched':searched, 'venues': venues})
    else:
        return render(request, 'events/search_venues.html')

def update_venue(request, pk):
    venue=get_object_or_404(Venue, pk=pk)
    form = VenueForm(request.POST or None, request.FILES or None, instance = venue)
    if request.method == "POST":
        if form.is_valid():
            form.save()
            return redirect('list-venues')
    return render(request, 'events/update_venue.html', {'form':form})

def add_event(request):
    submitted = False
    if request.method == "POST":
        if request.user.is_superuser:
            form = EventFormAdmin(request.POST)
            if form.is_valid():
                form.save()
                submitted = True
        else:
            form = EventForm(request.POST)
            if form.is_valid():
                event = form.save(commit=False)
                event.manager = request.user  #logged in user
                event.save()
                submitted = True
    else:
        if request.user.is_superuser:
            form = EventFormAdmin()
        else:
            form = EventForm()

    return render(request, 'events/add_event.html', {'form' : form, 'submitted':submitted})

def update_event(request, pk):
    event=get_object_or_404(Event, pk=pk)
    if request.user.is_superuser:
        form = EventFormAdmin(request.POST or None, instance = event)

    else:
        form = EventForm(request.POST or None, instance = event)

    if request.method == "POST":
        if form.is_valid():
            form.save()
            return redirect('list-events')

    return render(request, 'events/update_event.html', {'event': event, 'form':form})

def delete_event(request, pk):
    event=get_object_or_404(Event, pk=pk)
    if request.user == event.manager:
        event.delete()
        messages.success(request, "Event deleted successfully!!!")
    else:
        messages.success(request, "You are not authorized")

    return redirect('list-events')

def delete_venue(request, pk):
    if request.user.is_authenticated:
        venue=get_object_or_404(Venue, pk=pk)
        venue.delete()
        return redirect('list-venues')
    else:
        return redirect('login')

#Generate Text file of Venues
def venue_text(request):
    response = HttpResponse(content_type="text/plain")
    response['Content-Disposition'] = 'attachment; filename = venues.txt'

    venues = Venue.objects.all()
    #create blank list
    lines = []

    for venue in venues:
        lines.append(f'{venue.name}\n')
        lines.append(f'{venue.address}\n')
        lines.append(f'{venue.pin_code}\n')
        lines.append(f'{venue.phone_no}\n')
        lines.append(f'{venue.email_address}\n')
        lines.append(f'\n')

    response.writelines(lines)
    return response

#Generate Text file of Venues
def venue_csv(request):
    response = HttpResponse(content_type="text/csv")
    response['Content-Disposition'] = 'attachment; filename = venues.csv'

    writer = csv.writer(response)

     #Add column headings to the csv file
    writer.writerow(["Venue Name", "Address", "Pin Code", "Phone No", "Email Address"])

    venues = Venue.objects.all()

    for venue in venues:
        writer.writerow([venue.name, venue.address,venue.pin_code,venue.phone_no,venue.email_address])

    return response

def venue_pdf(request):
    buff = io.BytesIO()
    canv = canvas.Canvas(buff, pagesize=A4, bottomup=0)

    textobj = canv.beginText()
    textobj.setTextOrigin(inch, inch)
    textobj.setFont('Helvetica', 14)

    venues = Venue.objects.all()
    #create blank list
    lines = []

    for venue in venues:
        lines.append(venue.name)
        lines.append(venue.address)
        lines.append(venue.pin_code)
        lines.append(venue.phone_no)
        lines.append(venue.email_address)
        lines.append("============================")

    for line in lines:
        textobj.textLine(line)

    canv.drawText(textobj)
    canv.showPage()
    canv.save()
    buff.seek(0)

    return FileResponse(buff, as_attachment=True, filename='venues.pdf')

def my_events(request):
    if request.user.is_authenticated:
        me = request.user.id
        events = Event.objects.filter(attendees=me)
        return render(request, "events/my_events.html", {"events": events})
    else:
        messages.success(request, "You are not authorized")
        return redirect('home')

def search_events(request):
    if request.method == "POST":
        searched=request.POST["searched"]

        #multiple column search
        lookup = Q(name__icontains = searched) | Q(description__icontains = searched)
        #icontains case insensitive
        events =Event.objects.filter(lookup)
        return render(request, 'events/search_events.html', {'searched':searched, 'events': events})
    else:
        return render(request, 'events/search_events.html')

def admin_approval(request):
    #get venue lists
    venue_list = Venue.objects.all()

    #Get counts
    event_count = Event.objects.filter(approved=True).count()
    venue_count = Venue.objects.all().count()
    user_count = User.objects.all().count()
    events = Event.objects.all().order_by("-event_date")
    if request.user.is_superuser:
        if request.method=="POST":
            id_list = request.POST.getlist("boxes")

            #uncheck all events
            events.update(approved=False)

            #update the database
            for id in id_list:
                Event.objects.filter(pk=int(id)).update(approved=True)

            messages.success(request, "Events Approved !!!")
            return redirect("list-events")
    else:
        messages.success(request, "You are not authorized to access this page")
        return redirect('home')

    return render(request, 'events/admin_approval.html', {"events": events,
                                                          "event_count":event_count,
                                                          "venue_count": venue_count,
                                                          "user_count": user_count,
                                                          "venue_list": venue_list})

def venue_events(request, venue_id):
    #get the venue
    venue = Venue.objects.get(pk=venue_id)

    #get all events associated with this venue
    events = Event.objects.filter(venue=venue)
    #events = venue.event_set.all()

    if events:
        return render(request, 'events/venue_events.html', {"events": events})
    else:
        messages.success(request, "There are no events at this venue for this time")
        return redirect('admin-approval')

def show_event(request, event_id):
    event = Event.objects.get(pk=event_id)
    return render(request, 'events/show_event.html', {"event": event})


from django.shortcuts import render
from app.models import *
from .mail_sender import Mail_sender

# Create your views here.

def frontpage_view(request, type):
    products = Product.objects.all()
    companies = Company.objects.all()

    email_sent = False

    if request.method == "POST":
        if 'other_name' in request.POST:
            let = Letter()
            let.name = request.POST.get('name')
            let.email = request.POST.get('email')
            let.the_lucky_one = request.POST.get('other_name')
            let.save()
        elif 'frontpage_email' in request.POST and (type == 'privat'):
            Mail_sender.send_tip_mail(request.POST.get('frontpage_email'), request.POST.get('frontpage_name'))
            email_sent = True
        elif 'frontpage_email' in request.POST:
            Mail_sender.send_cat_request(request.POST.get('frontpage_email'))
            email_sent = True
        elif 'to_email' in request.POST:
            Mail_sender.send_wish_mail(request.POST.get('to_email'), request.POST.get('to_name'),
                                       request.POST.get('from_name'))
            email_sent = True
        else:
            sub = Subscriber()
            sub.name = request.POST.get('name')
            sub.email = request.POST.get('email')
            sub.save()

    template_url = 'frontpage.html' if (type == 'privat') else 'frontpage_company.html'

    return render(request, template_url, {
        'products': products,
        'companies': companies,
        'email_sent': email_sent,
    })

def about_view(request):
    return render(request, 'about.html')

def contact_view(request):
    return render(request, 'contact.html')

def package_view(request):
    packages = Package.objects.filter(is_visible=True)

    if request.method == "POST":
        Mail_sender.send_wish_mail(request.POST.get('to_email'), request.POST.get('to_name'),
                                   request.POST.get('from_name'), request.POST.get('package_name'))

    return render(request, 'packages.html', {
        'packages': packages
    })

def product_view(request, pId):
    product = Product.objects.get(id=pId)

    return render(request, 'product.html', {
        'product': product
    })


from random import randint
from typing import Text, Type
from PyQt5 import QtCore, QtGui, QtWidgets
import xlwt
import LoadFile
from selenium import webdriver
from selenium.webdriver.common.alert import Alert
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.common.exceptions import TimeoutException, UnexpectedAlertPresentException, NoAlertPresentException
from time import sleep
from urllib.parse import quote
from sys import platform

def saveContacts(validNumbers):
    import xlwt
    wbk = xlwt.Workbook()
    sheet = wbk.add_sheet("sheet", cell_overwrite_ok=True)
    for idx , num in enumerate(validNumbers):
        try:
            teext = str(num)
            sheet.write(idx, 0, teext)
        except AttributeError:
            pass

    wbk.save('WAContact.xls')

if __name__ == "__main__":
    import LoadFile
    listNumbers = LoadFile.CSVloadNumbers('set_1 _2000.csv')['Mobile Number']

    option  = Options()
    # option.add_argument('headless')
    option.add_argument("--incognito")
    delay = 30
    print(listNumbers[:5])
    input("Press enter to start checking")
    driver = webdriver.Chrome("chromedriver.exe", options=option)

    isValid = []

    sleep(10)
    for idx , num in enumerate(listNumbers):

        driver.get("https://whatsapptools.net/check-numbers")

        sleep(2)
        driver.find_element_by_xpath("/html/body/div/div/div/div/div/div/div/div/canvas")

        import bs4
        html_source = driver.page_source
        soup = bs4(html_source, 'html.parser')
        divs = soup.find('div', {"class":"_1pw2F"})
        print(divs.attrs["data-ref"])

        driver.find_element_by_xpath("/html/body/div/div/div/div/div/select/option[text()='India +91']").click()
        sleep(2)
        driver.find_element_by_xpath("/html/body/div/div/div/div/input").clear()
        sleep(2)
        driver.find_element_by_xpath("/html/body/div/div/div/div/input").send_keys(num)
        sleep(2)

        driver.find_element_by_xpath("/html/body/div/div/button").click()

        res = str(driver.find_element_by_xpath("/html/body/div/div/p").text)
        sleep(10)
        print(idx,num + " : result : " + res)
        if(res == 'Exists on WhatsApp!'):
            isValid.append(num)
        else:
            continue
        sleep(2)

    saveContacts(isValid)
    driver.close()
    # driver.find_("email").send_keys("abhishek.gupta1608@toppr.in")


###################
# utilities.py
#
# Various utility methods

# Arka Mallela MD, 2020-22
#####################

import os
import os.path as op
import sys
from contextlib import contextmanager

import matplotlib.pyplot as plt
import matplotlib.ticker
import statsmodels.api as sm

#Make discrete matplotlib color bar
def custom_cmap(colors, bad_val=0, padding=1.05):
    """ Create a custom colormap from a dictionary

    Args:
        colors <dict>: Levels to colornames
        padding <float>: Offset levels
    """
    assert (isinstance(colors, dict))
    assert (bad_val in colors)

    labels = sorted(colors.keys())
    clrs = [colors[l] for l in labels]
    levels = sorted([-1000, 0] + list(map(lambda x: x * padding, filter(None, labels))))
    cmap, norm = matplotlib.colors.from_levels_and_colors(levels, clrs)
    cmap.set_bad(color=colors[bad_val], alpha=1.)
    return cmap, norm, labels

#Snippet from online to format matplotlib ticker
class FormatScalarFormatter(matplotlib.ticker.ScalarFormatter):
    def __init__(self, fformat="%1.1f", offset=True, mathText=True):
        self.fformat = fformat
        matplotlib.ticker.ScalarFormatter.__init__(self,useOffset=offset,
                                                        useMathText=mathText)
    def _set_format(self, vmin, vmax):
        self.format = self.fformat
        if self._useMathText:
            self.format = '$%s$' % matplotlib.ticker._mathdefault(self.format)

#Utility methods
#Makes dir if does not exist, returns dir name for inline calling
def make_dir(dir_to_make):
    if not op.exists(dir_to_make):
        os.makedirs(dir_to_make)
    return dir_to_make

#Plot regression diagnostics on given model
def plot_regression_dx(model, var):
    fig = plt.figure(figsize=(12, 8))
    fig = sm.graphics.plot_regress_exog(model, var, fig=fig)
    plt.show()

#Add significance labels to matplotlib
def barplot_annotate_asterisk(num1, data, center, height, dh=.05, fs=None, maxasterix=None, ):
    """
    Annotate single sterisk group

    :param num1: number of left bar to put bracket over
    :param data: string to write or number for generating asterixes
    :param center: centers of all bars (like plt.bar() input)
    :param height: heights of all bars (like plt.bar() input)
    :param yerr: yerrs of all bars (like plt.bar() input)
    :param dh: height offset over bar / bar + yerr in axes coordinates (0 to 1)
    :param barh: bar height in axes coordinates (0 to 1)
    :param fs: font size
    :param maxasterix: maximum number of asterixes to write (for very small p-values)
    """

    if type(data) is str:
        text = data
    else:
        # * is p < 0.05
        # ** is p < 0.005
        # *** is p < 0.0005
        # etc.
        text = ''
        p = .05

        while data < p:
            text += '*'
            p /= 10.

            if maxasterix and len(text) == maxasterix:
                break

        if len(text) == 0:
            text = 'n.s.'

    text_loc = (center[num1], height[num1] + dh)

    kwargs = dict(ha='center', va='bottom')
    if fs is not None:
        kwargs['fontsize'] = fs

    plt.text(*text_loc, text, **kwargs)

#Add significance labels to matplotlib
def barplot_annotate_brackets(num1, num2, data, center, height, yerr=None, dh=.05, barh=.05, fs=None, maxasterix=None, ):
    """
    Annotate barplot with p-values.

    :param num1: number of left bar to put bracket over
    :param num2: number of right bar to put bracket over
    :param data: string to write or number for generating asterixes
    :param center: centers of all bars (like plt.bar() input)
    :param height: heights of all bars (like plt.bar() input)
    :param yerr: yerrs of all bars (like plt.bar() input)
    :param dh: height offset over bar / bar + yerr in axes coordinates (0 to 1)
    :param barh: bar height in axes coordinates (0 to 1)
    :param fs: font size
    :param maxasterix: maximum number of asterixes to write (for very small p-values)
    """

    if type(data) is str:
        text = data
    else:
        # * is p < 0.05
        # ** is p < 0.005
        # *** is p < 0.0005
        # etc.
        text = ''
        p = .05

        while data < p:
            text += '*'
            p /= 10.

            if maxasterix and len(text) == maxasterix:
                break

        if len(text) == 0:
            text = 'n. s.'

    lx, ly = center[num1], height[num1]
    rx, ry = center[num2], height[num2]

    if yerr:
        ly += yerr[num1]
        ry += yerr[num2]

    ax_y0, ax_y1 = plt.gca().get_ylim()
    dh *= (ax_y1 - ax_y0)
    barh *= (ax_y1 - ax_y0)

    y = max(ly, ry) + dh

    barx = [lx, lx, rx, rx]
    bary = [y, y+barh, y+barh, y]
    mid = ((lx+rx)/2, y+barh)

    plt.plot(barx, bary, c='black')

    kwargs = dict(ha='center', va='bottom')
    if fs is not None:
        kwargs['fontsize'] = fs

    plt.text(*mid, text, **kwargs)

#Convert CM

#Shift pandas column
def shift_column(df, colname, new_idx):
    col = df[colname]
    df.drop(labels=[colname], axis=1, inplace=True)
    df.insert(new_idx, colname, col)
    return df

#Supress stdout for a bit
@contextmanager
def suppress_stdout():
    with open(os.devnull, "w") as devnull:
        old_stdout = sys.stdout
        sys.stdout = devnull
        try:
            yield
        finally:
            sys.stdout = old_stdout

@contextmanager
def suppress_stderr():
    with open(os.devnull, "w") as devnull:
        old_stderr = sys.stderr
        sys.stderr = devnull
        try:
            yield
        finally:
            sys.stderr = old_stderr


from setuptools import setup
from setuptools import find_packages

install_requires = [
    "flask",
    "flask-socketio",
    "flask-sqlalchemy",
    "flask-login",
    "flask-migrate",
    "flask-wtf",
    "email-validator",
    "eventlet"
]

setup(
    name="pyChat",
    version="0.2",
    description="A real-time chat application developed using flask microframework and flask-socketio.",
    packages=find_packages(),
    author="John Paul Del Mundo",
    author_email="jpdelmundo223@gmail.com",
    license="MIT",
    requires=install_requires
)


from typing import Union, Optional, Iterable
from ..Stroke import Stroke
from ..Fill import Fill
from ..Clip import Clip
from ..Item import Item
from .Shape import Shape
from ..util.Vector2 import Vector2

class Circle(Shape):
    """
    Circle shape primitive:
    SVG circle is center alligned be default.
    """
    # create circle as an ellipse with same x and y radius
    def __init__(self,
            clip: Clip, position: Vector2,
            radius: Union[int, float],
            stroke: Optional[Stroke] = None, fill: Optional[Fill] = None,
            children: Iterable[Item] = [], rotation: Union[int, float] = 0
        ):
        super().__init__(clip, position,
                         stroke=stroke, fill=fill,
                         children=children, rotation=rotation)
        self.radius: Union[int, float] = radius

    def render(self) -> str:
        # Create circle SVG instance
        s = f'<circle cx="{self.position.x}" cy="{self.position.y}" r="{self.radius}"'
        # Apply rotation if needed
        if abs(self.rotation) > 1e-6:
            s += f' transform="rotate({self.rotation} {self.position.x} {self.position.y})"'
        # Apply stroke and fill
        if self.stroke != None:
            s += ' ' + self.stroke.render()
        if self.fill != None:
            s += ' ' + self.fill.render()
        else:
            s += ' ' + 'fill="#000000" fill-opacity="0.0"'
        s += ' />'
        # Render Children
        if len(self.children) > 0:
            s += '\n'
            s += self.render_children()
        return s

    def defs(self) -> str:
        # Create defs
        s = super().defs()
        stroke_width = 0 if self.stroke == None else self.stroke.width
        if self.clip != None and self.clip.is_inner():
            s += f'<circle cx="0" cy="0" r="{self.radius}"'
        elif self.clip != None and self.clip.is_outer():
            s += f'<circle cx="0" cy="0" r="{self.radius+stroke_width//2+1}"'
        # Apply rotation if needed
        if abs(self.rotation) > 1e-6:
            s += f' transform="rotate({self.rotation})"'
        return s + ' />\n</clipPath>'

    def get_height(self) -> Union[int, float]:
        return self.radius * 2

    def get_width(self) -> Union[int, float]:
        return self.radius * 2


import os
import numpy as np
import matplotlib.pyplot as plt
import cv2
import shutil
from PyQt5.QtWidgets import QMessageBox

class DistanceMapGenerator:
    def __init__(
            self,
            src_dir,
            sample_dir = "samples",
            label_dir = "labels") -> None:
        self.src_dir = src_dir
        self.sample_dir = sample_dir
        self.label_dir = label_dir

    def generate_filtered_dataset(self, dst_dir, folder_name):
        if not os.path.exists(dst_dir):
            os.makedirs(dst_dir)

        if not os.path.exists(os.path.join(dst_dir,folder_name)):
            os.makedirs(os.path.join(dst_dir,folder_name))
            os.makedirs(os.path.join(dst_dir,folder_name,self.sample_dir))
            os.makedirs(os.path.join(dst_dir,folder_name,self.label_dir))

        for file in os.listdir(os.path.join(self.src_dir,folder_name,self.sample_dir)):
            file_path_src = os.path.join(src_dir,folder_name, self.sample_dir,file)
            file_path_dst = os.path.join(dst_dir,folder_name, self.sample_dir,file)

            label_path_src = os.path.join(src_dir,folder_name, self.label_dir,file)
            label_path_dst = os.path.join(dst_dir,folder_name, self.label_dir,file)
            img_raw = cv2.imread(file_path_src)

            print(file_path_src)

            print(img_raw)

            plt.imshow(img_raw)
            plt.show(block=False)

            msgBox = QMessageBox()
            msgBox.setIcon(QMessageBox.Information)
            msgBox.setText("Message box pop up window")
            msgBox.setWindowTitle("QMessageBox Example")
            msgBox.setStandardButtons(QMessageBox.Ok | QMessageBox.Cancel)

            returnValue = msgBox.exec()
            if returnValue == QMessageBox.Ok:
                shutil.copy(file_path_src,file_path_dst )
                shutil.copy(label_path_src,label_path_dst )

            plt.close()

if __name__ == "__main__":
    src_dir = "/home/ws/sc1357/data/BBBC038"

    dst_dir = "/home/ws/sc1357/data/filtered_BBBC038"

    dmg_obj = DistanceMapGenerator(src_dir)
    dmg_obj.generate_filtered_dataset(dst_dir, "test")


from django.urls import path

from . import views

urlpatterns = [
    path('', views.index, name='index'),
    path('login', views.login_view, name='login'),
    path('logout', views.logout_view, name='logout'),
    path('register', views.register, name='register'),
    path('user', views.user, name='user'),
    path('problems', views.problems, name='problems'),
    path('scores', views.scores, name='scores'),
]


# -*- coding: utf-8 -*-
'''
Created on 2017年4月12日

@author: chenyitao
'''

import gevent

from conf import ParserSite
from common.queues import ParserQueues
from common import TDDCLogging
from base import StoragerBase

class ParseStorager(StoragerBase):
    '''
    classdocs
    '''

    FAMILY = 'valuable'

    _shared_state={}
    def __new__(cls,*args,**kwargs):
        obj=super(ParseStorager,cls).__new__(cls,*args,**kwargs)
        obj.__dict__=cls._shared_state
        return obj

    def __init__(self):
        super(ParseStorager, self).__init__(True, True)

    @staticmethod
    def pull(self, task):
        ParserQueues.PARSE.put(task)

    def _pull(self):
        while True:
            task = ParserQueues.PARSE.get()
            if not task:
                continue
            if not task.platform or not task.row_key:
                TDDCLogging.error('Task Exception(Parse DB Manager): [%s:%s]' % (task.platform,
                                                                                task.row_key))
                continue
            success, ret = self._db.get_from_hbase(task.platform + ParserSite.PLATFORM_SUFFIX,
                                                   task.row_key,
                                                   'source',
                                                   'content')
            if not success:
                ParserQueues.PARSE.put(task)
                gevent.sleep(1)
                continue
            if not ret:
                continue
            for _, value in ret.items():
                ParserQueues.WAITING_PARSE.put((task, value))
                break

def main():
    ParseStorager()
    while True:
        gevent.sleep()

if __name__ == '__main__':
    main()


from django.shortcuts import render
from django import forms
from django.http import HttpResponseRedirect
from django.shortcuts import render

# Create your views here.

class NewTaskForm(forms.Form):
    task = forms.CharField(label="New Task")

def index(request):
    if "tasks" not in request.session:
        request.session["tasks"] = []
    return render(request, "tasks/index.html", {
        'tasks': request.session["tasks"]
    })

def add(request):
    # if the form is valid, this is how we get data form the form
    if request.method == "POST":
        form = NewTaskForm(request.POST)
        if form.is_valid():
            task = form.cleaned_data["task"]
            request.session["tasks"] += [task]
            return HttpResponseRedirect('/tasks')
        else:  # we send back the existing data if it does not work
            return render(request, "tasks/add.html", {
                'form': form
            })

    return render(request, "tasks/add.html", {
        'form': NewTaskForm()
    })


#!/usr/bin/env python

from datetime import datetime
import hmac
from hashlib import sha1
import json
import urlparse  # TODO: remove in favour of better lib
import urllib

import requests

from pyptv.platform_ import Platform  # don't clobber the builtin platform
from pyptv.direction import Direction
from pyptv.stop import StopFactory
from pyptv.line import LineFactory
from pyptv.run import RunFactory
from pyptv.outlet import OutletFactory
from pyptv.disruption import DisruptionFactory
from pyptv.location import parse_location
from pyptv.utils import parse_datetime_tz

API_BASE_URL = "http://timetableapi.ptv.vic.gov.au/"

class PTVClient(object):
    MODES = {"train":      0,
             "tram":       1,
             "bus":        2,
             "vline":      3,
             "nightrider": 4,
             "ticket_outlet": 100,
             }

    FLAGS = {"RR": "Reservations Required",
             "GC": "Guaranteed Connection",
             "DOO": "Drop Off Only",
             "PUO": "Pick Up Only",
             "MO": "Mondays only",
             "TU": "Tuesdays only",
             "WE": "Wednesdays only",
             "TH": "Thursdays only",
             "FR": "Fridays only",
             "SS": "School days only",
             }

    def __init__(self, developer_id=None, api_key=None):

        self.developer_id = developer_id
        self.api_key = api_key

    def _api_request(self, api_path, timed=True):
        """Call some api end point and return the raw response.
        API request will have proper signing key appended.
        """

        parsed = urlparse.urlparse(api_path)

        # parse out current query
        query = urlparse.parse_qsl(parsed.query)

        # add timestamp
        if timed:
            now = datetime.utcnow().replace(microsecond=0).isoformat() + 'Z'
            query.append(('timestamp', now))

        # add developer id
        query.append(('devid', self.developer_id))

        unsigned_query = urllib.urlencode(query)
        unsigned_parsed = parsed._replace(query=unsigned_query)

        unsigned_path = unsigned_parsed.geturl()

        digest = hmac.new(self.api_key, unsigned_path, sha1)

        signature = digest.hexdigest()

        query.append(('signature', signature))
        signed_query = urllib.urlencode(query)
        signed_parsed = unsigned_parsed._replace(query=signed_query)

        signed_path = signed_parsed.geturl()

        signed_url = urlparse.urljoin(API_BASE_URL, signed_path)

        req = requests.get(signed_url)

        data = json.loads(req.content)

        return data

    # API methods:

    def healthcheck(self):
        """Send off a health check to check the status of the system, the
        local clock and the API credentials.
        """
        return self._api_request("/v2/healthcheck")

    def stops_nearby(self, location, mode=None, limit=None,
                     with_distance=False):
        """Return stops near a location.

        Args:
            location: one of (lat, lon), a Location object, or something that
                    has a location property (which would be a Location object)
            mode: (optional) filter results for only this tramsport mode
            limit: (optional) only return this many results
            with_distance: (optional) return tuples of (Stop, distance)
        Returns:
            List of stops or a list of tuples in the form (stop, distance) if
            with_distance is True
        """

        base_path = "/v2/nearme/latitude/{lat}/longitude/{lon}"

        lat, lon = parse_location(location)

        path = base_path.format(lat=lat, lon=lon)

        stops = self._api_request(path)

        stop_factory = StopFactory(self)

        out = [stop_factory.create(**stop['result']) for stop in stops]

        # only provide certain stop types if we are provided with a mode
        if mode is not None:
            out = [stop for stop in out if stop.transport_type == mode]

        # enforce limit if provided
        if limit is not None:
            out = out[:limit]

        # convert into tuple of (Stop, distance)
        if with_distance:
            out = [(stop, location.distance(stop.location)) for stop in out]

        return out

    def transport_pois_by_map(self, poi, location1,
                              location2, griddepth, limit=20):
        """Return a list of points of interest within a map grid defined by
        location1 & location2

        Arguments:
            poi: either a transport mode or outlet. A list of poi types can be
                passed in as a comma separated
            location1 & location2:
                - are one of (lat, lon), a Location object, or something that
                  has a location property (which would be a Location object).
                - define the top left corner (location1) and bottom right
                  corner (location2) of a rectangle on a map
            griddepth: number of cell blocks per cluster
            limit: minimum number of POIs required to create a cluster as well
                as the maximum number of POIs returned
        Returns:
            a dictionary in the format, the primary components being in the
            'locations' value
            {'locations': [ Stop or Outlet objects ],
             '...'
             }
        """

        lat1, lon1 = parse_location(location1)
        lat2, lon2 = parse_location(location2)

        base_path = "/v2/poi/{poi}/lat1/{lat1}/long1/{lon1}/" + \
                    "lat2/{lat2}/long2/{lon2}/" + \
                    "griddepth/{griddepth}/limit/{limit}"

        poi_ids = ','.join([str(self.MODES[p]) for p in poi.split(',')])

        path = base_path.format(poi=poi_ids, lat1=lat1, lon1=lon1,
                                lat2=lat2, lon2=lon2,
                                griddepth=griddepth, limit=limit)

        data = self._api_request(path)

        stop_factory = StopFactory(self)
        outlet_factory = OutletFactory(self)

        out = {}
        for k, v in data.items():
            if k == "locations":
                out['locations'] = []
                for location in v:
                    # either a Stop of an Outlet
                    if 'transport_type' in location:
                        item = stop_factory.create(**location)
                    else:
                        outlet_type = location.pop('outlet_type')
                        item = outlet_factory.create(transport_type=outlet_type,
                                                     **location)
                    out['locations'].append(item)
            else:
                out[k] = v

        return out

    def search(self, term):
        """All stops and lines that match the search term.

        Arguments:
            term: serch term
        Returns:
            list of Stops & Lines
        """

        path = "/v2/search/%s" % urllib.quote(term)

        data = self._api_request(path)

        stop_factory = StopFactory(self)
        line_factory = LineFactory(self)

        out = []
        for result in data:
            if result['type'] == 'stop':
                out.append(stop_factory.create(**result['result']))
            elif result['type'] == 'line':
                out.append(line_factory.create(**result['result']))
            else:
                out.append(result)

        return out

    def lines_by_mode(self, mode, name=None):
        """ all the lines for a particular transport mode """
        base_path = "/v2/lines/mode/{mode}"

        mode_id = self.MODES[mode]

        path = base_path.format(mode=mode_id)

        if name is not None:
            path += "?name=%s" % name

        data = self._api_request(path)

        line_factory = LineFactory(self)

        out = []
        for line in data:
            out.append(line_factory.create(**line))

        return out

    def stops_on_a_line(self, mode, line):
        """All stops for a particular transport mode on a given line
        Arguments:
            mode: transport mode
            line: the line_id of a particular line
        Returns:
            List of stops.
        """

        base_path = "/v2/mode/{mode}/line/{line}/stops-for-line"

        mode_id = self.MODES[mode]

        path = base_path.format(mode=mode_id, line=line)

        data = self._api_request(path)

        stop_factory = StopFactory(self)

        out = []
        for line in data:
            out.append(stop_factory.create(**line))

        return out

    def _process_departures(self, departures):
        """ common reponse parser for handling a list of departures """

        line_factory = LineFactory(self)
        stop_factory = StopFactory(self)
        run_factory = RunFactory(self)

        out = []
        for departure in departures:
            # - platform
            # -- direction
            # --- line
            platform_details = departure['platform']
            direction_details = platform_details.pop('direction')
            line_details = direction_details.pop('line')
            line = line_factory.create(**line_details)
            direction_details['line'] = line
            direction = Direction(**direction_details)
            platform_details['direction'] = direction
            # --- stop
            stop_details = platform_details.pop('stop')
            stop = stop_factory.create(**stop_details)
            platform_details['stop'] = stop
            platform = Platform(**platform_details)
            # - run
            run_details = departure['run']
            run = run_factory.create(**run_details)

            timetable = parse_datetime_tz(departure["time_timetable_utc"])
            if departure["time_realtime_utc"] is not None:
                realtime = parse_datetime_tz(departure["time_realtime_utc"])
            else:
                realtime = None

            if departure['flags']:
                flags = ', '.join([self.FLAGS[f] for f
                                   in departure['flags'].split('-')
                                   if f != 'E'])
            else:
                flags = None

            out.append({"platform": platform,
                        "run": run,
                        "flags": flags,
                        "time_timetable_utc": timetable,
                        "time_realtime_utc": realtime,
                        })
        return out

    def broad_next_departures(self, mode, stop, limit=5):
        """Departure times at a particular stop, irrespective of line or
        direction.

        Arguments:
            mode: transport mode
            stop: stop_id of a stop
            limit: max results to return
        Returns:
            A list of departures
        """

        base_path = "/v2/mode/{mode}/stop/{stop}/" + \
                    "departures/by-destination/limit/{limit}"
        mode_id = self.MODES[mode]
        path = base_path.format(mode=mode_id, stop=stop, limit=limit)
        departures = self._api_request(path)

        return self._process_departures(departures["values"])

    def specific_next_departures(self, mode, line, stop,
                                 direction, limit=5, for_utc=None):
        """Departure times at a particular stop for a given line and direction

        Arguments:
            mode: transport mode
            line: line_id of transport line
            stop: stop_id of a stop on the line
            direction: direction_id of run's direction
            limit: max results to return
            for_utc: (optional) date and time of the request
        Returns:
            A list of departures
        """

        base_path = "/v2/mode/{mode}/line/{line}/stop/{stop}/" + \
                    "directionid/{direction}/departures/all/limit/{limit}"

        mode_id = self.MODES[mode]

        path = base_path.format(mode=mode_id, line=line, stop=stop,
                                direction=direction, limit=limit)

        if for_utc is not None:
            path += "?for_utc=%s" % for_utc

        departures = self._api_request(path)

        return self._process_departures(departures["values"])

    def specific_next_departures_gtfs(self, mode, route_id, stop, direction,
                                      for_utc=None):
        """ TODO: explain how this differs from previous method """

        base_path = "/v2/mode/{mode}/route_id/{route_id}/stop/{stop}/" + \
                    "direction/{direction}/departures/all/limit/{limit}"

        path = base_path.format(mode=mode, route_id=route_id, stop=stop,
                                direction=direction)

        if for_utc is not None:
            path += "?for_utc=%s" % for_utc

        departures = self._api_request(path)

        return self._process_departures(departures["values"])

    def stopping_pattern(self, mode, run, stop, for_utc=None):
        """Stopping pattern for a particular run from a given stop

        Arguments:
            mode: transport mode
            run: transport run_id
            stop: stop_id of a stop
            for_utc: (optional) date and time of the request
        Returns:
            A list of departures
        """

        base_path = "/v2/mode/{mode}/run/{run}/stop/{stop}/stopping-pattern"
        mode_id = self.MODES[mode]
        path = base_path.format(mode=mode_id, run=run, stop=stop)

        if for_utc is not None:
            path += "?for_utc=%s" % for_utc

        data = self._api_request(path)

        return self._process_departures(data['values'])

    def disruptions(self, modes="general"):
        """Planned and unplanned disruptions on the transport network.

        Arguments:
            modes: one or more of the following in a comma separted string
                   format:
                       general
                       metro-bus
                       metro-train
                       metro-tram
                       regional-bus
                       regional-coach
                       regional-train
        """

        path = "/v2/disruptions/modes/%s" % modes

        data = self._api_request(path)

        factory = DisruptionFactory(self)

        out = []
        for mode, items in data.items():
            for item in items:
                out.append(factory.create(transport_type=mode, **item))
        return out


import threading
import time
import hashtagCrawler
from pymongo import MongoClient
class HashtagThread(threading.Thread):
    def __init__(self,threadID,name,location_list):
        threading.Thread.__init__(self)
        self.threadID = threadID
        self.name = name
        self.location_list = location_list
    def run(self):
        hashtagCrawler.getLocationsHashtags(self.location_list)
        print('Thread-'+self.name+' Finished!!')
def main():
    thread_pool = []
    client = MongoClient('localhost', 27017)
    db = client.locationhunter
    location_table = db.city_locations
    hashtag_helper = db.hashtag_helper
    count = 0
    cache_locations = []
    for city in location_table.find({}):
        locations = city['locations']
        locations = filterLocations(locations,hashtag_helper)
        cache_locations.extend(locations)
        if len(cache_locations) >500:
            thread = HashtagThread(count, "Thread"+city['city_id'],cache_locations)
            thread_pool.append(thread)
            cache_locations = list()
            count += 1
    thread = HashtagThread(count, "Thread" + city['city_id'], cache_locations)
    thread_pool.append(thread)
    count+=1
    print(str(count) + " Threads in total")
    for thread in thread_pool:
        thread.start()
    for thread in thread_pool:
        thread.join()
    print('All Job Finished')

def filterLocations(locations,helper):
    retLocation = []
    for location in locations:
        r = helper.find({'location':location['slug']})
        if r.count() >0:
            if not r[0]['mined']:
                retLocation.append(location)
        else:
            retLocation.append(location)
    return retLocation

if __name__ == '__main__':
    main()


import os
import sys
from colorama import init
from termcolor import colored
import logging

init()

##The enviroments needs to be loaded first !!!
from pathlib import Path
from dotenv import load_dotenv
env_name = os.getenv('FLASK_ENV', 'development')
if env_name == 'production':
    env_path = Path('.') / '.env.production'
elif env_name == 'development':
    env_path = Path('.') / '.env.development'
else:
    sys.exit(colored('FLASK_ENV accepted values are: production / development', 'red'))

load_dotenv(dotenv_path=env_path, verbose=True)

from flask import Flask, send_from_directory
from flask_cors import CORS
# from flask_restful import Api
from flask_jwt_extended import JWTManager

from app.config import app_config, setup_logger
from flask_mail import Mail
from flask_migrate import Migrate

app = Flask(__name__, static_folder='../frontend/build', static_url_path='/')
# app = Flask(__name__)

CORS(app) ## enable CORS on all routes
app.config.from_object(app_config[env_name])

# init extensions
mail = Mail()
migrate = Migrate()
jwt = JWTManager(app)

from app.models import db, ma
db.init_app(app)
ma.init_app(app)
mail.init_app(app)
migrate.init_app(app, db)

# init logging
setup_logger(app)

# init routes not used
# from app.resources.routes import create_routes
# api = Api(app=app)
# create_routes(api=api)

@app.route("/", defaults={'path':''})
@app.route("/app/<path:path>")
def serve(path):
    return send_from_directory(app.static_folder, 'index.html')

from app.controlller import register_controller
register_controller(app)

from app.commands import register_commands
register_commands(app)

# register handlers
from app.handler import (jwt_handler, http_handler)
jwt_handler(jwt)
http_handler(app)

app.logger.info(f"\n \
######################################################### \n \
#   ENV:        {env_name}                                \n \
#   DB_HOST:    {app.config['DB_HOST']}                   \n \
#   ENV:        {env_name}                                \n \
######################################################### ")

@app.route('/test')
def test():
    resp_test = {
            "name": "alex",
            "age": "12"
        }
    return resp_test, 200
    # return 'This is a test !'

if env_name == 'production':
    gunicorn_logger = logging.getLogger('gunicorn.error')
    app.logger.handlers = gunicorn_logger.handlers
    app.logger.setLevel(gunicorn_logger.level)


from django.urls import path
from . import views

app_name = 'todolist'
urlpatterns = [
    path('', views.IndexView.as_view(), name='show_todolist'),
    path('json/', views.show_json, name='show_json'),
    path('register/', views.register, name='register'),
    path('login/', views.login_user, name='login'),
    path('logout/', views.logout_user, name='logout'),
    path('add/', views.create_task, name='add_task'),
    path('<int:task_id>/delete-task/', views.delete_task, name='delete_task'),
    path('<int:task_id>/finish-task/', views.finish_task, name='finish_task'),
]


from django.utils.translation import ugettext_lazy as _

from fedoralink.common_namespaces.dc import DCObject
from fedoralink.fedorans import ACL
from fedoralink.indexer.fields import IndexedTextField, IndexedURIRefField

class AclCollection(DCObject):

    class Meta:
        rdf_types = (ACL.AclCollection, )

class Acl(DCObject):

    class Meta:
        rdf_types = (ACL.Acl, )

class Authorization(DCObject):

    agent        = IndexedURIRefField(ACL.agent, verbose_name=_('People allowed to access a resource'),
                                    multi_valued=True)

    agent_class  = IndexedURIRefField(ACL.agentClass, verbose_name=_('Groups of people allowed to access a resource'),
                                    multi_valued=True)

    mode         = IndexedURIRefField(ACL.mode, verbose_name=_('Resource access mode, either acl:Read or acl:Write'),
                                    multi_valued=True)

    access_to    = IndexedURIRefField(ACL.accessTo, verbose_name=_('Resource to which this object applies'),
                                    multi_valued=True)

    access_to_class = IndexedURIRefField(ACL.accessToClass, verbose_name=_('RDF class of resources to which this authorization applies'),
                                    multi_valued=True)

    class Meta:
        rdf_types = (ACL.Authorization, )


from Ui import UserEdit
from PyQt5 import QtCore, QtGui, QtWidgets

class Ui_MainWindow(object):
    def setupUi(self, MainWindow):
        MainWindow.setObjectName("MainWindow")
        MainWindow.resize(798, 600)
        self.centralwidget = QtWidgets.QWidget(MainWindow)
        self.centralwidget.setObjectName("centralwidget")
        self.verticalLayoutWidget = QtWidgets.QWidget(self.centralwidget)
        self.verticalLayoutWidget.setGeometry(QtCore.QRect(0, 10, 791, 541))
        self.verticalLayoutWidget.setObjectName("verticalLayoutWidget")
        self.verticalLayout = QtWidgets.QVBoxLayout(self.verticalLayoutWidget)
        self.verticalLayout.setContentsMargins(0, 0, 0, 0)
        self.verticalLayout.setObjectName("verticalLayout")
        self.horizontalLayout = QtWidgets.QHBoxLayout()
        self.horizontalLayout.setObjectName("horizontalLayout")
        self.dtBegin = QtWidgets.QDateEdit(self.verticalLayoutWidget)
        self.dtBegin.setObjectName("dtBegin")
        self.horizontalLayout.addWidget(self.dtBegin)
        self.dtEnd = QtWidgets.QDateEdit(self.verticalLayoutWidget)
        self.dtEnd.setObjectName("dtEnd")
        self.horizontalLayout.addWidget(self.dtEnd)
        self.verticalLayout.addLayout(self.horizontalLayout)
        self.twTinting = QtWidgets.QTableView(self.verticalLayoutWidget)
        self.twTinting.setObjectName("twTinting")
        self.verticalLayout.addWidget(self.twTinting)
        MainWindow.setCentralWidget(self.centralwidget)
        self.statusbar = QtWidgets.QStatusBar(MainWindow)
        self.statusbar.setObjectName("statusbar")
        MainWindow.setStatusBar(self.statusbar)
        self.toolBar = QtWidgets.QToolBar(MainWindow)
        self.toolBar.setObjectName("toolBar")
        MainWindow.addToolBar(QtCore.Qt.TopToolBarArea, self.toolBar)
        self.actionInsertRecord = QtWidgets.QAction(MainWindow)
        icon = QtGui.QIcon()
        icon.addPixmap(QtGui.QPixmap(":/newPrefix/Button-Add-icon.png"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.actionInsertRecord.setIcon(icon)
        self.actionInsertRecord.setObjectName("actionInsertRecord")
        self.actionDeleteRecord = QtWidgets.QAction(MainWindow)
        icon1 = QtGui.QIcon()
        icon1.addPixmap(QtGui.QPixmap(":/newPrefix/Button-Close-icon.png"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.actionDeleteRecord.setIcon(icon1)
        self.actionDeleteRecord.setObjectName("actionDeleteRecord")
        self.actionImportProduct = QtWidgets.QAction(MainWindow)
        icon2 = QtGui.QIcon()
        icon2.addPixmap(QtGui.QPixmap(":/newPrefix/file-import-icon.png"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.actionImportProduct.setIcon(icon2)
        self.actionImportProduct.setObjectName("actionImportProduct")
        self.actionExport = QtWidgets.QAction(MainWindow)
        icon3 = QtGui.QIcon()
        icon3.addPixmap(QtGui.QPixmap(":/newPrefix/USB-Flash-Card-With-Card-Reader-icon.png"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.actionExport.setIcon(icon3)
        self.actionExport.setObjectName("actionExport")
        self.actionUserEdit = QtWidgets.QAction(MainWindow)
        icon4 = QtGui.QIcon()
        icon4.addPixmap(QtGui.QPixmap(":/newPrefix/Person-Male-Light-icon.png"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.actionUserEdit.setIcon(icon4)
        self.actionUserEdit.setObjectName("actionUserEdit")
        self.actionUserChange = QtWidgets.QAction(MainWindow)
        icon5 = QtGui.QIcon()
        icon5.addPixmap(QtGui.QPixmap(":/newPrefix/User-icon.png"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.actionUserChange.setIcon(icon5)
        self.actionUserChange.setObjectName("actionUserChange")
        self.toolBar.addAction(self.actionInsertRecord)
        self.toolBar.addAction(self.actionDeleteRecord)
        self.toolBar.addAction(self.actionImportProduct)
        self.toolBar.addAction(self.actionExport)
        self.toolBar.addAction(self.actionUserEdit)
        self.toolBar.addAction(self.actionUserChange)

        self.retranslateUi(MainWindow)
        QtCore.QMetaObject.connectSlotsByName(MainWindow)

    def retranslateUi(self, MainWindow):
        _translate = QtCore.QCoreApplication.translate
        MainWindow.setWindowTitle(_translate("MainWindow", "MainWindow"))
        self.toolBar.setWindowTitle(_translate("MainWindow", "toolBar"))
        self.actionInsertRecord.setText(_translate("MainWindow", "Новая запись"))
        self.actionDeleteRecord.setText(_translate("MainWindow", "Удалить"))
        self.actionImportProduct.setText(_translate("MainWindow", "Импорт Краски"))
        self.actionExport.setText(_translate("MainWindow", "Экспорт Журнала"))
        self.actionUserEdit.setText(_translate("MainWindow", "Редактировать пользователей"))
        self.actionUserChange.setText(_translate("MainWindow", "Сменить пользователя"))
        self.actionUserChange.setToolTip(_translate("MainWindow", "Сменить пользователя"))
import Ui.resourse_rc

if __name__ == "__main__":
    import sys
    app = QtWidgets.QApplication(sys.argv)
    MainWindow = QtWidgets.QMainWindow()
    ui = Ui_MainWindow()
    ui.setupUi(MainWindow)
    MainWindow.show()
    sys.exit(app.exec_())


#!C:\Users\ivans_000\AppData\Local\Programs\Python\Python35-32\python

import cgi
import HostScannerDAL as HSD
import threading
import GUI_Display as GD

print("Content-type:text/html\r\n\r\n")
print("")

class Switch_Result ():
    def __init__(self, hst_id, hst_name):
        self.PPSD=HSD.HostScannerDAL()
        self.ip_add=hst_id
        self.h_nam=hst_name
        self.Switch_Results()

    def Switch_Results(self):
        #obtaining the data using the querry in the readportstatus method
        dat=self.PPSD.read_port_status(self.ip_add, self.h_nam)
        #printing the information as a table
        for d in dat:
            print('
'+str(d['scanId'])+'	'+
                   ''+str(d['PortNumber'])+'	'+
                  '' + str(d['IsPortOpen']) + '	' +
                  '' + str(d['ScanStartTime']) + '
')

        #thr2 = threading.Thread(target=self.CGUI())
        #thr2.start()
        #thr2.join()
    #CREATING THE GUI TABLE
    #def CGUI(self):
        #self.GDD = GD.ResultsDialog(self.ip_add, self.h_nam)

form = cgi.FieldStorage()
name=form.getvalue("host_name")
ip=form.getvalue("host_ip")
#name='Usuario.ad.bridgeport.edu'
#ip='10.12.55.82'
#PPSD=HSD.HostScannerDAL()
#dat=PPSD.read_port_status('10.12.55.82','Usuario.ad.bridgeport.edu')

#checking if we are receiving name and ip address
if ip and name is not None:
    print("
 RESULTS PAGE
")
    print("
 ")
    print(" SCAN_ID	")
    print(" PORT_NUMBER	")
    print(" IS_OPEN	")
    print(" SCAN_TIME	")
    Switch_Result(ip,name)
    print("
")
    print("")

#not received proper information
else:
    print("
 SCANNING PAGE </h>")
    print("
 Received host = Host ID not received
")
    print("")


import yaml
import numpy as np
from scipy.io import loadmat
from util.dataset_util import get_start_end_indeces
from util.dataset_util import get_grond_truth
from util.dataset_util import get_list_of_observations

class Dataset:
	def __init__(self, path='Sampled_dataset/'):
		self.Robot  = 1
		self.Dataset_num = 1
		self.start_time = 0
		self.durateion = 10

		self.R_gt = 0
		self.R_obs = 0
		self.R_odom = 0
		self.landmark_ids = 0
		self.landmarks_list = 0
		self.path = path
		self.sample_time = 0
		self.time_steps = 0

	def load_data(self, robot_num, dataset_num):
		'''
		Load the data from the .mat files smapled by the dataset matlab tools
		'''

		self.Robot = robot_num
		self.Dataset_num = dataset_num
		with open('config.yaml') as config_file:
			config = yaml.load(config_file, Loader=yaml.FullLoader)
			sample_time = config['dt'] # 0.02

		print("Loading datset#{0} for Robot#{1}".format(self.Dataset_num, self.Robot))
		#Get the Sampling time and number of time steps captured and sampled in the dataset
		self.sample_time = loadmat(self.path + 'Dataset{}'.format(self.Dataset_num))['sample_time']
		self.time_steps = loadmat(self.path + 'Dataset{}'.format(self.Dataset_num))['timesteps']

		#Data collected from the robot
		self.R_gt = loadmat(self.path + 'Dataset{}'.format(self.Dataset_num))['Robot{}_Groundtruth'.format(self.Robot)]
		self.R_obs = loadmat(self.path + 'Dataset{}'.format(self.Dataset_num))['Robot{}_Measurement'.format(self.Robot)]
		self.R_odom = loadmat(self.path + 'Dataset{}'.format(self.Dataset_num))['Robot{}_Odometry'.format(self.Robot)]

		#Landmarks data
		self.landmarks_list = loadmat(self.path + 'Dataset{}'.format(self.Dataset_num))['Landmark_Groundtruth'][:,0:3]
		barcodes = loadmat(self.path + 'Dataset{}'.format(self.Dataset_num))['Barcodes']

		self.landmarks_list[:,0] = barcodes[5:,1] #List of the barcode numbers
		self.landmark_ids = barcodes[5:,1]   #i.e. [72,27,54,70,36,18,25,9,81,16,90,61,45,7,63]
		self.landmarks_list = dict(zip(self.landmarks_list[:,0].astype(int),self.landmarks_list[:,1:3])) #Turn the landmark data to dictionary {barcode: [landmark_x, landmark_y]}

	def get_data(self, start_time=0, duration = 400):
		'''
		returns the portion of the dataset with the specified start and duration times in seconds

		'''
		self.start_time = start_time
		if (duration == 'Full' and start_time == 0):
			print("Full duration seleted")
			self.duration = (self.sample_time * self.time_steps)[0,0] - 1
			print("Duration is {}".format(self.duration))

		else:
			self.duration = duration
			print("Duration is {}".format(self.duration))

		#Round the time
		precision = 3  #based on the time step's resolution
		self.R_gt[:,0] = np.round(self.R_gt[:,0],precision)
		self.R_odom[:,0] = np.round(self.R_odom[:,0],precision)
		self.R_obs[:,0] = np.round(self.R_obs[:,0],precision)

		#Get the time stamps and find the start and ending indeces
		end_time = self.start_time + self.duration
		start_idx, end_idx = get_start_end_indeces(self.R_gt, self.start_time, end_time)
		observations_input = get_list_of_observations(self.R_obs, self.start_time, end_time)

		odometry_input = self.R_odom[start_idx:end_idx]
		observation_input = self.R_obs[start_idx:end_idx]
		ground_truth = self.R_gt[start_idx:end_idx]

		number_of_observations = len(observations_input)
		total_sample_number = len(self.R_gt[start_idx:end_idx,0])
		data_size = total_sample_number + number_of_observations

		inital_state = ground_truth[0,1:4]
		inital_time = ground_truth[0,0]

		return inital_state, self.landmark_ids, self.landmarks_list, data_size, ground_truth, odometry_input, observations_input

	def simulation_data():
		pass


#!/usr/bin/env python
# coding: utf-8

# In[4]:

import config
import win32com.client as win32
# # from campaign import campaign
# # h_body = campaign()

def outlook_email(to=None, attachments=None):
#     """Takes attachment parameter which is equal to path to attachment\n\n\
#     Example:\n\natt=r'C:\D-Drive\PM Routine\pm routine\Tasks assigned.txt\n\n\
#     outlook_email(attachment)"""
    to=to
    outlook = win32.Dispatch('outlook.application')
    mail = outlook.CreateItem(0)
    mail.To = config.TEST if not to else to
#     mail.Cc = config.CC
    mail.Subject = 'Risk Report'
    mail.HTMLBody = '
Hello,
Please find attached risk report.
    Kindly report any errors or missing data in this file.

Thanks
Remediation Team
'

    # To attach a file to the email (optional):
#     if attachment:
#         mail.Attachments.Add(attachment)

    if attachments:     #Testing loop through attachments
        [mail.Attachments.Add(attachment) for attachment in attachments]

    mail.Send()
    print('\nEmail sent!')
    return

# path = r"C:\Users\foo.bar\Desktop\Risk score\Severity\severity_v1\\"
# file = 'W0575NtEpsFwsD0_MobDmz 533618.csv'
# attachment=f'{path}{file}'
# outlook_email(attachment)


import logging
from concurrent import futures

from google.protobuf.timestamp_pb2 import Timestamp
import grpc
import SpotGlove_pb2
import SpotGlove_pb2_grpc
import base_config
import move_robot
import data_aquisition

robot_object = None
move_robot_object = None
spotStands = False

class SpotGloveServicer(SpotGlove_pb2_grpc.CommandSpotServicer):

    def Command(self, request, context):
        errorCode = 0
        result = True
        global spotStands
        if request.name == "stand" and spotStands is False:
            print("Spot needs to stand")
            move_robot_object.stand()
            spotStands = True
        elif request.name == "sit" and spotStands is True:
            print("Spot needs to sit")
            move_robot_object.sit()
            spotStands = False
        elif request.name == "shutdown":
            if robot_object.is_powered_on():
                move_robot_object.toggle_power()
            move_robot_object.shutdown()
            spotStands = False
        elif request.name == "estop_set":
            if move_robot_object.estop_keepalive:
                move_robot_object.toggle_estop()
                spotStands = False
            else:
                result = False
        elif request.name == "estop_remove":
            if not move_robot_object.lease:
                move_robot_object.start()
            if not move_robot_object.estop_keepalive:
                move_robot_object.toggle_estop()
            else:
                result = False
            if not robot_object.is_powered_on():
                move_robot_object.toggle_power()
        elif request.name == "move_forward" and spotStands is True:
            move_robot_object.move_forward()
        elif request.name == "move_backwards":
            move_robot_object.move_backwards()
        elif request.name == "move_left" and spotStands is True:
            move_robot_object.move_left()
        elif request.name == "move_right" and spotStands is True:
            move_robot_object.move_right()
        elif request.name == "turn_left" and spotStands is True:
            move_robot_object.turn_left()
        elif request.name == "turn_right" and spotStands is True:
            move_robot_object.turn_right()
        else:
            print("Unknown Command or Command can not be executed")
            result = False
            errorCode = 1
        print("Command received")
        timestamp = Timestamp()
        timestamp.GetCurrentTime()
        print(data_aquisition.get_state(robot_object).battery_states)
        return SpotGlove_pb2.CommandReply(timestamp=timestamp, result=result, errorCode=errorCode)

def serve():
    print("starting Server")
    server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
    SpotGlove_pb2_grpc.add_CommandSpotServicer_to_server(SpotGloveServicer(), server)
    server.add_insecure_port('[::]:50051')
    server.start()
    print("Server started")
    server.wait_for_termination()

if __name__ == '__main__':
    logging.basicConfig()
    robot_object = base_config.connect_and_authenticate("admin", "m2v97q65dllv", "192.168.80.3")
    move_robot_object = move_robot.MoveRobot(robot_object)

    # Start Robot
    move_robot_object.start()
    move_robot_object.toggle_estop()
    move_robot_object.toggle_power()
    print(data_aquisition.get_state(robot_object).battery_states)
    serve()


"""
This script crops or copys images from a folder. The edited image can be viewed during
the run-time and saved in another folder with the same image name using
Python Imaging Library (PIL) - Pillow.

Syntax to crop an image:

    Image.crop(box=None)
    box = The crop rectangle, as a (left, upper, right, lower)-tuple.

    Returns a rectangular region from this image. The box is a 4-tuple defining
    the left, upper, right, and lower pixel coordinate.

Syntax to copy an image:

    In the below code, just comment the line "image = image.crop((50, 140, 1000, 400))  # (left, upper, right, lower)"

Reference:
    https://pillow.readthedocs.io/en/3.1.x/reference/Image.html#PIL.Image.Image.crop

Dimension of the image used here is 1024 * 500
"""

import os
import glob

from PIL import Image

source = r"C:/Users/Desktop/source"

destination = r"C:/Users/Desktop/destination"

def main():

    for files in glob.glob(source + "/*.png"):

        filename, file_extension = os.path.splitext(files)
        image_name = os.path.join(destination, os.path.basename(filename) + '.png')

        image = Image.open(files)
        image = image.crop((50, 140, 1000, 400))  # (left, upper, right, lower)

        image.save(image_name)
        image.show()

if __name__ == '__main__':
    main()


import curses
import run, fonts
import nv0, submenu

screen = curses.initscr()

curses.noecho()
curses.cbreak()
curses.start_color()
curses.curs_set(0)
screen.keypad(1)

#------------------------------------Base de Datos------------------------------------
#-----------------Configuración-----------------
animation = True #. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . Animación activada SI/NO
flicker = True #. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . Parpadeo activado SI/NO

color_bold = ('white') #. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . Color de fondo.
color_lyrics = ('black') #. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . Color de letra.

cursor = 1
#-----------------------------------------------<<<

##-------------------Opcional-------------------

CURSOR = { #. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . Tipo decursores.
	1: '|> ',
	2: ' > ',
	3: '-> ',
	4: '- ',
	5: ''
}

COLOR_LYRICS = { #. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . Paleta para Letra.
	'white': curses.COLOR_WHITE,
	'blue': curses.COLOR_BLUE,
	'red': curses.COLOR_RED,
	'black': curses.COLOR_BLACK,
	'yellow': curses.COLOR_YELLOW,
	'cyan': curses.COLOR_CYAN,
	'green': curses.COLOR_GREEN,
}

COLOR_BOLD = { #. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . Paleta para fondo.
	'white': curses.COLOR_WHITE,
	'blue': curses.COLOR_BLUE,
	'red': curses.COLOR_RED,
	'black': curses.COLOR_BLACK,
	'yellow': curses.COLOR_YELLOW,
	'cyan': curses.COLOR_CYAN,
	'green': curses.COLOR_GREEN,
}

#-----------------------------------------------<<<

#----------------Animacion SI/NO----------------
if animation == True: #. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . Animación por defecto 12ms
	animation = 12

elif animation == False:
	animation = 0
#-----------------------------------------------<<<

#----------------Parpadeo SI/NO-----------------
if flicker == True:
	flicker = curses.A_BLINK

elif flicker == False:
	flicker = curses.A_STANDOUT
#----------------------------------------------<<<

#-------------------Colores--------------------
color_lyrics = COLOR_LYRICS[color_lyrics] #. . . . . . . . . . . . . . . . . . . . . . . . . . . . . Almacena los colores.
color_bold = COLOR_BOLD[color_bold]
cursor = CURSOR[cursor]

curses.init_pair(1, color_bold, color_lyrics)
#----------------------------------------------<<<
#-------------------------------------------------------------------------------------<<<

TEXT = {
	1: str('- Presione ESPACIO -'),
	2: str('~ A P R E N D I E N D O   P Y T H O N 3 ~'),
	3: str('Presione ENTER para confirmar...')
}

OPTION = {
	4: 'start',
	3: 'options',
	2: 'about',
	1: 'exit'
}

x = curses.LINES // 2
y = curses.COLS // 2

def process(num):
	screen.border()
	screen.refresh()

	rows = 5
	cols = 36

	y = (curses.LINES - rows) // 2
	x = (curses.COLS - cols) // 2

	win = curses.newwin(rows, cols, y, x)

	win.box()

	win.move(2, 11)
	win.addstr('Proximamente...', curses.A_BOLD)
	escape = False
	option = OPTION[num]

	while escape == False:
		curses.echo()
		key = win.getch(2, 2)
		win.move(2, 2)
		win.addstr('Proximamente...', curses.A_BOLD)

		if key == 10:
			opt(num, option)
			escape = True

		elif key == 27:
			opt(num, option)
			escape = True

def opt(num, opt):
	screen.border()
	escape = False
	fonts.tittle(screen, TEXT[2])

	up_down(num)

	option = str(opt)

	while escape == False:
		key = screen.getch(2, 1)

		if key == 258:
			num -= 1

			if num < 1: num = 1

			option = OPTION[num]

			screen.border()
			up_down(num)
			escape = False

		elif key == 259:
			num += 1

			if num > 4: num = 4

			option = OPTION[num]

			screen.border()
			up_down(num)
			escape = False

		elif key in [curses.KEY_ENTER, ord('\n'), 10]:
			escape = True
			curses.endwin()

			if option == ('exit'):
				main_exit(num)

			elif option == ('options'):
				process(3)

			elif option == ('about'):
				submenu.main() #------------------------------------------------------EDIT----------------------

			elif option == ('start'):
				process(4)

		elif key == curses.KEY_RESIZE:
			screen.erase()
			screen.refresh()
			escape = True
			curses.endwin()
			fonts.error()

def group():
	screen.addstr(x - 3, 3, 'Iniciar sesion      '),
	screen.addstr(x - 1, 3, 'Crear cuenta        '),
	screen.addstr(x + 1, 3, 'Saguir Demo      '),
	screen.addstr(x + 3, 3, 'Atras      ')

def up_down(num):
	fonts.tittle(screen, TEXT[2])

	if num == 4:
		screen.border()
		line = x - 3
		group()

		screen.addstr(line, 3,  '               ')
		fonts.animation_off(screen, '|> Iniciar sesion ', line, animation, flicker)

	elif num == 3:
		screen.border()
		line = x - 1
		group()

		screen.addstr(line, 3,  '             ')
		fonts.animation_off(screen, '|> Crear cuenta ', line, animation, flicker)

	elif num == 2:
		screen.border()
		line = x + 1
		group()

		screen.addstr(line, 3,  '              ')
		fonts.animation(screen, '|> Seguir Demo ', line, animation, flicker)

	elif num == 1:
		screen.border()
		line = x + 3
		group()

		screen.addstr(line, 3,  '      ')
		fonts.animation(screen, '|> Atras ', line, animation, flicker)

def main_space():
	fonts.flicker_center(screen, TEXT[1], x)

def main(num):
	fonts.tittle(screen, TEXT[2])
	screen.erase()
	screen.refresh()
	escape = True
	opt(num, 'start')

def main_exit(num):
	screen.erase()
	screen.refresh()
	curses.endwin()
	run.opt(4, 'options')


"""Video Writer Helper"""
import datetime
from multiprocessing.sharedctypes import Value
import cv2
import torch
import numpy as np
from os import path, makedirs
from typing import Optional, Tuple, Union
from torch.utils.tensorboard import SummaryWriter

from totri.util.format import tensor_to_chw

class VideoWriter():
    """Wrapper around cv2.VideoWriter

    After writing video files, the OpenCV handles must be released.
    This will be done automatically upon destruction, alternatively
    use VideoSummaryWriter as context manager.
    """

    def __init__(self, file_name: str, resolution: Tuple[int],
                 fps: Optional[int] = None,
                 fourcc: Optional[str] = None):
        """Init VideoWriter

        Args:
            file_name (str): Filename including appropriate extension
            resolution (Tuple[int]): (height, width) tuple
            fps (int, optional): Frames per second. Defaults to 25.
            fourcc (Optional[str], optional): `cv2.VideoWriter_fourcc` code, e.g. `'mp4v'` (default)
        """
        self.file_name = path.abspath(file_name)
        self.resolution = resolution
        self.fps = fps or 25
        self.fourcc = fourcc or cv2.VideoWriter_fourcc(*'mp4v')
        self.dirname = path.dirname(self.file_name)
        if not path.isdir(self.dirname):
            makedirs(self.dirname)
        self._writer = cv2.VideoWriter(
            self.file_name, self.fourcc, self.fps,
            (self.resolution[1], self.resolution[0]))
        self.frame_count = 0

    def __del__(self):
        self.release()

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        self.release()

    def release(self):
        if self._writer is not None:
            self._writer.release()
            self._writer = None

    def write_frame(self, img_tensor: Union[torch.Tensor, np.ndarray],
                    dataformats: str = 'CHW'):
        if self._writer is None:
            raise RuntimeError('Video writer has been closed. Cannot write new frames.')
        with torch.no_grad():
            # Transform to CHW format
            img_tensor = tensor_to_chw(img_tensor, dataformats)
            # Convert to uint8
            if img_tensor.dtype != torch.uint8:
                img_tensor = (img_tensor * 255).to(dtype=torch.uint8)
            # CHW to HWC
            img_tensor = img_tensor.permute(1, 2, 0)
            # Check dimensions
            if (img_tensor.shape[0] != self.resolution[0] or
                img_tensor.shape[1] != self.resolution[1]):
                raise ValueError(
                    f"Tensor has wrong dimensions."
                    f"Expected {self.resolution[0]}x{self.resolution[1]}, "
                    f"but got {img_tensor.shape[0]}x{img_tensor.shape[1]}.")
            # RGB TO BGR
            img_tensor_bgr = img_tensor[..., (2, 1, 0)]
            self._writer.write(img_tensor_bgr.contiguous().cpu().numpy())
            self.frame_count += 1

class RealTimeVideoWriter(VideoWriter):

    def __init__(self, *args, speed=1.0, **kwargs):
            super().__init__(*args, **kwargs)
            self.last_img_tensor = None
            self.last_time = None
            self.speed = float(speed)

    def write_frame(self, data: Union[torch.Tensor, np.ndarray],
                    dataformats: str = 'CHW', transform = lambda x: x):
        with torch.no_grad():
            frame_length = datetime.timedelta(seconds=self.speed/self.fps)
            now = datetime.datetime.now()
            if self.last_time is None:
                img_tensor = tensor_to_chw(transform(data), dataformats)
                super().write_frame(img_tensor, 'CHW')
                self.last_img_tensor = img_tensor
                self.last_time = now + frame_length
            elif self.last_time <= now:
                while self.last_time + frame_length < now:
                    super().write_frame(self.last_img_tensor, 'CHW')
                    self.last_time += frame_length
                img_tensor = tensor_to_chw(transform(data), dataformats)
                super().write_frame(img_tensor, 'CHW')
                self.last_img_tensor = img_tensor
                self.last_time += frame_length

class VideoSummaryWriter(SummaryWriter):
    """Extend SummaryWriter to write images as video

    After writing video files, the OpenCV handles must be released.
    This will be done automatically upon destruction, alternatively
    use VideoSummaryWriter as context manager.

    Video files will be put into the corresponding log_dir according
    to the given tag. To write video frames pass `video=True` to
    add_image or add_images.
    """

    def __init__(self, *args, extension: Optional[str] = None,
                 fourcc: Optional[str] = None, fps: Optional[int] = None,
                 **kwargs):
        super().__init__(*args, **kwargs)
        self._writers = {}
        self.extension = extension
        self.fourcc = fourcc
        self.fps = fps

    def __del__(self):
        self.release()

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        self.release()

    def release(self):
        """Release all writers"""
        for writer in self._writers.values():
            writer.release()

    def _add_video_frame(self, tag: str, img_tensor_chw: torch.Tensor):
        """Add img_tensor to the video corresponding to tag"""
        if tag not in self._writers:
            file_name = path.join(self.log_dir, tag + self.extension)
            resolution = tuple(img_tensor_chw.shape[1:])
            dirname = path.dirname(file_name)
            if not path.isdir(dirname):
                makedirs(dirname)
            self._writers[tag] = VideoWriter(
                file_name, resolution, self.fps, self.fourcc)
        self._writers[tag].write(img_tensor_chw, 'CHW')

    def add_image(self, tag, img_tensor, global_step=None, walltime=None,
                  dataformats='CHW', video=False):
        img_tensor = tensor_to_chw(img_tensor, dataformats)
        super().add_image(tag, img_tensor, global_step, walltime, 'CHW')
        if video:
            self._add_video_frame(tag, img_tensor)

    def add_images(self, tag, img_tensor, global_step=None, walltime=None,
                   dataformats='NCHW', video=False):
        img_tensor = tensor_to_chw(img_tensor, dataformats)
        super().add_image(tag, img_tensor, global_step, walltime, 'CHW')
        if video:
            self._add_video_frame(tag, img_tensor)


import os
import io
from PIL import Image

from google.cloud import automl
from google.cloud import storage

def extractImgMadeInAustralia(event, context):
    """Triggered by a change to a Cloud Storage bucket.
    Args:
         event (dict): Event payload.
         context (google.cloud.functions.Context): Metadata for the event.
    """
    file = event
    print(f"Processing file: {file['name']}.")

    prediction_client = automl.PredictionServiceClient()

    # Get the full path of the model.
    model_full_id = automl.AutoMlClient.model_path(os.environ['PROJECT_ID'], "us-central1", os.environ['MODEL_ID'])

    client = storage.Client()
    source_bucket = client.get_bucket(file['bucket'])
    source_blob = source_bucket.get_blob(file['name'])
    #content = np.asarray(bytearray(source_blob.download_as_string()), dtype="uint8")
    #content= cv2.imdecode(content, cv2.IMREAD_UNCHANGED)
    image_bytes = source_blob.download_as_bytes()

    img = Image.open(io.BytesIO(image_bytes))

    image = automl.Image(image_bytes=image_bytes)
    payload = automl.ExamplePayload(image=image)

    # params is additional domain-specific parameters.
    # score_threshold is used to filter the result
    # https://cloud.google.com/automl/docs/reference/rpc/google.cloud.automl.v1#predictrequest
    params = {"score_threshold": "0.5"}

    request = automl.PredictRequest(name=model_full_id, payload=payload, params=params)

    response = prediction_client.predict(request=request)

    x1 = 0
    y1 = 0
    x2 = 0
    y2 = 0

    for result in response.payload:
        # print("Predicted class name: {}".format(result.display_name))
        # print("Predicted class score: {}".format(result.image_object_detection.score))
        bounding_box = result.image_object_detection.bounding_box
        # print("Normalized Vertices:")
        i = 0
        for vertex in bounding_box.normalized_vertices:
            # print("\tX: {}, Y: {}".format(vertex.x, vertex.y))
            if i == 0:
                x1 = vertex.x
                y1 = vertex.y
            elif i == 1:
                x2 = vertex.x
                y2 = vertex.y

            i += 1

    w, h = img.size
    x1 = int(x1 * w)
    y1 = int(y1 * h)

    x2 = int(x2 * w)
    y2 = int(y2 * h)

    img_crop = img.crop((x1, y1, x2, y2))

    img_byte_array = io.BytesIO()
    img_crop.save(img_byte_array, format="GIF")
    img_byte_array.seek(0)

    destination_bucket = client.get_bucket(os.environ['BUCKET_EXTRACT'])
    destination_blob = destination_bucket.blob(file['name'])
    destination_blob.upload_from_string(img_byte_array.getvalue(), content_type='image/gif')

    print("Image extracted.")


from sqlalchemy import create_engine
import os
import sys

if os.path.exists("../parksareawesome.db"):
	engine = create_engine("sqlite:///../parksareawesome.db")

	result = engine.execute("select * from parks")
	#result = engine.execute("select * from "
	# "parks where park_name=:name", name='Yellowstone')

	for row in result:
		#print(row)
		db_id = row[0]
		park_id = row[1]
		park_name = row[2]
		park_state = row[3]
		park_description = row[4]
		park_weather = row[5]
		park_lat = row[6]
		park_long = row[7]
		if len(sys.argv) > 1:
			if sys.argv[1] == "-v":
				print("(" + str(db_id) + ", " + park_id + ", " + park_name + ", " + park_state +
					", " + park_description + ", " + park_weather + ", lat: " + str(park_lat) + ", long:" + str(park_long) +  ")")
		else:
			print("(" + str(db_id) + ", " + park_id + ", " + park_name + ", " + park_state +
				", lat: " + str(park_lat) + ", long:" + str(park_long) +  ")")
else:
	sys.exit("Parks Table Does Not Exist")


# -*- coding: utf-8 -*-

from Libs.TS_Request import ts_request
from Libs.TS_Download import ts_downloader
from Libs.TS_Re import ts_re
import json

class base_spyder():
    def __init__(self,the_url,limit_size = -1):
        self.start_url = the_url
        # print(self.start_url)
        self.ex_re = ts_re()

        self.ex_request = ts_request(url = self.start_url,
                                     demand_info = self.ex_re.make_small_dict_demand())
        self.ex_downloader = ts_downloader()

        self.image_url_list = []
        self.limit_size = limit_size

    def run(self):

        self.ex_request.run()
        self.image_url_list = self.ex_request.get_url_list()

        print(len(self.image_url_list))

        self.write_json(the_url_list = self.image_url_list)
        """
        self.ex_downloader.start_download(url_list=self.image_url_list,
                                         lim_size = self.limit_size)

        self.ex_downloader.rename()
        """
    def write_json(self, the_url_list):
        with open("URL_LIST.json",'w') as f:
            f.write(json.dumps(the_url_list,indent=4))

    def later_download(self,temp):
        pass


import os
from tqdm import tqdm
from tqdm.contrib.concurrent import process_map
import random
import time

def single_job(arg):
    task, s, prune_location, l, r, alpha = arg
    # do it quietly
    command = f"python token_pruning_bert4.py submit --target sing_octo --task {task} --sparsity {s} --location {prune_location} --learning_rate {l} --reg_learning_rate {r} --sparsity_reg_loss_alpha {alpha} --warmup_epochs 50 --tag 010202BERT4 > /dev/null 2>&1"
    os.system(command)

task = "STSB"
# sparsity = [0.3, 0.4, 0.5, 0.6]
sparsity = [0.4, 0.5, 0.6]
prune_locations = ["1,2,3"]
lr = [8e-5, 6e-5, 4e-5, 2e-5, 1e-5]
reg_lr = [0.05, 0.02, 0.01]
sparsity_reg_loss_alpha = [100.0, 200.0]

args = []
for s in sparsity:
    for prune_location in prune_locations:
        for alpha in sparsity_reg_loss_alpha:
            for l in lr:
                for r in reg_lr:
                    args.append((task, s, prune_location, l, r, alpha))

print("total jobs:", len(args))
single_job(args[0])
process_map(single_job, args, max_workers=2, chunksize=1)
print("all launched")