# some calculations about corona values
# start date somewhen in 3/2020
# added multiprocessing in 6/2020

import pandas as pd
import matplotlib.pyplot as plt
from multiprocessing import Process

# process files with these suffixes
localSuffixes = ["BW", "BY", "GER", "WORLD", "US"]

# flexinterval
flexInt = 7

def processWorker(localSuffix):
    print (f"------ processing {localSuffix} -----\n")

    #set the filename to process
    dataFile = "coronaData_" + localSuffix + ".csv"

    # read the data
    total = pd.read_csv(dataFile)

    # calculate the difference of totals

    i = 0
    # StillInfected - no of people infected but not curated
    # Diff - no of new infections from last value
    # percent - percentage of new infections from StillInfected (ratio)
    # DiffDiff - change of change (difference of changes from last change value to today)
    AktDiff = pd.DataFrame(columns=['StillInfected', 'Diff', 'Percent', 'DiffDiff'])
    lastDiff = 0
    # go through all values and calculate the addiional kpis for each day
    for x in range(len(total)):

        # get actual Number of infected people
        aktval=total.iloc[i]['Total']
        if (i != 0):
            lastval=total.iloc[i-1]['Total']
        else:
            lastval = 0

        # get actual Number of curated people
        aktCurated=total.iloc[i]['Curated']
        if (i != 0):
            lastCurated=total.iloc[i-1]['Curated']
        else:
            lastCurated = 0

        # number of deaths
        aktDeaths=total.iloc[i]['Deaths']

        # calculate number of still Infected
        stillInfected = aktval - aktCurated - aktDeaths

        # calculate difference between last value and actual value
        diff=aktval-lastval

        lastStillInfected = lastval - lastCurated

        # calculate percentage of change (how much new infected people in relation to still infected) in relation to last value
        if (lastStillInfected != 0):
            percent=(diff) / lastStillInfected *100
        else:
            #if first value, set to zero
            percent=0

        # calculate difference of difference (2nd diff)
        DiffDiff = diff-lastDiff
        lastDiff = diff

        # calculate relative rate to last week (assumption: incubation time approx 5 days, people who are positive will not infect more)
        if (i > 7):
            infectedLastWeek = AktDiff.iloc[i-5]['StillInfected']
            if (infectedLastWeek != 0):
                relInfectedToLastWeek = diff / infectedLastWeek *100
            else:
                relInfectedToLastWeek = 0
        else:
            infectedLastWeek = 0
            relInfectedToLastWeek = 0

        # copy last 30 days in container relInfectedToLastWeek30
        if (i > len(total)-20):
            relInfectedToLastWeek20 = relInfectedToLastWeek
        else:
            relInfectedToLastWeek20 = 0

        # calculate "reproduction" rate
        if (i > 7):
            #sum of new Infections day -8 to -5
            sum4Infected1 = 0
            #sum of new Infections day -4 to -1
            sum4Infected2 = 0
            # calculate the sum of new infections day -8 to -5 (they might not know that they are infected the day before)
            # be careful with the index! we are in day -1, because the array is not yet written so index (i) is today, but not present
            for k in range(4):
                sum4Infected1 =sum4Infected1 + AktDiff.iloc[i-k-5]['Diff']
            # calculate the sum of new infections day -4 to -1 (they might not know that they are infected the day before)
            for k in range(3):
                 sum4Infected2 =sum4Infected2 + AktDiff.iloc[i-k-1]['Diff']
            # add the number for today
            sum4Infected2 = sum4Infected2 + diff
            # calculate reproduction rate
            reproRate = sum4Infected2 / sum4Infected1
        else:
            reproRate = 0

        # copy last 30 days in container reproRate30
        if (i > len(total)-20):
            reproRate20 = reproRate
        else:
            reproRate20 = 0

        # calculate "reproductionflex" rate

        if (i > (2*flexInt)-1):
            #sum of new Infections day -2xflexint to -flexint
            sumFlexInfected1 = 0
            #sum of new Infections day flexint to -1
            sumFlexInfected2 = 0
            # calculate the sum of new infections day -8 to -5 (they might not know that they are infected the day before)
            # be careful with the index! we are in day -1, because the array is not yet written so index (i) is today, but not present
            for k in range(flexInt):
                sumFlexInfected1 =sumFlexInfected1 + AktDiff.iloc[i-k-flexInt-1]['Diff']
            # calculate the sum of new infections day -4 to -1 (they might not know that they are infected the day before)
            for k in range(flexInt-1):
                 sumFlexInfected2 =sumFlexInfected2 + AktDiff.iloc[i-k-1]['Diff']
            # add the number for today
            sumFlexInfected2 = sumFlexInfected2 + diff
            # calculate reproduction rate
            reproFlexRate = sumFlexInfected2 / sumFlexInfected1
        else:
            reproFlexRate = 0

        # copy last 30 days in container reproRate30
        if (i > len(total)-20):
            reproFlexRate20 = reproFlexRate
        else:
            reproFlexRate20 = 0

        AktDiff = AktDiff.append({'reproRate' : reproRate, 'reproRate20' : reproRate20,'reproFlexRate' : reproFlexRate, 'reproFlexRate20' : reproFlexRate20, 'relInfectedToLastWeek' : relInfectedToLastWeek, 'relInfectedToLastWeek20' : relInfectedToLastWeek20, 'StillInfected' : stillInfected, 'Diff' : diff, 'Percent': percent, 'DiffDiff' : DiffDiff}, ignore_index=True)

        i=i+1

    # add the calculated kpis to the values matrix
    outValue = pd.concat([total , AktDiff], axis=1)

    # create some nice charts
    SMALL_SIZE = 6
    MEDIUM_SIZE = 8
    BIGGER_SIZE = 8

    plt.rc('font', size=SMALL_SIZE)          # controls default text sizes
    plt.rc('axes', titlesize=SMALL_SIZE)     # fontsize of the axes title
    plt.rc('xtick', labelsize=SMALL_SIZE)    # fontsize of the tick labels
    plt.rc('ytick', labelsize=SMALL_SIZE)    # fontsize of the tick labels
    plt.rc('legend', fontsize=SMALL_SIZE)    # legend fontsize
    plt.rc('figure', titlesize=BIGGER_SIZE)  # fontsize of the figure title

    # get values of last date in strings
    lastDate = str(outValue.iloc[-1]['Date'])
    lastTotal = str(outValue.iloc[-1]['Total'])
    lastCurated = str(outValue.iloc[-1]['Curated'])
    lastDeaths = str(outValue.iloc[-1]['Deaths'])
    lastStillInfected = f"{outValue.iloc[-1]['StillInfected']:.0f}"

    print(f"Last date: {lastDate}\n")

    # ------------ first chart (containing of 4 subplots) ----------------
    fig, axes = plt.subplots(4,1)
    axes[0].xaxis.set_visible(False)
    axes[1].xaxis.set_visible(False)
    axes[2].xaxis.set_visible(False)

    # subchart 1: total
    outValue.plot(x='Date', y='Total', ax=axes[0], title='data taken from Berliner Morgenpost - ' + dataFile + " (" + lastDate + ")" +'\n\nAnzahl Infizierte (akt. Wert: ' + lastTotal + ")")
    axes[0].get_legend().remove()

    # subchart 2: curated
    outValue.plot(x='Date', y='Curated', color='green', ax=axes[1], title='Anzahl wieder gesund (akt. Wert: ' + lastCurated + ")")
    axes[1].get_legend().remove()

    # subchart 3: deaths
    outValue.plot(x='Date', y='Deaths', color='black', ax=axes[2], title='Anzahl Tote (akt. Wert: ' + lastDeaths + ")")
    axes[2].get_legend().remove()

    # subchart 4: stillInfected
    outValue.plot(x='Date', y='StillInfected', color='red', ax=axes[3], title='Anzahl noch infiziert (akt. Wert: ' + lastStillInfected + ")")
    axes[3].get_legend().remove()

    # autoformat the layout to fit
    plt.tight_layout()
    # save it in a file
    plt.savefig("Absolute_Values_" + localSuffix + ".png", dpi=300)

    # ------------ second chart (containing of 3 subplots): the calculated values -------------
    # get values of last date in strings
    lastPercent = f"{outValue.iloc[-1]['Percent']:.2f}"
    lastDiff = f"{outValue.iloc[-1]['Diff']:.0f}"
    lastDiffDiff = f"{outValue.iloc[-1]['DiffDiff']:.0f}"

    fig, axes = plt.subplots(3,1)
    axes[0].xaxis.set_visible(False)
    axes[1].xaxis.set_visible(False)

    outValue.plot.bar(x='Date', y='Percent', color='red', ax = axes[0], title='data taken from Berliner Morgenpost - ' + dataFile +" (" + lastDate + ")"+ '\n\nNeuinfektionen in Prozent zur Gesamtzahl der Infizierten (akt. Wert: ' + lastPercent + ")")
    outValue.plot.bar(x='Date', y='Diff', color='blue', ax = axes[1], title='Unterschied zum Vortag (absolut) (akt. Wert: ' + lastDiff + ")")
    outValue.plot.bar(x='Date', y='DiffDiff', color='blue', ax = axes[2], title='Unterschiedsänderung zum Vortag (absolut) (akt. Wert: ' + lastDiffDiff + ")")
    plt.tight_layout()

    # save it in a file
    plt.savefig("Relative_Values_" + localSuffix + ".png", dpi=300)

    ##
    #fig, axes = plt.subplots(2,1)

    # subplots with full nd second with only last 30 day due to scaling
    fig, axes = plt.subplots(4,1)
    axes[0].xaxis.set_visible(False)
    axes[1].xaxis.set_visible(False)
    axes[2].xaxis.set_visible(False)

    lastRelInfectedToLastWeek = f"{outValue.iloc[-1]['relInfectedToLastWeek']:.2f}"
    reproRate = f"{outValue.iloc[-1]['reproRate']:.2f}"

    outValue.plot.bar(x='Date', y='relInfectedToLastWeek', ax = axes[0] ,color='red', title='data taken from Berliner Morgenpost - ' + dataFile +" (" + lastDate + ")" +'\n\nRelativer Wert der Änderung zum Wert vor 5 Tagen (akt. Wert: ' + lastRelInfectedToLastWeek + ")")
    outValue.plot.bar(x='Date', y='relInfectedToLastWeek20', ax = axes[1] ,color='red', title='Relativer Wert der Änderung zum Wert vor 5 Tagen (letzte 20 Tageswerte) (akt. Wert: ' + lastRelInfectedToLastWeek + ")")
    outValue.plot.bar(x='Date', y='reproRate', ax = axes[2] ,color='blue', title='Reproduktionsrate Intervall 4 Tage (akt. Wert: ' + reproRate + ")")
    outValue.plot.bar(x='Date', y='reproRate20', ax = axes[3] ,color='blue', title='Reproduktionsrate Intervall 4 Tage (letzte 20 Tage)(akt. Wert: ' + reproRate + ")")
    plt.tight_layout()
    # save it in a file
    plt.savefig("Relative_Values_LastWeek" + localSuffix + ".png", dpi=300)

    # reprorates
    fig, axes = plt.subplots(4,1)
    axes[0].xaxis.set_visible(False)
    axes[1].xaxis.set_visible(False)
    axes[2].xaxis.set_visible(False)

    reproRate = f"{outValue.iloc[-1]['reproRate']:.2f}"
    reproFlexRate = f"{outValue.iloc[-1]['reproFlexRate']:.2f}"
    outValue.plot.bar(x='Date', y='reproFlexRate', ax = axes[0] ,color='red', title='data taken from Berliner Morgenpost - ' + dataFile +" (" + lastDate + ")" +'\n\nReprorate ' + " (Interval: " + f"{flexInt} Tage) :" + reproFlexRate )
    outValue.plot.bar(x='Date', y='reproFlexRate20', ax = axes[1] ,color='red', title='Reproduktionsrate Intervall ' + f"{flexInt}" + ' Tage (letzte 20 Tageswerte)')
    outValue.plot.bar(x='Date', y='reproRate', ax = axes[2] ,color='blue', title='Reproduktionsrate Intervall 4 Tage (akt. Wert: ' + reproRate + ")")
    outValue.plot.bar(x='Date', y='reproRate20', ax = axes[3] ,color='blue', title='Reproduktionsrate Intervall 4 Tage (letzte 20 Tage)(akt. Wert: ' + reproRate + ")")
    plt.tight_layout()
    # save it in a file
    plt.savefig("Reprorate" + localSuffix + ".png", dpi=300)
    print(f"Suffix {localSuffix} done.")
    return

if __name__ == '__main__':
    jobs = []
    for localSuffix in localSuffixes:
        print(f"starting process for suffix {localSuffix}")
        # spawn the worker
        x = Process(target=processWorker, args=(localSuffix,))
        x.start()
        jobs.append(x)
    # all processes started
    for x in jobs:
        # wait for all jobs to close
        x.join()


from django.db import models
from utils.models import BaseSlugNameModel
import uuid

from .formation_area_model import FormationArea

class SubFormationArea(BaseSlugNameModel):

    name = models.CharField(
        max_length=36,
        default=str(uuid.uuid4()),
        unique=True,
        null=False,
        blank=False
    )
    description = models.TextField(
        max_length=1000,
        null=True,
        blank=True
    )

    formation_area = models.ForeignKey(
        FormationArea,
        related_name='sub_formation_areas',
        on_delete=models.CASCADE
    )

    class Meta:
        db_table = 'sub_formation_area'
        verbose_name = 'sub_formation_area'
        verbose_name_plural = 'sub_formation_areas'

    def get_absolute_url(self):
        return f'api/formation-area/{self.formation_area}/sub-formation-area/{self.slug}/'

    def get_formation_area_slug(self):
        return self.formation_area.slug


import sys

sys.path.append("..")

from fastapi import Depends, APIRouter, Request, Response, HTTPException
from Modules import auth
from sqlalchemy.orm import Session
from models import model

router = APIRouter(
    prefix="/orientation",
    tags=["orientation"],
    responses={401: {"user": "Not Authorised"}},
)

@router.get("/getlist")
async def get_list(db: Session = Depends(auth.get_db), User: None = Depends(auth.get_current_user)):
    persons = db.query(model.Person).filter(model.Person.cultivator_id != None).all()
    for item in persons:  # remove the cultivators
        if item.cultivator_id == 0:
            persons.remove(item)
    return {"persons": persons}


#!/usr/bin/env python3.6

import requests
import json
import sys
import os
from functools import reduce
import datetime

class GithubEvents(object):
    config = {}
    issuesEvents = []

    def __init__(self, config):
        self.config = config

    def __getIssueEvents(self, url):
        headers = {
            'Content-Type': 'application/json',
            'Accept': 'application/vnd.github.starfox-preview+json',
            'Authorization': 'token {0}'.format(self.config['github']['api_token'])
        }
        payload = {'per_page': 100}
        response = requests.get(url, params=payload, headers=headers)

        return response.json()

    def __calcTimeInColumns(self, events):
        moveCardEvents = list(filter(
            lambda event: 'moved_columns_in_project' == event['event'] or 'added_to_project' == event['event'],
            events
        ))

        cyclesTimes = []
        for event in moveCardEvents:
            if 'previous_column_name' not in event['project_card']:
                continue

            startCol = event['project_card']['previous_column_name']
            startEvent = next(iter(list(filter(
                lambda event: event['project_card']['column_name'] == startCol,
                moveCardEvents
            ))))

            lastColTime = datetime.datetime.strptime(startEvent['created_at'], '%Y-%m-%dT%H:%M:%SZ')
            moveTimeToThisCol = datetime.datetime.strptime(event['created_at'], '%Y-%m-%dT%H:%M:%SZ')

            duration = moveTimeToThisCol - lastColTime
            element = {
                'column': startCol,
                'end_column': event['project_card']['column_name'],
                'duration': duration.total_seconds(),
            }

            colElement = list(filter(
                lambda cycle: startCol == cycle['column'],
                cyclesTimes
            ))
            if not colElement:
                cyclesTimes.append(element)
                continue

            element = next(iter(colElement))
            index = cyclesTimes.index(element)
            cyclesTimes[index] = element

        return cyclesTimes

    def getDateStart(self, events_url):
        events = self.__getIssueEvents(events_url)
        moveCardEvents = list(filter(
            lambda event: 'moved_columns_in_project' == event['event'] or 'added_to_project' == event['event'],
            events
        ))

        for event in moveCardEvents:
            if event['project_card']['column_name'] == 'In progress':
                return event['created_at']

        for event in moveCardEvents:
            if event['project_card']['column_name'] == 'Review':
                return event['created_at']

        for event in moveCardEvents:
            if event['project_card']['column_name'] == 'Testing':
                return event['created_at']

        return None

    def getIssuesEvents(self, issues):
        eventsList = {}
        for issue in issues:
            events = self.__getIssueEvents(issue['events_url'])
            item = { issue['id']: events }
            eventsList.update(item)

        return eventsList

    def getCardsCyclesTimes(self, issues):
        cardsCyclesTimes = {}
        for issue in issues:
            events = self.issuesEvents[str(issue['id'])]
            issueCyclesTimes = self.__calcTimeInColumns(events)
            item = { issue['id']: issueCyclesTimes }
            cardsCyclesTimes.update(item)

        return cardsCyclesTimes


import requests
import re
import json
import ast
import os
import ui
import threading
import tarfile
import math
import time
import plistlib
import console
import shutil
import sqlite3
from Managers import DBManager, TypeManager
from Utilities import LogThread
from distutils.version import LooseVersion

class Cheatsheet (object):
	def __init__(self):
		self.__version = ''
		self.__globalversion = ''
		self.__name = ''
		self.__aliases = []
		self.__icon = None
		self.__id = ''
		self.__path = None
		self.__status = ''
		self.__stats = ''
		self.__onlineid = ''

	@property
	def onlineid(self):
		return self.__onlineid

	@onlineid.setter
	def onlineid(self, id):
		self.__onlineid = id

	@property
	def version(self):
		return self.__version

	@version.setter
	def version(self, version):
		self.__version = version

	@property
	def globalversion(self):
		return self.__globalversion

	@globalversion.setter
	def globalversion(self, globalversion):
		self.__globalversion = globalversion

	@property
	def name(self):
		return self.__name

	@name.setter
	def name(self, name):
		self.__name = name

	@property
	def aliases(self):
		return self.__aliases

	@aliases.setter
	def aliases(self, aliases):
		self.__aliases = aliases

	@property
	def image(self):
		return self.__icon

	@image.setter
	def image(self, icon):
		self.__icon = icon

	@property
	def id(self):
		return self.__id

	@id.setter
	def id(self, id):
		self.__id = id

	@property
	def path(self):
		return self.__path

	@path.setter
	def path(self, path):
		self.__path = path

	@property
	def status(self):
		return self.__status

	@status.setter
	def status(self, status):
		self.__status = status

	@property
	def stats(self):
		return self.__stats

	@stats.setter
	def stats(self, stats):
		self.__stats = stats

class CheatsheetManager (object):
	def __init__(self, serverManager, iconPath, typeIconPath):
		self.typeManager = TypeManager.TypeManager(typeIconPath)
		self.serverManager = serverManager
		self.iconPath = iconPath
		self.typeIconPath = typeIconPath
		self.localServer = None
		self.jsonServerLocation = 'zzz/cheatsheets/cheat.json'
		self.downloadServerLocation = 'zzz/cheatsheets/%@.tgz'
		self.plistPath = 'Contents/Info.plist'
		self.indexPath = 'Contents/Resources/docSet.dsidx'
		self.cheatsheetFolder = 'Docsets/Cheatsheets'
		self.headers = {'User-Agent': 'PyDoc-Pythonista'}
		self.cheatsheets = None
		self.downloading = []
		self.updateAvailable = []
		self.workThreads = []
		self.downloadThreads = []
		self.uiUpdateThreads = []
		self.__createCheatsheetFolder()
		self.createInitialSearchIndexAllDocsets()

	def getAvailableCheatsheets(self):
		cheatsheets = self.__getOnlineCheatsheets()
		for d in self.__getDownloadedCheatsheets():
			for c in cheatsheets:
				if c.name == d.name:
					c.status = 'installed'
					c.path = d.path
					c.id = d.id
					c.version = d.version
		for d in self.updateAvailable:
			for c in cheatsheets:
				if c.name == d.name:
					c.status = "Update Available"
		for d in self.__getDownloadingCheatsheets():
			for c in cheatsheets:
				if c.name == d.name:
					c.status = d.status
					c.version = d.version
					try:
						c.stats = d.stats
					except KeyError:
						c.stats = 'downloading'
		return cheatsheets

	def __getOnlineCheatsheets(self):
		if self.cheatsheets == None:
			self.cheatsheets = self.__getCheatsheets()
		return self.cheatsheets

	def __getDownloadedCheatsheets(self):
		ds = []
		dbManager = DBManager.DBManager()
		t = dbManager.InstalledDocsetsByType('cheatsheet')
		ds = []
		for d in t:
			aa = Cheatsheet()
			aa.name = d[1]
			aa.id = d[0]
			aa.path = os.path.join(os.path.abspath('.'),d[2])
			aa.image = self.__getIconWithName(d[4])
			aa.version = d[5]
			ds.append(aa)
		return ds

	def __getDownloadingCheatsheets(self):
		return self.downloading

	def getDownloadedCheatsheets(self):
		return self.__getDownloadedCheatsheets()

	def __getCheatsheets(self):
		server = self.serverManager.getDownloadServer(self.localServer)
		url = server.url
		if not url[-1] == '/':
			url = url + '/'
		url = url + self.jsonServerLocation
		data = requests.get(url).text
		data = ast.literal_eval(data)

		cheatsheets = []
		icon = self.__getIconWithName('cheatsheet')
		for k,d in data['cheatsheets'].items():
			c = Cheatsheet()
			c.name = d['name']
			c.aliases = d['aliases']
			c.globalversion = data['global_version']
			c.version = d['version']
			c.image = icon
			c.onlineid = k
			c.status = 'online'
			cheatsheets.append(c)
		return sorted(cheatsheets, key=lambda x: x.name.lower())

	def checkDocsetsForUpdates(self, docsets):
		console.show_activity('Checking for updates...')
		self.cheatsheets = None
		online = self.__getOnlineCheatsheets()
		for d in docsets:
			if d.status == 'installed':
				console.show_activity('Checking ' + d.name + ' for update...')
				for f in online:
					if f.name == d.name:
						if LooseVersion(str(d.version).replace('/','')) < LooseVersion(str(f.version).replace('/','')):
							d.status = 'Update Available'
							d.version = f.version
							self.updateAvailable.append(d)

	def __getIconWithName(self, name):
		imgPath = os.path.join(os.path.abspath('.'), self.iconPath, name+'.png')
		if not os.path.exists(imgPath):
			imgPath = os.path.join(os.path.abspath('.'), self.iconPath, 'Other.png')
		return ui.Image.named(imgPath)

	def __createCheatsheetFolder(self):
		if not os.path.exists(self.cheatsheetFolder):
			os.mkdir(self.cheatsheetFolder)

	def downloadCheatsheet(self, cheatsheet, action, refresh_main_view):
		if not cheatsheet in self.downloading:
			cheatsheet.status = 'downloading'
			self.downloading.append(cheatsheet)
			removeSoon = []
			for d in self.updateAvailable:
				if d.name == cheatsheet.name:
					removeSoon.append(d)
			for d in removeSoon:
				self.updateAvailable.remove(d)
			cheatsheet.status = 'downloading'
			action()
			workThread = LogThread.LogThread(target=self.__determineUrlAndDownload, args=(cheatsheet,action,refresh_main_view,))
			self.workThreads.append(workThread)
			workThread.start()

	def __determineUrlAndDownload(self, cheatsheet, action, refresh_main_view):
		cheatsheet.stats = 'getting download link'
		action()
		downloadLink = self.__getDownloadLink(cheatsheet.onlineid)
		downloadThread = LogThread.LogThread(target=self.downloadFile, args=(downloadLink,cheatsheet,refresh_main_view,))
		self.downloadThreads.append(downloadThread)
		downloadThread.start()
		updateThread = LogThread.LogThread(target=self.updateUi, args=(action,downloadThread,))
		self.uiUpdateThreads.append(updateThread)
		updateThread.start()

	def updateUi(self, action, t):
		while t.is_alive():
			action()
			time.sleep(0.5)
		action()

	def __getDownloadLink(self, id):
		server = self.serverManager.getDownloadServer(self.localServer)
		url = server.url
		if not url[-1] == '/':
			url = url + '/'
		url = url + self.downloadServerLocation
		url = url.replace('%@', id)
		return url

	def downloadFile(self, url, cheatsheet, refresh_main_view):
		local_filename = self.__downloadFile(url, cheatsheet)
		#self.__downloadFile(url+'.tarix', cheatsheet)
		cheatsheet.status = 'waiting for install'
		self.installCheatsheet(local_filename, cheatsheet, refresh_main_view)

	def __downloadFile(self, url, cheatsheet):
		local_filename = self.cheatsheetFolder+'/'+url.split('/')[-1]
		r = requests.get(url, headers = self.headers, stream=True)
		ret = None
		if r.status_code == 200:
			ret = local_filename
			total_length = r.headers.get('content-length')
			dl = 0
			last = 0
			if os.path.exists(local_filename):
				os.remove(local_filename)
			with open(local_filename, 'wb') as f:
				for chunk in r.iter_content(chunk_size=1024):
					if chunk: # filter out keep-alive new chunks
						dl += len(chunk)
						f.write(chunk)
						if not total_length == None:
							done = 100 * dl / int(total_length)
							cheatsheet.stats = str(round(done,2)) + '% ' + str(self.convertSize(dl)) + ' / '+ str(self.convertSize(float(total_length)))
						else:
							 cheatsheet.stats = str(self.convertSize(dl))

		r.close()
		return ret

	def installCheatsheet(self, filename, cheatsheet, refresh_main_view):
		extract_location = self.cheatsheetFolder
		cheatsheet.status = 'Preparing to install: This might take a while.'
		tar = tarfile.open(filename, 'r:gz')
		n = [name for name in tar.getnames() if '/' not in name][0]
		m = os.path.join(self.cheatsheetFolder, n)
		tar.extractall(path=extract_location, members = self.track_progress(tar, cheatsheet, len(tar.getmembers())))
		tar.close()
		os.remove(filename)
		dbManager = DBManager.DBManager()
		dbManager.DocsetInstalled(cheatsheet.name, m, 'cheatsheet', 'cheatsheet', cheatsheet.version)
		if cheatsheet in self.downloading:
			self.downloading.remove(cheatsheet)
		self.indexCheatsheet(cheatsheet, refresh_main_view, m)

	def track_progress(self, members, cheatsheet, totalFiles):
		i = 0
		for member in members:
			i = i + 1
			done = 100 * i / totalFiles
			cheatsheet.status = 'installing: ' + str(round(done,2)) + '% ' + str(i) + ' / '+ str(totalFiles)
			yield member

	def indexCheatsheet(self, cheatsheet, refresh_main_view, path):
		cheatsheet.status = 'indexing'
		indexPath = os.path.join(path, self.indexPath)
		conn = sqlite3.connect(indexPath)
		sql = 'SELECT count(*) FROM sqlite_master WHERE type = \'table\' AND name = \'searchIndex\''
		c = conn.execute(sql)
		data = c.fetchone()
		if int(data[0]) == 0:
			sql = 'CREATE TABLE searchIndex(rowid INTEGER PRIMARY KEY, name TEXT, type TEXT, path TEXT)'
			c = conn.execute(sql)
			conn.commit()
			sql = 'SELECT f.ZPATH, m.ZANCHOR, t.ZTOKENNAME, ty.ZTYPENAME, t.rowid FROM ZTOKEN t, ZTOKENTYPE ty, ZFILEPATH f, ZTOKENMETAINFORMATION m WHERE ty.Z_PK = t.ZTOKENTYPE AND f.Z_PK = m.ZFILE AND m.ZTOKEN = t.Z_PK ORDER BY t.ZTOKENNAME'
			c = conn.execute(sql)
			data = c.fetchall()
			for t in data:
				conn.execute("insert into searchIndex values (?, ?, ?, ?)", (t[4], t[2], self.typeManager.getTypeForName(t[3]).name, t[0] ))
				conn.commit()
		else:
			sql = 'SELECT rowid, type FROM searchIndex'
			c = conn.execute(sql)
			data = c.fetchall()
			for t in data:
				newType = self.typeManager.getTypeForName(t[1])
				if not newType == None and not newType.name == t[1]:
					conn.execute("UPDATE searchIndex SET type=(?) WHERE rowid = (?)", (newType.name, t[0] ))
				conn.commit()
		indexSql = 'CREATE INDEX ix_searchIndex_name ON searchIndex(name)'
		conn.execute(indexSql)
		conn.close()
		self.postProcess(cheatsheet, refresh_main_view)

	def createInitialSearchIndexAllDocsets(self):
		docsets = self.getDownloadedCheatsheets()
		for d in docsets:
			indexPath = os.path.join(d.path, self.indexPath)
			conn = sqlite3.connect(indexPath)
			conn = sqlite3.connect(indexPath)
			indexSql = 'CREATE INDEX IF NOT EXISTS ix_searchIndex_name ON searchIndex(name)'
			conn.execute(indexSql)
			conn.close()

	def postProcess(self, cheatsheet, refresh_main_view):
		cheatsheet.status = 'installed'
		refresh_main_view()

	def convertSize(self, size):
		if (size == 0):
			return '0B'
		size_name = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB")
		i = int(math.floor(math.log(size,1024)))
		p = math.pow(1024,i)
		s = round(size/p,2)
		return '%s %s' % (s,size_name[i])

	def deleteCheatsheet(self, cheatsheet, post_action, confirm = True):
		but = 1
		if confirm:
			but = console.alert('Are you sure?', 'Would you like to delete the cheatsheet, ' +  cheatsheet.name, 'Ok')
		if but == 1:
			dbmanager = DBManager.DBManager()
			dbmanager.DocsetRemoved(cheatsheet.id)
			shutil.rmtree(cheatsheet.path)
			cheatsheet.status = 'online'
			if not post_action == None:
				post_action()
			cheatsheet.path = None

	def getTypesForCheatsheet(self, cheatsheet):
		types = []
		path = cheatsheet.path
		indexPath = os.path.join(path, self.indexPath)
		conn = sqlite3.connect(indexPath)
		sql = 'SELECT type FROM searchIndex GROUP BY type ORDER BY type COLLATE NOCASE'
		c = conn.execute(sql)
		data = c.fetchall()
		conn.close()
		for t in data:
			types.append(self.typeManager.getTypeForName(t[0]))
		return types

	def getIndexesbyTypeForCheatsheet(self, cheatsheet, type):
		indexes = []
		path = cheatsheet.path
		indexPath = os.path.join(path, self.indexPath)
		conn = sqlite3.connect(indexPath)
		sql = 'SELECT type, name, path FROM searchIndex WHERE type = (?) ORDER BY name COLLATE NOCASE'
		c = conn.execute(sql, (type.name,))
		data = c.fetchall()
		conn.close()
		dTypes ={}
		type = None
		for t in data:
			if t[0] in dTypes.keys():
				type= dTypes[t[0]]
			else:
				type = self.typeManager.getTypeForName(t[0])
				dTypes[t[0]] = type
			indexes.append({'type':type, 'name':t[1],'path':t[2]})
		return indexes

	def getIndexesbyTypeAndNameForDocset(self, cheatsheet, typeName, name):
		indexes = []
		path = cheatsheet.path
		indexPath = os.path.join(path, self.indexPath)
		conn = sqlite3.connect(indexPath)
		sql = 'SELECT type, name, path FROM searchIndex WHERE type = (?) AND name LIKE (?) ORDER BY name COLLATE NOCASE'
		c = conn.execute(sql, (typeName, name,))
		data = c.fetchall()
		conn.close()
		dTypes ={}
		type = None
		for t in data:
			if t[0] in dTypes.keys():
				type= dTypes[t[0]]
			else:
				type = self.typeManager.getTypeForName(t[0])
				dTypes[t[0]] = type
			indexes.append({'type':type, 'name':t[1],'path':t[2]})
		return indexes

	def getIndexesByNameForDocset(self, cheatsheet, name):
		indexes = []
		path = cheatsheet.path
		indexPath = os.path.join(path, self.indexPath)
		conn = sqlite3.connect(indexPath)
		sql = 'SELECT type, name, path FROM searchIndex WHERE name LIKE (?) ORDER BY name COLLATE NOCASE'
		c = conn.execute(sql, (name,))
		data = c.fetchall()
		conn.close()
		dTypes ={}
		type = None
		for t in data:
			if t[0] in dTypes.keys():
				type= dTypes[t[0]]
			else:
				type = self.typeManager.getTypeForName(t[0])
				dTypes[t[0]] = type
			indexes.append({'type':type, 'name':t[1],'path':t[2]})
		return indexes

	def getIndexesForCheatsheet(self, cheatsheet):
		indexes = []
		path = cheatsheet.path
		indexPath = os.path.join(path, self.indexPath)
		conn = sqlite3.connect(indexPath)
		sql = 'SELECT type, name, path FROM searchIndex ORDER BY name COLLATE NOCASE'
		c = conn.execute(sql)
		data = c.fetchall()
		conn.close()
		dTypes ={}
		type = None
		for i in data:
			if t[0] in dTypes.keys():
				type= dTypes[t[0]]
			else:
				type = self.typeManager.getTypeForName(t[0])
				dTypes[t[0]] = type
			indexes.append({'type':type, 'name':t[1],'path':t[2]})
		return types

	def getIndexesbyNameForAllCheatsheet(self, name):
		if name == None or name == '':
			return {}
		else:
			docsets = self.getDownloadedCheatsheets()
			indexes = {}
			for d in docsets:
				ind = self.getIndexesbyNameForDocsetSearch(d, name)
				for k in ind:
					if not k in indexes.keys():
						indexes[k] = []
					indexes[k].extend(ind[k])
			return indexes

	def getIndexesbyNameForDocsetSearch(self, docset, name):
		if name == None or name == '':
			return []
		else:
			ind = {}
			path = docset.path
			indexPath = os.path.join(path, self.indexPath)
			conn = sqlite3.connect(indexPath)
			sql = 'SELECT type, name, path FROM searchIndex WHERE name LIKE (?) ORDER BY name COLLATE NOCASE'
			c = conn.execute(sql, (name, ))
			data = {'first' : c.fetchall()}

			sql = 'SELECT type, name, path FROM searchIndex WHERE name LIKE (?) AND name NOT LIKE (?) ORDER BY name COLLATE NOCASE'
			c = conn.execute(sql, (name.replace(' ','%'), name, ))
			data['second'] = c.fetchall()

			sql = 'SELECT type, name, path FROM searchIndex WHERE name LIKE (?) AND name NOT LIKE (?) AND name NOT LIKE (?) ORDER BY name COLLATE NOCASE'
			c = conn.execute(sql, (name.replace(' ','%')+'%', name.replace(' ','%'), name, ))
			data['third'] = c.fetchall()

			sql = 'SELECT type, name, path FROM searchIndex WHERE name LIKE (?) AND name NOT LIKE (?) AND name NOT LIKE (?) AND name NOT LIKE (?) ORDER BY name COLLATE NOCASE'
			c = conn.execute(sql, ('%'+name.replace(' ','%')+'%',name.replace(' ','%')+'%',name.replace(' ','%'), name, ))
			data['fourth'] = c.fetchall()

			conn.close()
			dTypes = {}
			for k in data:
				ind[k] = []
				for t in data[k]:
					url = 'file://' + os.path.join(path, 'Contents/Resources/Documents', t[2])
					url = url.replace(' ', '%20')
					type = None
					if t[0] in dTypes.keys():
						type= dTypes[t[0]]
					else:
						type = self.typeManager.getTypeForName(t[0])
						dTypes[t[0]] = type
					ind[k].append({'name':t[1], 'path':url, 'icon':docset.image,'docsetname':docset.name,'type':type, 'callbackOverride':'', 'docset': docset})
			return ind

if __name__ == '__main__':
	import ServerManager
	c = CheatsheetManager(ServerManager.ServerManager(), '../Images/icons')


from collections import deque
from sound.music_player import MusicPlayer
import logging
from logging import getLogger
from excluded.config import midi_output

class NoteString:
    """
    Stores notes in a string, similar to lights
    When a new note is pushed in, and old one is pushed out
    """

    def __init__(self, max_len, midi_port=midi_output):
        self.mp: MusicPlayer = MusicPlayer(midi_port)
        self.contents: deque = deque(maxlen=max_len)
        for x in range(len(self.contents)):
            self.contents[x] = None
        logging.basicConfig(level=logging.INFO)
        self.logger = getLogger('NoteString')

    def add(self, note: int):
        self.stop_last_note()
        self.contents.appendleft(note)
        self.mp.play_note(note)
        self.logger.info(f'Added note {note}')

    def stop_last_note(self):
        if len(self.contents) == self.contents.maxlen:
            last_note = self.contents[self.contents.maxlen - 1]
            self.mp.stop_note(last_note)
            self.logger.info(f'Removed note {last_note}')


'''
Created on Jan 18, 2016

@author: azaringh
'''
import redis
import json
import csv
import os

stats_name = 'chicago:ge-1/0/5:traffic statistics'

r = redis.StrictRedis(host='10.10.4.252', port=6379, db=0)
csv_file = os.getenv('HOME') + '/' + stats_name.replace(':', '_').replace('/', '-').replace(' ', '_') + '.csv'
f = open(csv_file, 'w+')   # Read and write

def create_csv():
    csvwriter = csv.writer(f)
    stat_list = r.lrange(stats_name, 0 , -1)
    hflag = True
    for stat in reversed(stat_list):
        j = json.loads(stat)

        d = { 'time': j['timestamp'],
              'input-bytes': j['stats'][0]['input-bytes'][0]['data'],
              'input-bps': j['stats'][0]['input-bps'][0]['data'],
              'output-pps': j['stats'][0]['output-pps'][0]['data'],
              'output-bps': j['stats'][0]['output-bps'][0]['data'],
              'output-bytes': j['stats'][0]['output-bytes'][0]['data'],
              'input-pps': j['stats'][0]['input-pps'][0]['data'],
              'input-packets': j['stats'][0]['input-packets'][0]['data'],
              'output-packets': j['stats'][0]['output-packets'][0]['data']
             }
        if hflag == True:
            header = d.keys()
            csvwriter.writerow(header)
            hflag = False
        csvwriter.writerow(d.values())
    return

create_csv()


import re
from collections import namedtuple

class Files:
    def __init__(self, document, filepaths):
        self.d = document
        self.fl = filepaths  # list

    def order_files(self):
        File = namedtuple("File", ["num", "filepath"])
        # On récupère le modèle du nom de chemin complet vers un fichier
        exempleChemin = str(self.fl[0])
        exempleNom = exempleChemin.split("/")[-1]

        # On adapte l'ordonnancement des fichiers à CDS
        if exempleNom[:3] == "CdS":
            # Modèle de nom CdS02_Konv002-02_0027
            ordered_files = sorted([File(int(re.search(r"(CdS\d+_Konv\d+-?\d*_)(\d+)", f.name).group(2)), f)
                                    for f in self.fl])
        else:
            ordered_files = sorted([File(int(re.search(r"(.*f)(\d+)", f.name).group(2)), f) for f in self.fl])

        return ordered_files


'''
    Paper: Towards K-means friendly spaces: simultaneous deep learning and clustering
    Example implementation: https://github.com/xuyxu/Deep-Clustering-Network
'''

import torch
import torch.nn as nn
import numpy as np
import tqdm
import os
import itertools
import yaml
from sklearn.cluster import KMeans
from dataset import create_mnist_dataset, visualize_mnist_data
from sklearn.metrics import adjusted_rand_score, normalized_mutual_info_score
from utils import Logger, create_mlp
from scipy.optimize import linear_sum_assignment

'''
    Neural network
'''
class MNISTFeatureExtractor(nn.Module):
    def __init__(self, hidden_dims = [500, 500, 2000], feature_dim = 10) -> None:
        super().__init__()

        # self.encoder = create_mlp(784, feature_dim, hidden_dims, act_fn = nn.ReLU)
        # self.decoder = create_mlp(feature_dim, 784, list(reversed(hidden_dims)), act_fn = nn.ReLU)

        self.encoder = []
        encoder_dims = [784] + hidden_dims
        for in_dim, out_dim in zip(encoder_dims[:-1], encoder_dims[1:]):
            self.encoder.append(nn.Linear(in_dim, out_dim))
            self.encoder.append(nn.ReLU())
            # self.encoder.append(nn.BatchNorm1d(out_dim))
        self.encoder.append(nn.Linear(encoder_dims[-1], feature_dim))
        self.encoder = nn.Sequential(*self.encoder)

        self.decoder = []
        decoder_dims = [feature_dim] + list(reversed(hidden_dims))
        for in_dim, out_dim in zip(decoder_dims[:-1], decoder_dims[1:]):
            self.decoder.append(nn.Linear(in_dim, out_dim))
            self.decoder.append(nn.ReLU())
            # self.decoder.append(nn.BatchNorm1d(out_dim))
        self.decoder.append(nn.Linear(decoder_dims[-1], 784))
        self.decoder = nn.Sequential(*self.decoder)

    def forward(self, x):
        feature = self.extract_feature(x)
        recon_x = self.decoder(feature)
        return feature, recon_x

    def extract_feature(self, x):
        return self.encoder(x.view(-1, 784))

class MNISTCNNFeatureExtractor(nn.Module):
    def __init__(self, feature_dim = 10) -> None:
        super().__init__()

        self.encoder = nn.Sequential(
            nn.Conv2d(1, 64, 4, 2, 1),
            nn.BatchNorm2d(64),
            nn.ReLU(),
            nn.Conv2d(64, 128, 4, 2, 1),
            nn.BatchNorm2d(128),
            nn.ReLU(),
            nn.Conv2d(128, 256, 3, 2, 1),
            nn.BatchNorm2d(256),
            nn.ReLU(),
            nn.Flatten(),
            nn.Linear(256 * 4 * 4, 512),
            nn.ReLU(),
            nn.Linear(512, feature_dim)
        )

        self.decoder1 = nn.Sequential(
            nn.Linear(feature_dim, 512),
            nn.ReLU(),
            nn.Linear(512, 256 * 4 * 4),
        )
        self.decoder2 = nn.Sequential(
            nn.ConvTranspose2d(256, 128, 3, 2, 1),
            nn.BatchNorm2d(128),
            nn.ReLU(),
            nn.ConvTranspose2d(128, 64, 4, 2, 1),
            nn.BatchNorm2d(64),
            nn.ReLU(),
            nn.ConvTranspose2d(64, 1, 4, 2, 1),
            nn.Flatten()
        )

    def extract_feature(self, x):
        return self.encoder(x)

    def forward(self, x):
        feature = self.extract_feature(x)
        z = self.decoder1(feature)
        z = z.view(-1, 256, 4, 4)
        z = self.decoder2(z)
        return feature, z

'''
    Clustering
'''
class MyKmeans(object):
    def __init__(self, feature_dim, n_clusters) -> None:
        self.feature_dim = feature_dim
        self.n_clusters = n_clusters
        self.M = np.zeros((n_clusters, feature_dim))
        self.cluster_counter = np.ones((n_clusters)) * 100

    def init_cluster(self, features):
        kmeans = KMeans(self.n_clusters, n_init = 20)
        kmeans = kmeans.fit(features)
        self.M = kmeans.cluster_centers_

    def update_M(self, features, clusters):
        for f, c in zip(features, clusters):
            self.cluster_counter[c] += 1
            eta = 1.0 / self.cluster_counter[c]
            self.M[c] = (1 - eta) * self.M[c] + eta * f

    def assign_cluster(self, features):
        return np.argmin(np.sqrt(np.sum((features[:, None] - self.M[None])**2, axis = -1)), axis = -1)

    def save(self, path):
        np.savez(path, M = self.M, cluster_counter = self.cluster_counter)

    def load(self, path):
        with np.load(path) as f:
            self.M = f['M']
            self.cluster_counter = f['cluster_counter']

'''
    ACC
'''
def acc_func(y, ypred):
    """
    https://github.com/sarsbug/DCN_keras/blob/master/metrics.py
    Calculating the clustering accuracy. The predicted result must have the same number of clusters as the ground truth.

    ypred: 1-D numpy vector, predicted labels
    y: 1-D numpy vector, ground truth
    The problem of finding the best permutation to calculate the clustering accuracy is a linear assignment problem.
    This function construct a N-by-N cost matrix, then pass it to scipy.optimize.linear_sum_assignment to solve the assignment problem.

    """
    assert len(y) > 0

    s = np.unique(ypred)
    t = np.unique(y)

    N = len(np.unique(ypred))
    C = np.zeros((N, N), dtype = np.int32)
    for i in range(N):
        for j in range(N):
            idx = np.logical_and(ypred == s[i], y == t[j])
            C[i][j] = np.count_nonzero(idx)

    # convert the C matrix to the 'true' cost
    Cmax = np.amax(C)
    C = Cmax - C
    row,col = linear_sum_assignment(C)
    # calculating the accuracy according to the optimal assignment
    count = 0
    for i in range(N):
        idx = np.logical_and(ypred == s[row[i]], y == t[col[i]] )
        count += np.count_nonzero(idx)

    return 1.0*count/len(y)

'''
    Main function
'''
def main(
    feature_dim = 10,
    hidden_dims = [500, 500, 2000],
    pretrain_epoch = 5,
    epoch = 20,
    lr = 1e-3,
    lamda = 0.1,
    batch_size = 512,
    log_path = './towards_kmeans_friendly_log/'
):
    # Dataset
    dataset = create_mnist_dataset(True)

    # DNN
    device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
    # model = MNISTFeatureExtractor(hidden_dims, feature_dim).to(device)
    model = MNISTCNNFeatureExtractor(feature_dim).to(device)
    optimizer = torch.optim.Adam(model.parameters(), lr = lr, weight_decay = 5e-4)

    # Kmeans
    kmeans = MyKmeans(feature_dim, 10)

    # Log
    log = Logger(log_path)

    # Pre-training
    print('Start pre-training.')
    dataset_loader = torch.utils.data.DataLoader(dataset, batch_size, True)
    model.train()
    for e in itertools.count(1):
        bar = tqdm.tqdm(dataset_loader)
        for img, _ in bar:
            img = img.to(device)
            _, recon_img = model(img)
            recon_loss = (img.view(-1, 784) - recon_img).pow(2).sum()
            optimizer.zero_grad()
            recon_loss.backward()
            optimizer.step()
            bar.set_description(f'Epoch {e}/{pretrain_epoch} - Recon_loss: {recon_loss.item():.6f}')

        if e >= pretrain_epoch:
            break

    # Initialize kmeans
    print('Initialize kmeans.')
    model.eval()
    with torch.no_grad():
        features = torch.cat([model.extract_feature(img.to(device)) for img, _ in torch.utils.data.DataLoader(dataset, batch_size, False)]).cpu().numpy()
    kmeans.init_cluster(features)

    # Update both DNN and kmeans
    print('Start running DCN.')
    dataset_loader = torch.utils.data.DataLoader(dataset, batch_size, False)
    for e in itertools.count(1):
        model.train()
        bar = tqdm.tqdm(dataset_loader)
        for img, _ in bar:
            img = img.to(device)

            with torch.no_grad():
                features_nograd = model.extract_feature(img).cpu().numpy()
            cluster_indices = kmeans.assign_cluster(features_nograd)
            kmeans.update_M(features_nograd, cluster_indices)

            features, recon_img = model(img)
            # Reconstruction loss
            recon_loss = (img.view(-1, 784) - recon_img).pow(2).sum()
            # Cluster loss
            M_ts = torch.from_numpy(kmeans.M).to(device)
            M_ts = torch.index_select(M_ts, 0, torch.from_numpy(cluster_indices).to(device))
            cluster_loss = 0.5 * lamda * (features - M_ts).pow(2).sum()
            # Total loss
            total_loss = recon_loss + cluster_loss
            optimizer.zero_grad()
            total_loss.backward()
            optimizer.step()
            bar.set_description(f'Epoch {e}/{epoch} - Recon_loss: {recon_loss.item():.6f} | Cluster_loss: {cluster_loss.item():.6f} | Total_loss: {total_loss.item():.6f}')

        if e % 5 == 0:
            model.eval()
            # Evaluation
            y_gt, y_pred = [], []
            with torch.no_grad():
                for img, label in dataset_loader:
                    img = img.to(device)
                    features = model.extract_feature(img).cpu().numpy()
                    clusters = kmeans.assign_cluster(features)
                    y_pred.append(clusters)
                    y_gt.append(label.cpu().numpy())
            y_gt, y_pred = np.concatenate(y_gt), np.concatenate(y_pred)
            ari = adjusted_rand_score(y_gt, y_pred)
            nmi = normalized_mutual_info_score(y_gt, y_pred)
            acc = acc_func(y_gt, y_pred)
            print(f'Epoch {e}/{epoch} - ARI: {ari:.6f} | NMI: {nmi:.6f} | ACC: {acc:.6f}')
            log.add(e, {'ARI' : ari, 'NMI' : nmi, 'ACC' : acc}, 'Eval/')

            # Visualize the performance (Sometimes the empty cluster problem exists!)
            vis_imgs = []
            for i in range(10):
                test_indices = np.where(y_pred == i)[0][:16]
                if len(test_indices) < 16:
                    vis_imgs.append(torch.zeros((16, 1, 28, 28)))
                else:
                    cluster_vis_imgs = [dataset[idx][0] for idx in test_indices]
                    vis_imgs.append(torch.stack(cluster_vis_imgs))
            vis_imgs = torch.cat(vis_imgs)
            visualize_mnist_data(vis_imgs, save_path = log_path + f'eval_epoch{e}.png', n_row = 16)

        if e >= epoch:
            break

    # Save models
    state_dict = {
        'model' : model.state_dict(),
        'optimizer' : optimizer.state_dict()
    }
    if not os.path.exists(log_path):
        os.mkdir(log_path)
    torch.save(state_dict, log_path + 'dnn.pkl')
    kmeans.save(log_path + 'kmeans.npz')

if __name__ == '__main__':
    config = dict(
        feature_dim = 10,
        hidden_dims = [500, 500, 2000],
        pretrain_epoch = 100,
        epoch = 100,
        lr = 1e-3,
        lamda = 1,
        batch_size = 512,
        log_path = './towards_kmeans_friendly_log13/'
    )
    with open('config.yaml', 'w') as f:
        yaml.dump(config, f)
    main(**config)


from flask import Flask, render_template, jsonify
import pandas as pd
import requests
import etl

# adding new comment

# Access NOAA API and return pandas dataframe for year range selected

def call_api(year):
    global df
    the_json = (requests.get("https://www.ngdc.noaa.gov/hazel/hazard-service/api/v1/volcanoes?maxYear=2022&minYear=2000")).json()
    empty_df = pd.DataFrame(columns=["id","name","country","year","morphology","vei","deathsTotal",\
                    "missingTotal","injuriesTotal","damageMillionsDollarsTotal",\
                    "housesDestroyedTotal"])

    try:
        df = pd.DataFrame(data=the_json["items"])
        df = df[["id","name","country","year","morphology","vei","deathsTotal",\
                    "missingTotal","injuriesTotal","damageMillionsDollarsTotal",\
                    "housesDestroyedTotal"]]
    except:
        print("Error making dataframe")
        return empty_df

    # Define Year Range Dataframes

    df_dict = {}

    try:
        df_2000_2022 = df.copy()
        df_2000_2022.index.name = "2000-2022"
        df_dict[df_2000_2022.index.name] = df_2000_2022

        df_2000_2011 = df.loc[((df["year"]>=2000) & (df["year"]<=2011)), :]
        df_2000_2011.index.name = "2000-2011"
        df_dict[df_2000_2011.index.name] = df_2000_2011

        df_2012_2022 = df.loc[((df["year"]>=2012) & (df["year"]<=2022)), :]
        df_2012_2022.index.name = "2012-2022"
        df_dict[df_2012_2022.index.name] = df_2012_2022

        df_2000_2004 = df.loc[((df["year"]>=2000) & (df["year"]<=2004)), :]
        df_2000_2004.index.name = "2000-2004"
        df_dict[df_2000_2004.index.name] = df_2000_2004

        df_2005_2009 = df.loc[((df["year"]>=2005) & (df["year"]<=2009)), :]
        df_2005_2009.index.name = "2005-2009"
        df_dict[df_2005_2009.index.name] = df_2005_2009

        df_2010_2014 = df.loc[((df["year"]>=2010) & (df["year"]<=2014)), :]
        df_2010_2014.index.name = "2010-2014"
        df_dict[df_2010_2014.index.name] = df_2010_2014

        df_2015_2019 = df.loc[((df["year"]>=2015) & (df["year"]<=2019)), :]
        df_2015_2019.index.name = "2015-2019"
        df_dict[df_2015_2019.index.name] = df_2015_2019

        df_2020_2022 = df.loc[((df["year"]>=2020) & (df["year"]<=2022)), :]
        df_2020_2022.index.name = "2020-2022"
        df_dict[df_2020_2022.index.name] = df_2020_2022

    except:
        print("Error making dataframe")
        return empty_df

    year_range_list = df_dict.keys()

    if year in year_range_list:

        return df_dict[year]

    else:

        print("Error finding year range")
        return empty_df

# Make sure year range selection is valid

def validate_year(year):
    if year in ["2000-2022", "2000-2011", "2012-2022", "2000-2004",\
        "2005-2009", "2010-2014", "2015-2019", "2020-2022"]:
        return year

    else:
        return "2000-2022"

#############################################################
# BEGIN FLASK ROUTING
#############################################################

app = Flask(__name__)
app.config["JSON_SORT_KEYS"] = False
app.config["SEND_FILE_MAX_PAGE_DEFAULT"] = 0

#############################################################
# Index endpoint
#############################################################

@app.route("/")
def index():
    etl.load()
    return render_template("index.html")

#############################################################

@app.route("/map")
def map():
    return render_template("map.html")

@app.route("/new")
def new():
    return render_template("new.html")

@app.route("/volcano")
def ServeVolcano():
   return render_template("volcano.html")

@app.route("/year")
def ServeYear():
   return render_template("year.html")

#############################################################

@app.route("/readmongodb")
def ReadMongoDB():
    data = etl.fetch()
    return jsonify(data)

@app.route("/volcanoes_by_year/<year>")
def volcanoes_by_year(year):
    global df
    df = call_api(validate_year(year))
    vol_by_yr = df.groupby("year")["id"].count().to_dict()

    return jsonify(vol_by_yr)

@app.route("/volcanos_by_country/<year>")
def volcanos_by_country(year):
    global df
    df = call_api(validate_year(year))
    vol_by_ctry = df.groupby(["country"])["id"].count().sort_values(ascending=False).to_dict()

    return jsonify(vol_by_ctry)

@app.route("/volcanos_by_morphology/<year>")
def volcanos_by_morphology(year):
    global df
    df = call_api(validate_year(year))
    vol_by_morph = df.groupby(["morphology"])["id"].count().sort_values(ascending=False).to_dict()

    return jsonify(vol_by_morph)

@app.route("/volcanos_by_vei/<year>")
def volcanos_by_vei(year):
    global df
    df = call_api(validate_year(year))
    vol_by_vei = df.groupby(["vei"])["id"].count().sort_values(ascending=False).to_dict()

    return jsonify(vol_by_vei)

@app.route("/summary_data/<year>")
def summary_data(year):
    global df
    df = call_api(validate_year(year))
    deaths = round(df["deathsTotal"].sum())
    injuries = round(df["injuriesTotal"].sum())
    houses = round(df["housesDestroyedTotal"].sum())
    damages = round(df["damageMillionsDollarsTotal"].sum())*1_000_000

    summ_data = {"Deaths": "{:0,}".format(deaths), "Injuries": "{:0,}".format(injuries),
                 "Houses Destroyed": "{:0,}".format(houses), "Damages": "${:0,}".format(damages)}

    return jsonify(summ_data)

@app.route("/graphs")
def makeGraph():
    return render_template("graphs.html")

#############################################################
# END FLASK ROUTING
#############################################################

if __name__ == "__main__":
    app.run(debug=True)


import torch.cuda
from dataset_processing.dataset_processing_pipeline import data_pipeline
import torch.nn as nn
from torch.optim import lr_scheduler
import torch.optim as optim
import json
from model.model import CNNNLPModel
from tqdm import tqdm

from saving_loading_model.saving import save_checkpoint
from training.train import train
from training.evaluate import evaluate

def main(
        train_file,
        test_file,
        config_file,
        checkpoint_path,
        best_model_path
    ):
    device = 'cuda' if torch.cuda.is_available() else 'cpu'

    with open(config_file, 'r') as j:
        config = json.loads(j.read())

    for k,v in config['model'].items():
        v = float(v)
        if v < 1.0:
            config['model'][k] = float(v)
        else:
            config['model'][k] = int(v)

    for k,v in config['training'].items():
        v = float(v)
        if v < 1.0:
            config['training'][k] = float(v)
        else:
            config['training'][k] = int(v)

    train_itr, val_itr, test_itr = data_pipeline(
        train_file,
        test_file,
        config['training']['max_vocab'],
        config['training']['min_freq'],
        config['training']['batch_size'],
        device
    )

    model = CNNNLPModel(
        config['training']['max_vocab']+2,
        config['model']['emb_dim'],
        config['model']['hid_dim'],
        config['model']['model_layer'],
        config['model']['model_kernel_size'],
        config['model']['model_dropout'],
        device
    )
    optimizer = optim.Adam(model.parameters())
    criterion = nn.BCEWithLogitsLoss()
    scheduler = lr_scheduler.StepLR(optimizer,step_size=10,gamma=0.5)
    num_epochs = config['training']['n_epoch']
    clip = config['training']['clip']
    is_best = False
    best_valid_loss = float('inf')
    model = model.to(device)
    for epoch in tqdm(range(num_epochs)):

        train_loss = train(model, train_itr, optimizer, criterion, clip)
        valid_loss = evaluate(model, val_itr, criterion)

        # if (epoch + 1) % 2 == 0:
        print("training loss {}, validation_loss{}".format(train_loss,valid_loss))

        if valid_loss < best_valid_loss:
            best_valid_loss = valid_loss
            is_best = True
        checkpoint = {
            'epoch': epoch + 1,
            'valid_loss_min': valid_loss,
            'state_dict': model.state_dict(),
            'optimizer': optimizer.state_dict(),
        }
        save_checkpoint(checkpoint, is_best, checkpoint_path, best_model_path)
        scheduler.step()

if __name__=="__main__":
    train_file = r'C:\Users\Ankan\Downloads\sub_sub_train.csv'
    test_file = r'C:\Users\Ankan\Downloads\sub_test.csv'
    config_file = r'C:\Users\Ankan\Desktop\Github\CNN-with-NLP\config.json'
    ckpt_path = r'C:\Users\Ankan\Desktop\Github\CNN-with-NLP\ckpt_best_path\latest.pt'
    best_model_path = r'C:\Users\Ankan\Desktop\Github\CNN-with-NLP\ckpt_best_path\best.pt'
    main(train_file,test_file,config_file,ckpt_path,best_model_path)


import pika
import csv
import datetime

connection = pika.BlockingConnection(pika.ConnectionParameters(host="localhost"))
channel = connection.channel()
channel.queue_declare(queue="task_queue", durable=True)

def write_file(message_body):
    """
    Write a csv file in current working directory that appends with new
    values for each new message sent from the meter, as well as a timestamp,
    the value in kilowatts, and the sum of the powers (meter + PV), as requested in
    the project specifications.
    :param message body: string with the randomly generated watt amount
    :return: csv file object
    """
    timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    meter_value_in_kilowatts = int(message_body) / 1000

    with open("output_file.csv", "a") as f:
        writer = csv.DictWriter(
            f, fieldnames=["timestamp", "meter_value", "pv_power_value", "power_sum"]
        )
        if f.tell() == 0:  # only write header row once, at position 0
            writer.writeheader()
        writer.writerow(
            {
                "timestamp": timestamp,
                "meter_value": f"{message_body} watts",
                "pv_power_value": f"{meter_value_in_kilowatts} kilowatts",
                "power_sum": sum([int(message_body), meter_value_in_kilowatts]),
            }
        )

def callback(ch, method, properties, body):
    print(f"Recieved meter reading: {body.decode()} watts")
    write_file(body.decode())
    # basic_ack confirms successful delivery
    ch.basic_ack(delivery_tag=method.delivery_tag)

# basic_qos limits the unacknowledged deliveries to 1
channel.basic_qos(prefetch_count=1)
channel.basic_consume(queue="task_queue", on_message_callback=callback)

channel.start_consuming()


import plotly.graph_objects as go
import numpy as np
from scipy.interpolate import lagrange
from numpy.polynomial.polynomial import Polynomial
bots = [i+1 for i in range(15)]
time = [1.570,3.072,4.902,7.116,9.775,12.198,15.136,17.763,21.305,25.232,28.219,32.909,36.303,40.997,45.345]
poly = lagrange(bots,time)
x_new = np.arange(1,15.1,0.1)
y_new = Polynomial(poly.coef[::-1])(x_new)
print(type(y_new),type(x_new))
fig = go.Figure()
fig.add_trace(go.Scatter(x=bots,y=time,mode='lines'))
# fig.add_trace(go.Scatter(x=x_new,y=y_new,mode='lines'))
fig.add_trace(go.Scatter(x=x_new,y=np.interp(x_new,bots,time),mode='lines'))

fig.show()


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from scipy import stats
from sklearn.preprocessing import OneHotEncoder
from sklearn.model_selection import train_test_split
from sklearn import linear_model
from sklearn.metrics import mean_squared_error
# import sys

# np.set_printoptions(threshold=sys.maxsize)
# https://archive.ics.uci.edu/dataset/183/communities+and+crime dataset
df = pd.read_csv('data/communities.data', names=['state', 'county', 'community', 'population', 'householdsize'],
                 usecols=[0, 1, 2, 4, 5], na_values='?', dtype=float)

ar = np.genfromtxt('data/communities.data', dtype=float, names=['state', 'county', 'community', 'population',
                                                                'householdsize'],
                   usecols=[0, 1, 2, 4, 5], missing_values='?', delimiter=',', filling_values=0)

def fill_df_na():
    df.fillna(0)
    for i in ['county', 'community', 'population', 'householdsize']:
        df[i].fillna(df[i].mean(), inplace=True)

def fill_ar_na():
    for i in ['county', 'community', 'population', 'householdsize']:
        n = 0
        mean = np.mean(ar[i][ar[i] != 0])
        for a in ar[i]:
            ar[i][n] = mean if a == 0. else a
            n += 1

def normalize_df():
    for i in ['county', 'community', 'population', 'householdsize']:
        mean = df[i].mean()
        std = df[i].std()
        for a in range(len(df[i])):
            df.at[a, i] = (df.at[a, i] - mean) / std

def normalize_ar():
    min_max_county = (np.std(ar['county']), np.mean(ar['county']))
    min_max_community = (np.std(ar['community']), np.mean(ar['community']))
    min_max_population = (np.std(ar['population']), np.mean(ar['population']))
    min_max_householdsize = (np.std(ar['householdsize']), np.mean(ar['householdsize']))
    n = 0
    for i in ar:
        ar[n][1] = (i[1] - min_max_county[1]) / min_max_county[0]
        ar[n][2] = (i[2] - min_max_community[1]) / min_max_community[0]
        ar[n][3] = (i[3] - min_max_population[1]) / min_max_population[0]
        ar[n][4] = (i[4] - min_max_householdsize[1]) / min_max_householdsize[0]
        n += 1

normalize_df()
normalize_ar()

def plotting_ar(atr, *args):
    for i in range(1, len(args) + 1):
        plt.subplot(2, 3, i % 5 if i % 5 != 0 else 5)
        plt.hist(ar[atr][args[i - 1][0]:args[i - 1][1]], label=f'{args[i - 1][0]} - {args[i - 1][1]}')
        plt.legend()
        plt.xlabel(atr.title())
    plt.tight_layout()
    # plt.show()

# plotting_ar('community', (10, 100), (100, 200), (200, 300), (300, 400), (400, 500), (500, 600), (600, 700), (700, 800),
#                          (800, 900), (900, 1000))

def plotting_df(atr, *args):
    for i in range(1, len(args) + 1):
        plt.subplot(2, 3, i % 5 if i % 5 != 0 else 5)
        plt.hist(df[atr][args[i - 1][0]:args[i - 1][1]], label=f'{args[i - 1][0]} - {args[i - 1][1]}')
        plt.legend()
        plt.xlabel(atr.title())
    plt.tight_layout()
    # plt.show()

# plotting_df('community', (10, 100), (100, 200), (200, 300), (300, 400), (400, 500), (500, 600), (600, 700), (700, 800),
#                          (800, 900), (900, 1000))

def cor_plot_df(atr_1, atr_2):
    sns.kdeplot(data=df, x=atr_1, hue=atr_2)
    # plt.show()

def cor_plot_ar(atr_1, atr_2):
    data_frame = pd.DataFrame(ar)
    sns.kdeplot(data=data_frame, x=atr_1, hue=atr_2)
    # plt.show()

def pearson_ar(atr_1, atr_2):
    ar_pearson = stats.pearsonr(ar[atr_1], ar[atr_2]).statistic
    return ar_pearson

def pearson_df(atr_1, atr_2):
    df_pearson = stats.pearsonr(df[atr_1], df[atr_2]).statistic
    return df_pearson

# print(pearson('community', 'householdsize'))

def one_hot_ar():
    global ar
    ar = ar.view(np.float64).reshape(ar.shape + (-1,))
    with open('data/states.txt', 'r') as file:
        states = [i.replace('\n', '') for i in file.readlines()]
        states.insert(0, '')
        states.extend(['California' for i in range(6)])
    sta = []
    for i in range(len(ar[:, 0])):
        sta.append(states[int(ar[:,0][i])])
    sta = np.array(sta).reshape((len(ar[:, 0]), 1))
    oh = OneHotEncoder()
    new = oh.fit_transform(sta).toarray()
    new = np.hstack((new, ar))
    return new

def one_hot_df():
    global df
    with open('data/states.txt', 'r') as file:
        states = [i.replace('\n', '') for i in file.readlines()]
        states.insert(0, '')
        states.extend(['California' for i in range(6)])
    df.replace([np.float64(i) for i in range(1, 58)], states, inplace=True)
    oh = OneHotEncoder()
    new = oh.fit_transform(df['state'].to_numpy().reshape((-1, 1))).toarray()
    new = pd.DataFrame(new)
    new = pd.concat([df, new], axis=1).drop('state', axis=1)
    return new

def viz_3d_df():
    fig = plt.figure(figsize=(8, 6))
    ax = fig.add_subplot(111, projection='3d')
    xs = df['community']
    ys = df['householdsize']
    zs = df['county']
    ax.scatter(xs, ys, zs, s=50, alpha=0.6, edgecolors='w')
    ax.set_xlabel('Community')
    ax.set_ylabel('Household Size')
    ax.set_zlabel('County')
    # plt.show()

def viz_3d_ar():
    fig = plt.figure(figsize=(8, 6))
    ax = fig.add_subplot(111, projection='3d')
    xs = ar['community']
    ys = ar['householdsize']
    zs = ar['county']
    ax.scatter(xs, ys, zs, s=50, alpha=0.6, edgecolors='w')
    ax.set_xlabel('Community')
    ax.set_ylabel('Household Size')
    ax.set_zlabel('County')
    # plt.show()

def np_reg():
    data = train_test_split(ar, test_size=0.5, train_size=0.5)
    reg = linear_model.LinearRegression()
    reg.fit(data[0]['community'].reshape(1, -1), data[0]['householdsize'].reshape(1, -1))
    print(mean_squared_error(data[1]['householdsize'].reshape(1, -1),
                             reg.predict(data[1]['community'].reshape(1, -1))))

    reg1 = linear_model.Lasso(alpha=0.1)
    reg1.fit(data[0]['community'].reshape(1, -1), data[0]['householdsize'].reshape(1, -1))
    print(mean_squared_error(data[1]['householdsize'].reshape(1, -1),
                             reg1.predict(data[1]['community'].reshape(1, -1))))

    reg2 = linear_model.LassoLars(alpha=.1)
    reg2.fit(data[0]['community'].reshape(1, -1), data[0]['householdsize'].reshape(1, -1))
    print(mean_squared_error(data[1]['householdsize'].reshape(1, -1),
                             reg2.predict(data[1]['community'].reshape(1, -1))))

def df_reg():
    data = train_test_split(df, test_size=0.5, train_size=0.5)
    reg = linear_model.LinearRegression()
    reg.fit(data[0]['community'].to_numpy().reshape(1, -1), data[0]['householdsize'].to_numpy().reshape(1, -1))
    print(mean_squared_error(data[1]['householdsize'].to_numpy().reshape(1, -1),
                             reg.predict(data[1]['community'].to_numpy().reshape(1, -1))))

    reg1 = linear_model.Lasso(alpha=0.1)
    reg1.fit(data[0]['community'].to_numpy().reshape(1, -1), data[0]['householdsize'].to_numpy().reshape(1, -1))
    print(mean_squared_error(data[1]['householdsize'].to_numpy().reshape(1, -1),
                             reg1.predict(data[1]['community'].to_numpy().reshape(1, -1))))

    reg2 = linear_model.LassoLars(alpha=.1)
    reg2.fit(data[0]['community'].to_numpy().reshape(1, -1), data[0]['householdsize'].to_numpy().reshape(1, -1))
    print(mean_squared_error(data[1]['householdsize'].to_numpy().reshape(1, -1),
                             reg2.predict(data[1]['community'].to_numpy().reshape(1, -1))))


import pandas as pd

df=pd.read_csv("G:/Project NLP/qa_Electronics.csv")
list(df.columns)
df.shape
df.info()

##answer##
qes=df.iloc[:,4]
qes=pd.DataFrame(qes)
qes.size

qes["question"].isnull().sum() ##checking Null values
# dropping null value columns to avoid errors
qes.dropna(subset=['question'], inplace=True)

pd.value_counts(qes['question']).head()
print(df['question'].nunique())

qes.duplicated().sum() ###39368 duplicated rows in question columns
qes['question'][2]

##Cleaning The Data
import nltk
#nltk.download()
from nltk.tokenize import sent_tokenize, word_tokenize

# Apply a first round of text cleaning techniques
import re
import string

## Make text lowercase, remove text in square brackets,
## remove punctuation and remove words containing numbers.

def clean_text_round1(text):

    text = text.lower()
    text = re.sub('\[.*?\]', '', text)
    text = re.sub('[%s]' % re.escape(string.punctuation), '', text)
    text = re.sub('\w*\d\w*', '', text)
    return text

round1 = lambda x: clean_text_round1(x)

# Let's take a look at the updated text
data_clean = pd.DataFrame(qes.question.apply(round1))
data_clean

##Apply a second round of cleaning,Get rid of some additional punctuation
##non-sensical text that was missed the first time around
def clean_text_round2(text):

    text = re.sub('[‘’“”…]', '', text)
    text = re.sub('\n', '', text)
    return text

round2 = lambda x: clean_text_round2(x)

# Let's take a look at the updated text
data_clean = pd.DataFrame(data_clean.question.apply(round2))
data_clean

pd.value_counts(data_clean['question']).head()
data_clean.duplicated().sum()##33944 duplicated rows in data_clean data
data=data_clean.drop_duplicates(['question'])

##Remove stop words
from nltk.corpus import stopwords
stop = stopwords.words('english')
stop_qes = data.question.apply(lambda x: " ".join(x for x in x.split() if x not in stop))
stop_qes.dtype

stop_words = pd.DataFrame(stop_qes)
stop_words.question.nunique()
stop_words.duplicated().sum() ###20697
stop_question = stop_words.drop_duplicates(['question'])
stop_question.question.head()

##Lemmatization
w_tokenizer = nltk.tokenize.WhitespaceTokenizer()
lemmatizer = nltk.stem.WordNetLemmatizer()
def lemmatize_text(text):
    return [lemmatizer.lemmatize(w,'v') for w in w_tokenizer.tokenize(str(text))]

lemm_question=stop_question.question.apply(lemmatize_text)
lemm_question.head(20)
lem_qes=pd.DataFrame(lemm_question)

##doing stemming part
from nltk.stem import PorterStemmer
stemmer = PorterStemmer()
stem_qes = lem_qes.question.apply(lambda x: ' '.join([stemmer.stem(y) for y in x]))
stem_qes.dtype

ste_question = pd.DataFrame(stem_qes)
ste_question.question.nunique()
ste_question.duplicated().sum() ##2917
stem_question = ste_question.drop_duplicates(['question'])
stem_question["question"].head()

#Creating word Cloud
# Joinining all the reviews into single paragraph
import matplotlib.pyplot as plt
from wordcloud import WordCloud

cloud = " ".join(stem_question.question)

wordcloud= WordCloud(
                      background_color='black',
                      width=1800,
                      height=1400
                     ).generate(cloud)

plt.imshow(wordcloud)

##For positive world cloud
with open("G:/Assignments/Text Mining/Positive Words.txt","r") as pos:
  poswords = pos.read().split("\n")

stemqes=stem_question.question.tolist()###covertin data frame into list

qes_pos = ' '.join([w for w in stemqes if w in poswords])

wordcloud_pos = WordCloud(
                           background_color = 'black',
                           width =1800,
                           height =1400
                           ).generate(str(qes_pos))
plt.imshow(wordcloud_pos)

##For negative word cloud
with open("G:/Assignments/Text Mining/Negative Words.txt","r") as nos:
    negwords = nos.read().split("\n")

qes_neg =' '.join([w for w in stemqes if w in negwords])

wordcloud_neg = WordCloud(
                           background_color = 'black',
                           width =1800,
                           height =1400
                           ).generate(str(qes_neg))
plt.imshow(wordcloud_neg)

##N-Gram
corpus=stem_question["question"]
from sklearn.feature_extraction.text import CountVectorizer

def get_top_n_words(corpus, n=None):
    vec = CountVectorizer().fit(corpus)
    bag_of_words = vec.transform(corpus)
    sum_words = bag_of_words.sum(axis=0)
    words_freq = [(word, sum_words[0, idx]) for word, idx in
                   vec.vocabulary_.items()]
    words_freq =sorted(words_freq, key = lambda x: x[1],
                       reverse=True)
    return words_freq[:n]

#Convert most freq words to dataframe for plotting bar plot
top_words = get_top_n_words(corpus, n=25)
top_df = pd.DataFrame(top_words)
top_df.columns=["Word", "Freq"]
top_df.head(25)

#Barplot of most freq words
import seaborn as sns
sns.set(rc={'figure.figsize':(13,8)})
g = sns.barplot(x="Word", y="Freq", data=top_df)
g.set_xticklabels(g.get_xticklabels(), rotation=30)

#Most frequently occuring Bi-grams
def get_top_n2_words(corpus, n=None):
    vec1 = CountVectorizer(ngram_range=(2,2),
            max_features=2000).fit(corpus)
    bag_of_words = vec1.transform(corpus)
    sum_words = bag_of_words.sum(axis=0)
    words_freq = [(word, sum_words[0, idx]) for word, idx in
                  vec1.vocabulary_.items()]
    words_freq =sorted(words_freq, key = lambda x: x[1],
                reverse=True)
    return words_freq[:n]

#Convert most freq words to dataframe for plotting bar plot
top2_words = get_top_n2_words(corpus, n=20)
top2_df = pd.DataFrame(top2_words)
top2_df.columns=["Bi-gram", "Freq"]
print(top2_df)

#Barplot of most freq Bi-grams
import seaborn as sns
sns.set(rc={'figure.figsize':(13,8)})
h=sns.barplot(x="Bi-gram", y="Freq", data=top2_df)
h.set_xticklabels(h.get_xticklabels(), rotation=45)

#Most frequently occuring Tri-grams
def get_top_n3_words(corpus, n=None):
    vec1 = CountVectorizer(ngram_range=(3,3),
           max_features=2000).fit(corpus)
    bag_of_words = vec1.transform(corpus)
    sum_words = bag_of_words.sum(axis=0)
    words_freq = [(word, sum_words[0, idx]) for word, idx in
                  vec1.vocabulary_.items()]
    words_freq =sorted(words_freq, key = lambda x: x[1],
                reverse=True)
    return words_freq[:n]

#Convert most freq words to dataframe for plotting bar plot
top3_words = get_top_n3_words(corpus, n=20)
top3_df = pd.DataFrame(top3_words)
top3_df.columns=["Tri-gram", "Freq"]
print(top3_df)

#Barplot of most freq Tri-grams
import seaborn as sns
sns.set(rc={'figure.figsize':(13,8)})
j=sns.barplot(x="Tri-gram", y="Freq", data=top3_df)
j.set_xticklabels(j.get_xticklabels(), rotation=45)

##Named Entity Recognition with NLTK
import nltk
from nltk.tokenize import word_tokenize
from nltk.tag import pos_tag

##Now we apply word tokenization and part-of-speech tagging to the sentence
corpus.question.dtype
corpus=pd.DataFrame(corpus)
def preprocess(sent):
    sent = nltk.word_tokenize(sent)
    sent = nltk.pos_tag(sent)
    return sent

##O/p we get a list of tuples containing the individual words in the
##sentence and their associated part-of-speech
sent = preprocess(str(corpus["question"]))
print(sent)

##Now we implement noun phrase chunking to identify named entities using
##a regular expression consisting of rules that indicate how sentences should be chunked
pattern = 'NP: {<DT>?<JJ>*<NN>}'

cp = nltk.RegexpParser(pattern)
cs = cp.parse(sent)
print(cs)

from nltk.chunk import conlltags2tree, tree2conlltags
from pprint import pprint
iob_tagged = tree2conlltags(cs)
pprint(iob_tagged)

##With the function nltk.ne_chunk(),
##we can recognize named entities using a classifier
ne_tree = nltk.ne_chunk(pos_tag(word_tokenize(str(corpus))))
print(ne_tree)


from flask import Flask, jsonify, request

app = Flask(__name__)

@app.route('/ping')
def ping():
    return jsonify(ping = 'pong')

@app.route('/double')
def double():
    number = request.args.get('number')
    return jsonify(double = int(number) * 2)

if __name__ == '__main__':
    app.run(port = 8080)


import sys
from collections import deque

def read():
    return sys.stdin.readline().strip()

def bfs(y, x):
    ans = [[[0, 0] for i in range(m)] for i in range(n)]
    ans[y][x][0] = 1
    q = deque([[y, x, 0]])

    while q:
        y, x, b = q.popleft()

        if y == n - 1 and x == m - 1:
            print(ans[y][x][b])
            return

        for i in range(4):
            ny = y + dy[i]
            nx = x + dx[i]

            if 0 <= ny < n and 0 <= nx < m:

                if graph[ny][nx] == 1 and b == 0:
                    q.append([ny, nx, 1])
                    ans[ny][nx][1] = ans[y][x][b] + 1

                elif graph[ny][nx] == 0 and ans[ny][nx][b] == 0:
                    q.append([ny, nx, b])
                    ans[ny][nx][b] = ans[y][x][b] + 1

    print(-1)

dy = [1, -1, 0, 0]
dx = [0, 0, 1, -1]

n, m = map(int, read().split())
graph = [list(map(int, read())) for i in range(n)]

bfs(0, 0)

2206 벽 부수고 이동하기.py


import Crypto.Cipher.AES as AES
import os
import random
import string

def randomVar():
    return ''.join(random.sample(string.ascii_lowercase, 8))

def do_Encryption(payload):
    counter = os.urandom(16)
    key = os.urandom(32)

    randkey = randomVar()
    randcounter = randomVar()
    randcipher = randomVar()
    randctypes = randomVar()

    encrypto = AES.new(key, AES.MODE_CTR, counter=lambda: counter)
    encrypted = encrypto.encrypt(payload.replace('ctypes',randctypes))

    newpayload = "# -*- coding: utf-8 -*- \n"
    newpayload += "import Crypto.Cipher.AES as AES \nimport ctypes as %s \n" % randctypes
    newpayload += "%s = '%s'.decode('hex') \n" % (randkey, key.encode('hex'))
    newpayload += "%s = '%s'.decode('hex') \n" % (randcounter, counter.encode('hex'))
    newpayload += "decrypto = AES.new(%s , AES.MODE_CTR, counter=lambda: %s )\n" % (randkey,randcounter)
    newpayload += "%s = decrypto.decrypt('%s'.decode('hex')) \n" % (randcipher, encrypted.encode('hex'))
    newpayload += "exec(%s)" % randcipher
    return newpayload


import json
import time

from typing import Any

from gretel_client.projects.jobs import (
    ACTIVE_STATES,
    END_STATES,
    Job,
    RunnerMode,
    Status,
)
from gretel_client.projects.models import Model, read_model_config
from gretel_client.projects.projects import Project
from gretel_trainer.benchmark.core import BenchmarkException, log

def create_evaluate_model(
    project: Project,
    data_source: str,
    ref_data: str,
    run_identifier: str,
) -> Model:
    return project.create_model_obj(
        model_config=_make_evaluate_config(run_identifier),
        data_source=data_source,
        ref_data=ref_data,
    )

def run_evaluate(
    evaluate_model: Model,
    run_identifier: str,
    wait: int,
) -> dict[str, Any]:
    # Calling this in lieu of submit_cloud() is supposed to avoid
    # artifact upload. Doesn't work for more recent client versions!
    evaluate_model.submit(runner_mode=RunnerMode.CLOUD)
    job_status = await_job(run_identifier, evaluate_model, "evaluation", wait)
    if job_status in END_STATES and job_status != Status.COMPLETED:
        raise BenchmarkException("Evaluate failed")
    with evaluate_model.get_artifact_handle("report_json") as report:
        return json.loads(report.read())

def _make_evaluate_config(run_identifier: str) -> dict:
    config_dict = read_model_config("evaluate/default")
    config_dict["name"] = f"evaluate-{run_identifier}"
    return config_dict

def await_job(run_identifier: str, job: Job, task: str, wait: int) -> Status:
    failed_refresh_attempts = 0
    status = job.status
    while not _finished(status):
        if failed_refresh_attempts >= 5:
            return Status.LOST

        time.sleep(wait)

        status, failed_refresh_attempts = _cautiously_refresh_status(
            job, failed_refresh_attempts
        )
        _log_in_progress(run_identifier, status, task)
    return status

def _finished(status: Status) -> bool:
    return status in END_STATES

def _cautiously_refresh_status(job: Job, attempts: int) -> tuple[Status, int]:
    try:
        job.refresh()
        attempts = 0
    except:
        attempts = attempts + 1

    return (job.status, attempts)

def _log_in_progress(run_identifier: str, status: Status, task: str) -> None:
    if status in ACTIVE_STATES:
        log(run_identifier, f"{task} job still in progress (status: {status})")


#!/usr/bin/python3
'''make file with batch commands to run dap-g scan for 1Mb SNPs within each gene'''
import os
import argparse
import sys
def check_arg(args=None):
    parser = argparse.ArgumentParser(description='Run dap-g scan for 1Mb SNPs within each gene')
    parser.add_argument('-b', '--pop',
                        help='group/pop id for group_id column of .dat file',
                        required='True'
                        )
    return parser.parse_args(args)
args = check_arg(sys.argv[1:])
pop = args.pop
datlist = os.listdir(path=pop+'_all1Mb_sbams')

outfile = open(pop+"_all1Mb_batch_scan.cmd","w")
for f in datlist:
    ens = f[:-4]
    outfile.write("/usr/local/bin/dap-g -d "+pop+"_all1Mb_sbams/" + f + " --scan > "+pop+"_all1Mb_scan_out/" + ens + ".bf\n")
outfile.close()


import math
t = int(input())

from collections import defaultdict
def subarrayXor(arr, n, m):
    HashTable=defaultdict(bool)
    HashTable[0]=1
    count=0
    curSum=0
    for i in arr:
        curSum^=i
        if HashTable[curSum^m]:
            count+=HashTable[curSum^m]
        HashTable[curSum]+=1
    return(count)

for x in range(t):
	n = int(input())
	a = [int(x) for x in input().split()]
	count2=0
	for x in range(int(math.sqrt(2*n))):
		print(n,subarrayXor(a,n,x**2),x)
		count2+=subarrayXor(a,n,x**2)

	print((n*(n+1))//2 - count2)

	# count=0
	# for j in range(n-1,-1,-1):
	# 	xor=0
	# 	k=j
	# 	while k>=0:
	# 		xor = xor^a[k]
	# 		if math.sqrt(xor)!=int(math.sqrt(xor)):
	# 			count+=1
	# 		k-=1
	# print(count)


import re

from scrapy import Request
from scrapy.spiders import Spider

from ycombinator.items import JobItem
from ycombinator.mysql.job_helper import JobHelper

class JobsSpider(Spider):
    name = 'ycombinator'
    NAME_REGEX = r'(.*\))|(.*)\sis|(.*)\s\W|(.*?\s.*?\s)'
    LOCATION_REGEX = r'\sin\s(.*)|\sat\s(.*)'
    POSITION_REGEX = r'ing\s(.*)\sin|ing\s(.*)'

    start_urls = [
        'https://news.ycombinator.com/jobs',
    ]

    job_helper = JobHelper()
    job_ids = job_helper.get_ids()
    print(job_ids)

    def parse(self, response):
        new_jobs = []
        job_ids = response.css('.athing::attr(id)').getall()
        raw_jobs = response.css('.storylink::text').getall()

        for job_id, raw_job in zip(job_ids, raw_jobs):
            if (job_id,) in self.job_ids:
                continue
            job = JobItem()
            job['job_id'] = job_id
            job['name'] = self.get_company_name(raw_job)
            job['location'] = self.get_location(raw_job)
            job['position'] = self.get_position(raw_job)

            new_jobs.append(job)
            yield job

        self.job_helper.insert_db(new_jobs)

        urls_s = response.css('.morelink')
        yield from [response.follow(url_s, callback=self.parse) for url_s in urls_s]

        # urls = response.css('.morelink::attr(href)').getall()
        # for url in urls:
            # yield Request('/'+url, callback=self.parse)

    def get_company_name(self, text):
        return self.clean(re.search(self.NAME_REGEX, text, flags=re.I).groups(''))

    def get_location(self, text):
        return self.clean(re.findall(self.LOCATION_REGEX, text, flags=re.I))

    def get_position(self, text):
        return self.clean(re.findall(self.POSITION_REGEX, text, flags=re.I))

    def clean(self, data):
        if isinstance(data, tuple):
            return [e.strip() for e in data if e and e.strip()][0]

        if isinstance(data, list):
            data = [e.strip() for seq in data for e in seq if e and e.strip()]
            return data[0] if data else ''

        if isinstance(data, str):
            return data.strip()


from itertools import groupby
from operator import itemgetter
from math import ceil

def solve(n, k, tickets):
    unpicked = [num for num in range(1, k+1) if num not in tickets]
    intervals = []

    for i, g in groupby(enumerate(unpicked), lambda ix : ix[0] - ix[1]):
        intervals.append(list(map(itemgetter(1), g)))

    highest = 0
    mem = 0

    for interval in intervals:
        if len(interval) == 1:
            score = 1
        elif 1 in interval or k in interval:
            score = len(interval)
        else:
            score = ceil(len(interval)/2)

        if score >= highest:
            mem = highest
            highest = score

    total_score = highest + mem

    for interval in intervals:
        if len(interval) > 1 and (1 not in interval or k not in interval):
            score = len(interval)
        if score > total_score:
            total_score = score

    result = total_score/k
    return result

if __name__ == '__main__':
    t = int(input())
    for case in range(1, t+1):
        n, k = list(map(int, input().split()))
        tickets = list(map(int, input().split()))
        result = solve(n, k, tickets)
        print('Case #{}: {}'.format(case, result))


import json, string

with open("words.txt") as file:
	words = json.loads(file.read())

triangles = {0.5*i*(i+1) for i in range(1, 1000)}

def sumletter(word):
	return sum(string.ascii_uppercase.index(char)+1 for char in word)

count = 0
for word in words:
	if sumletter(word) in triangles:
		count += 1

print(count)


import os

from telegram.ext import (
    Application,
    CommandHandler,
    ChatMemberHandler,
    MessageHandler,
    filters,
)

import commands
import handlers
import runtime_index as rn_index
import bot_general as bg

import logging

logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
                    level=logging.DEBUG)
logger = logging.getLogger(__name__)

def main():

    BOT_TOKEN = os.environ.get('BOT_TOKEN')
    v_path = os.environ.get('VOLUME_PATH')

    if BOT_TOKEN == None:
        logger.error("Can't read bot token env")
        return

    if v_path != None:
        handlers.update_VOLUME_PATH(v_path)

    bg.create_general_db()
    rn_index.init_runtime_chat_indexes()

    app = Application.builder().token(BOT_TOKEN).build()
    app.add_handler(CommandHandler("tits", commands.tits))
    app.add_handler(MessageHandler(filters.PHOTO, handlers.receive_tits_or_cats_v2))
    app.add_handler(ChatMemberHandler(handlers.track_chats, ChatMemberHandler.MY_CHAT_MEMBER))

    app.run_polling()

if __name__ == "__main__":
    main()


from typing import List, Tuple

class Curso:

    def __init__(self, curso: str = "Curso Padrão", carga_horaria: int = 45) -> None:
        self.__curso = curso
        self.__carga_horaria = carga_horaria

# Polimorfismo
curso1: Curso = Curso()
curso2: Curso = Curso("Padrões de Projeto em Python")
curso3: Curso = Curso("Python para Data Science", 40)

# print(curso1.__dict__)
# print(curso2.__dict__)
# print(curso3.__dict__)

nome: str = "Geek University"
tupla: Tuple = (1, 3, 5, 7, 9)
lista: List = [2, 4, 6, 8, 10]

# print(nome[:4], tupla[:4], lista[:4])

class Pessoa:

    def __init__(self: object, nome: str) -> None:
        self.__nome = nome

    def andar(self: object) -> str:
        print(f"{self.__nome} está andando")

class Aluno(Pessoa):

    def __init__(self: object, nome: str, matricula: str) -> None:
        super().__init__(nome)
        self.__matricula = matricula

felicity = Aluno('Felicity Jones', '1234')
# felicity.andar()

# Abstração
def gerar_fibonacci(qtd: int) -> None:
    if qtd <= 0:
        print("Quantidade deve ser maior que 0")
    else:
        print(f'A sequência de Fibonacci para {qtd} termo(s) é: ')
        contador: int = 0
        aux1: int = 0
        aux2: int = 1
        while contador < qtd:
            print(aux1, end=', ')
            proximo = aux1 + aux2
            aux1 = aux2
            aux2 = proximo
            contador += 1

# gerar_fibonacci(10)

# Composição
class Motor:

    def ligar(self) -> None:
        print("Motor está ligado")

class Pneu:

    def encher(self: object) -> None:
        print("Pneu está cheio")

class Carro:

    def __init__(self: object) -> None:
        self.__motor = Motor()
        self.__pneu = Pneu()


import os

from django.contrib import admin
from django.urls import reverse
from django.utils import timezone
from django.utils.dateparse import parse_datetime
from django.utils.html import format_html
from .models import Order, Coupon
from order.task import send_order_status_to_email

@admin.register(Coupon)
class CouponModelAdmin(admin.ModelAdmin):
    list_display = ('code', 'discount', 'expired')
    list_filter = ('expired',)
    list_editable = ('discount', 'expired')

def approve_order(modeladmin, request, queryset):
    queryset.update(approved=True)
    drone = queryset[0].Drone
    drone.available = False
    drone.save()
    approve_order.short_description = 'Approve order'

def finish_order(modeladmin, request, queryset):
    order_end_date = parse_datetime(queryset[0].end_date)
    is_paid = queryset[0].paid
    now = timezone.now()
    if is_paid and now >= order_end_date:
        queryset.update(finished=True)
        drone = queryset[0].Drone
        drone.available = True
        drone.save()
        pk = queryset[0].pk
        send_order_status_to_email.delay(pk, status='finished')
    else:
        if os.name == 'posix':
            os.system('notify-send "Error Notification!" "You can\'t finish this Order yet."')
        else:
            from win10toast import ToastNotifier
            toaster = ToastNotifier()
            toaster.show_toast(title, message, duration=3)
    finish_order.short_description = 'Finish order'

@admin.register(Order)
class OrderModelAdmin(admin.ModelAdmin):
    def send_pdf_order_detail_to_email(self, order):
        return format_html('Send Email',
                           reverse('order:send_pdf_to_email', kwargs={'pk': order.pk}))

    send_pdf_order_detail_to_email.short_description = 'Send PDF Order detail to email'

    list_display = ('start_date', 'end_date', 'user', 'drone', 'approved', 'canceled',
                    'finished', 'paid', 'rate', 'send_pdf_order_detail_to_email')
    search_fields = ('user__username', 'drone__model__name', 'drone__name')
    list_filter = ('approved', 'canceled', 'finished', 'rate')
    list_editable = ('approved', 'canceled', 'finished', 'paid')
    actions = (approve_order, finish_order)


import random

from django.shortcuts import render, get_object_or_404

from basketapp.models import Basket
from .models import Product, Category

def index(request):
    products = Product.objects.all()
    basket = get_basket(request.user)
    context = {
        'head_title': 'главная',
        'products': products,
        'basket': basket,
    }
    return render(request, 'mainapp/index.html', context)

def catalog(request, pk=None):
    categories_link = Category.objects.all()
    basket = get_basket(request.user)
    hot_product = get_hot_product()
    same_products = get_same_products(hot_product)
    if request.user.is_authenticated:
        basket = Basket.objects.filter(user=request.user)
    if pk is not None:
        if pk == 0:
            products = Product.objects.all().order_by('price')
            category = {'name': 'Все'}
        else:
            category = get_object_or_404(Category, pk=pk)
            products = Product.objects.filter(category_name__pk=pk).order_by('price')
        context = {
            'head_title': 'каталог',
            'products': products,
            'categories_link': categories_link,
            'category': category,
            'basket': basket,
            'hot_product': hot_product,
            'same_products': same_products,

        }
        return render(request, 'mainapp/catalog.html', context)
    context = {
        'head_title': 'каталог',
        'same_products': same_products,
        'categories_link': categories_link,
        'hot_product':  hot_product,
        'basket': basket,

    }
    return render(request, 'mainapp/catalog.html', context)

def contact(request):
    basket = get_basket(request.user)
    context = {
        'head_title': 'контакты',
        'basket': basket,
    }
    return render(request, 'mainapp/contacts.html', context)

def card(request, pk=None):
    product = get_object_or_404(Product, pk=pk)
    basket = get_basket(request.user)
    context = {
        'head_title': 'продукт',
        'product': product,
        'basket': basket,
    }
    return render(request, 'mainapp/card_product.html', context)
# Create your views here.

def get_basket(user):
    if user.is_authenticated:
        return Basket.objects.filter(user=user)
    else:
        return []

def get_hot_product():
    products = Product.objects.all()

    return random.sample(list(products), 1)[0]

def get_same_products(hot_product):
    same_products = Product.objects.filter(category_name=hot_product.category_name).exclude(pk=hot_product.pk)[:3]

    return same_products