# Bu araç @keyiflerolsun tarafından | @KekikAkademi için yazılmıştır.

from Kolektif     import app, cache
from flask        import render_template, jsonify
from KekikSpatula import Doviz

@app.route("/dovizGorsel")
@cache.cached(timeout=60)
def doviz_gorsel():
    doviz = Doviz()

    return render_template(
        "veriSayfasi.html",
        veriler    = doviz.veri["veri"],
        anahtarlar = doviz.anahtarlar,
        baslik     = f"«{doviz.veri['kaynak']}» Güncel Döviz Verileri"
    )

@app.route("/doviz")
@cache.cached(timeout=60)
def doviz_json():
    doviz = Doviz()

    return jsonify(
        kaynak    = doviz.veri["kaynak"],
        saglayici = "@keyiflerolsun",
        veri      = doviz.veri["veri"]
    )


# -*- coding: utf-8 -*-
"""
Created on Thu May 18 09:28:06 2017

@author: Trail1
"""

import os, os.path
from PIL import Image
from os import walk
from datetime import datetime
import pandas as pd
import numpy as np

rgbcode_dict = { (99, 214, 104, 255) : "G",
                 (255, 151, 77, 255) : "O",
                 (242, 60, 50, 255)  : "R",
                 (129, 31, 31, 255)  : "D",
                 (255, 255, 255, 255): "W"}

def str2tuple(tuple_str):
    r, g, b, alpha = tuple_str.replace("(","").replace(")","").split(",")
    return tuple([int(r), int(g), int(b), int(alpha)])
#rgb_df1 = pd.read_csv("rgb_traffic.csv")
#rgb_df1["rgb_code"] = rgb_df1["RGB"].apply(lambda x : str2tuple(x))

png_file_dir = r"""./screenshots_cropped_DC"""

png_flielist = []
for root, dirs, files in walk(png_file_dir):
    for file in files:
        if file.endswith('.png'):
            png_flielist.append(file)

png_flielist = png_flielist[:]

rgb_collection = []
unique_rgb_collection = []
for i, png in enumerate(png_flielist[:]):
    png_datetime_str = png.split(".")[0].split("_")[1]+"_"+png.split(".")[0].split("_")[2]
    png_datetime = datetime.strptime(png_datetime_str, "%Y%m%d_%H%M%S")
    print(png, i+1, "/", len(png_flielist))
    img1 = Image.open(os.path.join(png_file_dir,r"""DC_{0}.png""".format(png_datetime_str)))

    newimdata = []
    for color in img1.getdata():
        if color in [(99, 214, 104, 255), (255, 151, 77, 255), (242, 60, 50, 255), (129, 31, 31, 255)]:
            # G, O, R, D
            # D: (145, 59, 59, 255) - (R, G, B, alpha)
            #    (137, 45, 45, 255)
            #
        #if color in [(132, 202, 80, 255), (222, 241, 208, 255), (165, 216, 127, 255)]:
            #print("XXX")
            newimdata.append(color)
        else:
            newimdata.append((255, 255, 255, 255))

    img2 = Image.new(img1.mode,img1.size)
    img2.putdata(newimdata)
    #img2.save(r"""test_ATL_{0}.png""".format(png_datetime_str))

    df1 = pd.DataFrame({"rgb" : newimdata})
    df1["DROG"] = df1["rgb"].apply(lambda x : rgbcode_dict[x])
    df1["datetime"] = png_datetime_str
    df2 = pd.pivot_table(df1, values="rgb", index="datetime", columns="DROG", aggfunc="count").reset_index()
    rgb_collection.append(df2)

df3 = pd.concat(rgb_collection)

df3.to_csv("DC_screenshots_cropped_DROG_Counts.csv")


"""
"""
from dotenv import load_dotenv
from boxsdk import OAuth2, Client
from os import environ

load_dotenv()

# The OAuth takes an access token but the UI calls it a developer token
auth = OAuth2(
    client_id=environ.get('CLIENT_ID'),
    client_secret=environ.get('CLIENT_SECRET'),
    access_token=environ.get('DEVELOPER_TOKEN'),
)
client = Client(auth)

user = client.user().get()
print(f'The current user ID is {user.id}')


import sys
from io import StringIO
import unittest

class TestClass(unittest.TestCase):
    maxDiff = None
    def assertIO(self, input, output):
        stdout, stdin = sys.stdout, sys.stdin
        sys.stdout, sys.stdin = StringIO(), StringIO(input)
        resolve()
        sys.stdout.seek(0)
        out = sys.stdout.read()[:-1]
        sys.stdout, sys.stdin = stdout, stdin
        self.assertEqual(out, output)

    def test_Sample_Input_1(self):
        input = """5
10 3 5 2 3 6
10 3 5 1 1000000000 1000000000
139 2 139 1 1 1
139 1 1 1 1 1
139 7 10 3845 26982 30923"""
        output = """11
10
1
139
436604"""
        self.assertIO(input, output)

def resolve():
  inf = 10**18+1
  T = int(input())
  for _ in range(T):
    # 1 を使いたくないパターンがあるっぽい。
    # 1 のコストがとても高いとか。
    N,A,B,X,Y,Z = map(int, input().split(" "))
    P = 0

    # コスパが悪いならば A, B は使わない。
    if A*X <= Y: A, Y = 1, X
    if B*X <= Z: B, Z = 1, X
    if Z == X and B == 1 and Y == X and A == 1:
      print(N*X)
      continue
    if Z == X and B == 1:
      print((N//A)*Y + (N%A)*X)
      continue
    if Y == X and A == 1:
      print((N//B)*Z + (N%B)*X)
      continue
    print(N,A,B,X,Y,Z)
    # できるだけ 1 を使わない方針で考えよう
    # 1

import sys
if sys.argv[-1] == './Main.py':
  resolve()

if __name__ == "__main__":
  unittest.main()


import datetime
from dataclasses import dataclass, field
from decimal import Decimal
from typing import List, Dict

from domain.common.investments import StockDividend
from domain.common.portfolio_item import PortfolioItem

@dataclass
class CashDividendPosition:
    date: datetime.date
    total: Decimal = Decimal(0)
    stocks: Dict = field(default_factory=dict)

    def __post_init__(self):
        if isinstance(self.date, str):
            self.date = datetime.datetime.strptime(self.date, "%Y%m%d").date()

    def add_dividend(self, dividend: StockDividend):
        self.total += dividend.amount
        if dividend.ticker in self.stocks:
            self.stocks[dividend.ticker] += dividend.amount
        else:
            self.stocks[dividend.ticker] = dividend.amount
        if self.stocks[dividend.ticker] <= 0:
            self.stocks.pop(dividend.ticker)

    def to_dict(self):
        return {
            **self.__dict__,
            "date": self.date.strftime("%Y%m%d")
        }

@dataclass
class CashDividendsSummary(PortfolioItem):
    history: List[CashDividendPosition] = field(default_factory=list)
    _history_map: Dict[datetime.date, CashDividendPosition] = field(init=False, repr=False)

    def __post_init__(self):
        if all(isinstance(h, dict) for h in self.history):
            self.history = [CashDividendPosition(**c) for c in self.history]
        self._history_map = {c.date: c for c in self.history}

    def add_dividend(self, dividend: StockDividend):
        month_start = dividend.date.replace(day=1)
        position = self.get_position_for_date(month_start)
        position.add_dividend(dividend)

    def get_position_for_date(self, date: datetime.date):
        if date in self._history_map:
            return self._history_map[date]
        return self.create_position(date)

    def create_position(self, date: datetime.date):
        position = CashDividendPosition(date)
        self.history.append(position)
        self._history_map[date] = position
        return position

    @property
    def sk(self) -> str:
        return "CASH_DIVIDENDS#"

    def to_json(self):
        return {
            **super().to_json(),
            "history": [h.to_dict() for h in self.history]
        }


#!/usr/bin/env python
# coding: utf-8

# # KNN Algorithm

# In[1]:

#imports
from sklearn.datasets import load_iris
import pandas as pd
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
import numpy as np
from scipy.spatial import distance
from scipy import stats
from sklearn.metrics import accuracy_score, recall_score, precision_score, f1_score,cohen_kappa_score

# In[2]:

iris = load_iris()
features = pd.DataFrame(iris.data,columns=iris.feature_names)
target = pd.DataFrame(iris.target,columns=['target'])
df = pd.concat([features,target],axis=1)
print(df)

# In[3]:

print(iris.DESCR)

# In[4]:

x = iris.data
y = iris.target
xtrain, xtest, ytrain, ytest = train_test_split(x,y,test_size=0.5,random_state=0)

# In[5]:

corrs = []
for x in features:
    corrs.append(abs(df['target'].corr(df[x])))
plt.figure(figsize=(8,5))
plt.xticks(np.arange(len(corrs)),features)
plt.ylabel('correlation')
plt.xlabel('features')
plt.bar(np.arange(len(corrs)),corrs, width=0.4)
plt.show()

# In[15]:

#KNN function for one new data point
def knn(x_train,y_train,x_test,k=5):
    score=[]
    for loop in zip(x_train,y_train):
        score.append([distance.euclidean(x_test,loop[0]),loop[1]])
    score.sort(key = lambda x : x[0])
    score = np.array(score)
    return int(stats.mode(score[:k,1])[0])

# In[16]:

#Implimenting KNN function for whole testing list
pred = []
for x in xtest:
    pred.append(knn(xtrain,ytrain,x,k=5))
pred = np.array(pred)

# In[18]:

#Scores
print('accuracy: ',accuracy_score(ytest,pred))
print('precision: ',precision_score(ytest,pred,average='micro'))
print('recall: ',recall_score(ytest,pred,average='micro'))
print('f1: ',f1_score(ytest,pred,average='micro'))
print('cohen kappa: ',cohen_kappa_score(ytest,pred))

# In[19]:

vals  = np.array([accuracy_score(ytest,pred),precision_score(ytest,pred,average='micro'),recall_score(ytest,pred,average='micro'),f1_score(ytest,pred,average='micro'),cohen_kappa_score(ytest,pred)])*100
plt.figure(figsize=(8,5))
plt.xticks(np.arange(5),['Accuracy','Precision','Recall','F1','Cohen Kappa'])
plt.ylabel('correlation')
plt.xlabel('features')
plt.bar(np.arange(5),vals, width=0.4)
plt.show()


from splinter import Browser
from bs4 import BeautifulSoup
import pandas as pd
import datetime as dt
import pprint

def scrape_all():

    # Initiate headless driver for deployment
    browser = Browser("chrome", executable_path="chromedriver", headless=True)
    news_title, news_paragraph = mars_news(browser)

    # Run all scraping functions and store results in dictionary
    data = {
        "news_title": news_title,
        "news_paragraph": news_paragraph,
        "featured_image": featured_image(browser),
        "facts": mars_facts(),
        "Mars Hemispheres": mars_hemispheres(browser),
        "last_modified": dt.datetime.now()
        }

    # Quit the browser & retrieve the data gathered:
    browser.quit()
    return data

def mars_news(browser):
    # Visit the mars nasa news site
    url = 'https://mars.nasa.gov/news/'
    browser.visit(url)

    # Optional delay for loading the page
    browser.is_element_present_by_css("ul.item_list li.slide", wait_time=1)

    # Set up the html parser
    html = browser.html
    news_soup = BeautifulSoup(html, 'html.parser')

    # Add try/except for error handling
    try:
        slide_elem = news_soup.select_one('ul.item_list li.slide')
        # Use parent element to find the first 'a' tag and save it as 'news_title'
        news_title = slide_elem.find("div", class_="content_title").get_text()
        # Use parent element to find the paragraph text
        news_p = slide_elem.find("div", class_="article_teaser_body").get_text()
        # Begin Scraping
        slide_elem.find('div', class_='content_title')
        return news_title, news_p
    except AttributeError:
        return None, None

# '### Featured Images'
def featured_image(browser):
    # Visit URL
    url = 'https://www.jpl.nasa.gov/spaceimages/?search=&catagory=Mars'
    browser.visit(url)

    # Find and click the full image button using Splinter
    full_image_elem = browser.find_by_id('full_image')
    full_image_elem.click()
    browser.is_element_present_by_text('more info', wait_time=1)

    # Find the more info button and click that using Splinter
    more_info_elem = browser.links.find_by_partial_text('more info')
    more_info_elem.click()

    # Parse
    html = browser.html
    img_soup = BeautifulSoup(html, 'html.parser')

    try:
        # Find the relative image url
        img_url_rel = img_soup.select_one('figure.lede a img').get('src')
    except AttributeError:
        return None

    # Use the base URL to create an absolute URL
    img_url = f'https://www.jpl.nasa.gov{img_url_rel}'
    img_url

    return img_url

def mars_facts():
    # Setting up code with DataFrame
    try:
        # use 'read_html' to scrape the facts table into a dataframe
        df = pd.read_html('http://space-facts.com/mars/')[0]
    except BaseException:
        return None
    # Assign columns and set index of dataframe
    df.columns=['description','value']
    df.set_index('description', inplace=True)
    # Convert dataframe into HTML format, add bootstrap
    return df.to_html()

# Challenge Work - first define hemispheres variable
def mars_hemispheres(browser):
    # Visit URL
    url = 'https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars'
    browser.visit(url)

    # Creating an empty dictinary in order to put information in
    hemisphere_final = []

    # Finding the pictures by tag function
    browser.is_element_present_by_css("thumb", wait_time=1)
    pictures = browser.find_by_tag('h3')

    # Loop the four pictures in order for them to render
    for x in range(0,4):

        # click each picture on the astrogeology.usgs.gov webpage
        pictures[x].click()

        # Parse
        html = browser.html
        hemisphere_beautifulsoup = BeautifulSoup(html, 'html.parser')

        # Finding image url
        img_url_rel = hemisphere_beautifulsoup.select_one('.wide-image').get('src')
        hemisphere_title = hemisphere_beautifulsoup.find('h2', class_='title').get_text()
        img_url = f'https://astrogeology.usgs.gov{img_url_rel}'

        # Now deliver the info into the new dictionary
        hemisphere_dict = {}
        hemisphere_dict['title'] = hemisphere_title
        hemisphere_dict['img_url'] = img_url
        hemisphere_final.append(hemisphere_dict)

        browser.back()

        browser.is_element_present_by_css("thumb", wait_time=1)
        pictures = browser.find_by_tag('h3')

    # Return the completed list with all 4 hemispheres side by side on website
    return hemisphere_final

if __name__ == "__main__":

    print(scrape_all())


from flask import Flask, render_template, request, redirect, url_for
import os

app = Flask(__name__)

@app.route("/")
def get_index():
    return render_template("index.html")

@app.route("/result")
def result():
    this_x = request.args['x']
    this_y = request.args['y']
    button_value = request.args['action']

    if button_value == "Add":
        return redirect(url_for("add", x=this_x, y=this_y))

    if button_value == "Mult":
        return redirect(url_for("mult", x=this_x, y=this_y))

@app.route("/add/<int:x>/<int:y>")
def add(x, y):
    s = x + y
    return "{0} + {1} = {2}".format(x, y, s)

@app.route("/mult/<int:x>/<int:y>")
def mult(x, y):
    s = x * y
    return "{0} * {1} = {2}".format(x, y, s)

if __name__ == "__main__":
   app.run(host=os.getenv('IP', '0.0.0.0'), port=int(os.getenv('PORT', 8080)), debug=True)


import pickle

import gridfs
import pymongo
from bson import ObjectId

from db.exceptions import DataDoesNotExist
from seq_labeling.pgm import *
from diffusion.enum import Method
from settings import logger, MONGO_URL

class DBManager:
    def __init__(self, db_name):
        self.client = pymongo.MongoClient(MONGO_URL)
        self.db = self.client[db_name]

class EvidenceManager(DBManager):
    def __init__(self, project):
        self.project = project
        db_name = self.get_db_name(project)
        super().__init__(db_name)
        self.train_sets_col = self.client['train_sets'][f'{project.db}_{project.name}']

    def get_db_name(self, project):
        return f'{project.db}_evid_{project.name}'

    def get_one(self, user_id, train_set):
        set_id = self.__get_train_set_id(train_set)
        if set_id is None:
            raise DataDoesNotExist('No evidences exist for training set given')
        if not isinstance(user_id, ObjectId):
            user_id = ObjectId(user_id)
        fs = gridfs.GridFS(self.db)
        doc = fs.find_one({'set_id': set_id, 'user_id': user_id})
        if doc is None:
            raise ValueError(f'No evidence exists for user id {user_id}')
        return self._str_to_sequences(doc.read())

    def __find_by_set_id(self, set_id):
        fs = gridfs.GridFS(self.db)
        return fs.find({'set_id': set_id}, no_cursor_timeout=True)

    def __get_train_set_id(self, train_set):
        set_id = None
        for doc in self.train_sets_col.find():
            if set(doc['train_set']) == set(train_set):
                set_id = doc['_id']
        return set_id

    def get_many(self, train_set):
        """
        Return dictionary of user id's to the lists of the sequences. Each sequence is a list of (obs, state) tuples.
        :param user_ids:
        :return:
        """
        set_id = self.__get_train_set_id(train_set)
        if set_id is None:
            raise DataDoesNotExist('No evidences exist for training set given')

        documents = self.__find_by_set_id(set_id)
        dic = {
            doc.user_id: self._str_to_sequences(doc.read()) for doc in documents
        }
        if dic:
            return dic
        else:
            raise DataDoesNotExist(f'No evidences exist for training set given')

    def get_many_generator(self, train_set):
        """
        Get the generator of (user_id, sequences) tuples which each "sequences" is a list of (obs, state) tuples.
        :param user_ids:
        :return:
        """
        set_id = self.__get_train_set_id(train_set)
        if set_id is None:
            raise DataDoesNotExist('No evidences exist for training set given')

        documents = self.__find_by_set_id(set_id)
        if next(iter(documents.clone())):
            for doc in documents:
                yield doc['user_id'], self._str_to_sequences(doc.read())
        else:
            raise DataDoesNotExist('No evidences exist for training set given')

    def insert(self, evidences, train_set):
        """
        :param evidences: dictionary of user id's to the sequences.
        :return:
        """
        set_id = self.__get_train_set_id(train_set)
        if set_id is None:
            set_id = self.train_sets_col.insert_one({'train_set': train_set}).inserted_id

        fs = gridfs.GridFS(self.db)
        logger.info('inserting %d evidence documents ...', len(evidences))
        i = 0
        for uid in evidences:
            fs.put(bytes(self._sequences_to_str(evidences[uid]), encoding='utf8'), user_id=uid, set_id=set_id)
            i += 1
            if i % 10000 == 0:
                logger.info('%d documents inserted', i)

    def _sequences_to_str(self, sequences):
        return str([[(obs.astype(int).tolist(), state) for obs, state in seq] for seq in sequences])

    def _str_to_sequences(self, seq_str):
        return [[(np.array(obs, dtype=bool), state) for obs, state in seq] for seq in eval(seq_str)]

    def create_index(self):
        """
        Create index on 'set_id' key of the collection if it does not exist.
        :return:
        """
        collection = self.db.get_collection('fs.files')
        if len(collection.index_information()) < 2:
            collection.create_index('set_id')

class ParentSensEvidManager(EvidenceManager):
    def get_db_name(self, project):
        return f'{project.db}_parent_evid_{project.name}'

class SeqLabelDBManager:
    def __init__(self, project, method):
        self.project = project
        self.client = pymongo.MongoClient(MONGO_URL)
        self.db_name = f'{project.db}_{method.value}_{project.name}'
        self.db = self.client[self.db_name]
        self.method = method

    def db_exists(self):
        db_names = self.client.list_database_names()
        return self.db_name in db_names

    def insert(self, models):
        fs = gridfs.GridFS(self.db)

        logger.debug('inserting %d model documents ...', len(models))
        i = 0
        for uid in models:
            doc = self._get_doc(models[uid])
            fs.put(bytes(str(doc), encoding='utf8'), user_id=uid)
            i += 1
            if i % 10000 == 0:
                logger.debug('%d documents inserted', i)

    def fetch_all(self):
        fs = gridfs.GridFS(self.db)
        models = {}
        i = 0
        for doc in fs.find():
            memm = self._doc_to_model(doc)
            models[doc.user_id] = memm
            i += 1
            if i % 10000 == 0:
                logger.debug('%d models fetched', i)
        return models

    def fetch_one(self, user_id):
        if not isinstance(user_id, ObjectId):
            user_id = ObjectId(user_id)
        fs = gridfs.GridFS(self.db)
        doc = fs.find_one({'user_id': user_id})
        if doc is None:
            return None
        model = self._doc_to_model(doc)
        return model

    def _get_doc(self, model):
        doc = {
            'orig_indexes': model.orig_indexes,
            'lambda': model.Lambda.tolist(),
        }
        if hasattr(model, 'td_param'):
            doc['td_param'] = model.td_param
        return doc

    def _doc_to_model(self, doc):
        data = doc.read()
        model_data = eval(data)
        if self.method in [Method.LONG_MEMM, Method.MULTI_STATE_LONG_MEMM]:
            model = LongMEMM()
        elif self.method in [Method.BIN_MEMM, Method.MULTI_STATE_BIN_MEMM]:
            model = BinMEMM()
        elif self.method == Method.PARENT_SENS_TD_MEMM:
            model = ParentTDMEMM()
        elif self.method == Method.LONG_PARENT_SENS_TD_MEMM:
            model = LongParentTDMEMM()
        else:
            model = TDMEMM()
        orig_indexes = model_data['orig_indexes']
        Lambda = np.fromiter(model_data['lambda'], np.float64)
        model.set_params(Lambda, orig_indexes)
        if 'td_param' in model_data:
            model.td_param = model_data['td_param']
        return model

class CRFManager(SeqLabelDBManager):
    def _get_doc(self, crf):
        return {
            'orig_indexes': crf.orig_indexes,
            'model_filename': crf.model_filename
        }

    def _doc_to_model(self, doc):
        data = eval(doc.read())
        crf = self._get_model_instance()
        crf.set_params(data['orig_indexes'], data['model_filename'])
        return crf

    def _get_model_instance(self):
        if self.method in [Method.LONG_CRF, Method.MULTI_STATE_LONG_CRF]:
            return CRF()
        elif self.method in [Method.BIN_CRF, Method.MULTI_STATE_BIN_CRF]:
            return BinCRF()
        elif self.method in [Method.TD_CRF, Method.MULTI_STATE_TD_CRF]:
            return TDCRF()
        else:
            raise ValueError(f"Invalid method {self.method.value} for CRF manager")


from dependencies.dependency import aq_inner
from dependencies.dependency import aq_parent
from dependencies.dependency import getToolByName

def upgrade(tool):
    """ Refactor ARs listing to allow sorting by priority
    """
    # Hack prevent out-of-date upgrading
    # Related: PR #1484
    # https://github.com/bikalabs/Bika-LIMS/pull/1484
    from lims.upgrade import skip_pre315
    if skip_pre315(aq_parent(aq_inner(tool))):
        return True

    def addIndex(cat, *args):
        try:
            cat.addIndex(*args)
        except:
            pass

    portal = aq_parent(aq_inner(tool))
    # Create new indexes
    bc = getToolByName(portal, 'bika_catalog')
    addIndex(bc, 'Priority', 'FieldIndex')
    addIndex(bc, 'BatchUID', 'FieldIndex')
    bc.manage_reindexIndex(ids=['Priority', 'BatchUID',])

    bac = getToolByName(portal, 'bika_analysis_catalog')
    addIndex(bac, 'Priority', 'FieldIndex')
    bac.manage_reindexIndex(ids=['Priority',])

    return True


# -*- coding: utf-8 -*-
import re

import scrapy

from zhipin.items import ZhipinItem

class BossZhipinSpider(scrapy.Spider):
    name = 'boss_zhipin'
    allowed_domains = ['https://www.zhipin.com']
    url = 'https://www.zhipin.com/c101020100/h_101020100/?query=python&page=%s'
    offset = 1
    start_urls = [url % offset]

    # https: // www.zhipin.com / c101020100 / h_101020100 /?query = python & page = 1
    # https: // www.zhipin.com / c101020100 / h_101020100 /?query = python & page = 10

    def parse(self, response):
        item = ZhipinItem()
        for response_part in response.css('#main > div > div.job-list > ul').extract():
            company_size = []
            company_info = re.findall('(.*?)
', response_part)
            # 算出列表长度，拿到偶数位的数据
            # for idx in range(1, len(company_info)+1, 2):
            for idx, val in enumerate(company_info):
                if int(idx) % 2 != 0:
                    if '' in company_info[idx]:
                        new_item = company_info[idx].rsplit('')[-1]
                        company_size.append(new_item)
                    else:
                        # 没带直接加到company_size中
                        company_size.append(company_info[idx])
            result = zip(re.findall('title">(.*?)
', response_part),
                         re.findall('(.*?)', response_part),
                         re.findall('ka="search_list_company_\d+_custompage" target="_blank">(.*?)', response_part),
                         company_size,
                         re.findall('发布于(.*?)
', response_part))
            for job_item in result:
                """
                处理元组数据，返回item
                """
                item['job_title'] = job_item[0]
                item['job_salary'] = job_item[1]
                item['job_company'] = job_item[2]
                item['company_size'] = job_item[3]
                item['publish_date'] = job_item[4]
                yield item

        if self.offset < 10:
            self.offset += 1
            yield scrapy.Request(self.url % self.offset, callback=self.parse, dont_filter=True)


# -*- coding: utf-8 -*-
#
# YT documentation build configuration file, created by
# sphinx-quickstart on Tue Jan 10 11:33:37 2017.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import yt.wrapper

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#
# needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    "sphinx.ext.autodoc",
    "sphinx.ext.todo",
    "sphinx.ext.viewcode"
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ["_templates"]

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
#
# source_suffix = ['.rst', '.md']
source_suffix = ".rst"

# The master toctree document.
master_doc = "index"

# General information about the project.
project = u"YTsaurus"
copyright = u"2023, YTsaurus Team"
author = u"YTsaurus Team"

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = yt.wrapper.__version__
# The full version, including alpha/beta/rc tags.
release = yt.wrapper.__version__

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = "en"

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This patterns also effect to html_static_path and html_extra_path
exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = "sphinx"

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
#
html_theme = "alabaster"

html_sidebars = {
    "**": [
        "about.html",
        "navigation.html",
        "searchbox.html"
    ]
}

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#
html_theme_options = {
    "description": "Python and CLI interfaces for the system",
    "extra_nav_links": {
        "YTsaurus site": "https://ytsaurus.tech",
        "YTsaurus docs": "https://ytsaurus.tech/docs/en/",
        "Python docs": "https://ytsaurus.tech/docs/en/api/python/start",
        "YSON docs": "https://ytsaurus.tech/docs/en/user-guide/storage/yson"
    }
}

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ["_static"]

# -- Options for HTMLHelp output ------------------------------------------

# Output file base name for HTML help builder.
htmlhelp_basename = "YTdoc"

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
    # The paper size ('letterpaper' or 'a4paper').
    #
    # 'papersize': 'letterpaper',

    # The font size ('10pt', '11pt' or '12pt').
    #
    # 'pointsize': '10pt',

    # Additional stuff for the LaTeX preamble.
    #
    # 'preamble': '',

    # Latex figure (float) alignment
    #
    # 'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
    (master_doc, "YT.tex", u"YT Documentation",
     u"YT Team", "manual"),
]

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, "yt", u"YT Documentation",
     [author], 1)
]

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    (master_doc, "YT", u"YT Documentation",
     author, "YT", "Python and CLI interfaces for the system.",
     "Miscellaneous"),
]

# -- Options for Epub output ----------------------------------------------

# Bibliographic Dublin Core info.
epub_title = project
epub_author = author
epub_publisher = author
epub_copyright = copyright

# The unique identifier of the text. This can be a ISBN number
# or the project homepage.
#
# epub_identifier = ''

# A unique identification for the text.
#
# epub_uid = ''

# A list of files that should not be packed into the epub file.
epub_exclude_files = ["search.html"]

autoclass_content = "both"


from PIL import ImageGrab, ImageOps
import time
import math
from copy import deepcopy
import keyboard
#import mouse #i want to move scren, just comment if no like
import numpy as np
bboxthing = [1186,242,1426,722] #please put the bbox coords here k thnx
clicks = 1
held = ["null"]
def GetState(info):
    board = []

    ScreenShot = ImageOps.grayscale(ImageGrab.grab(bbox=(bboxthing[0],bboxthing[1],bboxthing[2],bboxthing[3])))# old is 306,173,546,653
    #ScreenShot.save("test.png")#not needed unless testing
    pixels = ScreenShot.getdata()
    for y in range(12,480,24):
        row = []
        for x in range(12,240,24):

            pixel = np.array(pixels[x + (y * 240)])
            if pixel in [131,92,120,1611,79,122,71,61,161]:
                row.append("0")
            else:
                row.append("#")
            if pixel in [59,46,35,39,60,80,65]:

                if pixel == 65:
                    info.append([1,[[0, 1], [1, 0], [1, 1], [2, 0]]])
                elif pixel == 39:
                    info.append([2,[[0, 0], [1, 0], [1, 1], [2, 1]]])
                elif pixel == 46:
                    info.append([3,[[0, 1], [1, 0], [1, 1], [2, 1]]])
                elif pixel == 80:
                    info.append([4,[[0, 0], [0, 1], [1, 0], [1, 1]]])
                elif pixel == 59:
                    info.append([5,[[0, 0], [1, 0], [2, 0], [3, 0]]])
                elif pixel == 60:
                    info.append([6,[[0, 1], [1, 1], [2, 0], [2, 1]]])
                elif pixel == 35:
                    info.append([7,[[0, 0], [0, 1], [1, 1], [2, 1]]])
                y2 = (y-12) // 24
                x2 = (x-12) // 24
                while True:
                    y2 -=1
                    if y2 < 0:
                        break
                    if board[y2][x2] == "0":
                        board[y2][x2] = "#"
                        break
        board.append(row)

    return np.array(board)
def drop(poss,board):
    while True:
        for num in poss:
            if num[1] < 19:
                if board[num[1] + 1][num[0]] == "0":
                    break
            else:
                break
        else:
            for num in poss:
                num[1] += 1
            continue
        break
    return poss

def tryAll(board,poss,AllPossibles,moves,rotate,typ,types):
    possibles = []

    while poss[-1][0] <= 9:
        move = [rotate,poss[0][0]]
        temp = drop(deepcopy(poss),board)
        if not(temp in AllPossibles or temp in possibles):
            possibles.append(temp)
            types.append(typ)
            moves.append(move)
        for num in poss:
            num[0] += 1
    return possibles
def moveTopLeft(pos):
    pos = sorted(pos,key = lambda y : y[1])
    while pos[0][1] > 0 or pos[0][1] < 0:
        value = 0
        if pos[0][1] > 0:
            value = -1
        else:
            value = 1
        for num in pos:

            num[1] += value
    pos = sorted(pos,key = lambda x : x[0])
    while pos[0][0] > 0 or pos[0][0] < 0:
        value = 0
        if pos[0][0] > 0:
            value = -1
        else:
            value = 1
        for num in pos:
            num[0] += value

def move(board,coords,boards,moves,typ):

    theMove = []
    for y in range(len(boards)):

        if (boards[y] == board).all():
            theMove = moves[y]

    for num in range(theMove[0]):
        keyboard.press("up arrow")
        keyboard.release("up arrow")
        #time.sleep(0.1)
    for x in range(9):
        keyboard.press("left arrow")
        keyboard.release("left arrow")

    absolute = theMove[1]

    while absolute != 0:
        keyboard.press("right arrow")
        keyboard.release("right arrow")
        #time.sleep(0.3)
        absolute -=1
    keyboard.press(" ")
    keyboard.release(" ")

def Grader(board,coordNew,moves,typ,types): #coords are kinda useful
    scores = []
    pointModif = [
        100000, #no holes
        150, #can score
        40, #max blocks covered
        10, #max blocks covered (self blocks)
        10000, #least distance from bottom

    ]

    #phase 1 : any air pockets that'll be made if placed here?

    for i in range(len(coordNew)):
        scores.append(0)
        currentBoard = board[i]
        good = True
        for e in range(len(coordNew[i])):
            the = coordNew[i][e]
            if (the[0]<19): #19 is floor
                if (currentBoard[the[0]+1][the[1]] == "#"):
                    good = False
                    break
        if (good):
            scores[i] += pointModif[0]

    #phase 2 of testing : can it score points? (normally you'd want to stack high for big points but screw that)

    for i in range(len(board)):
        currentBoard = board[i]
        for e in range(len(coordNew[i])):
            if (not "#" in currentBoard[coordNew[i][e][0]]):
                scores[i] += pointModif[1]

    #phase 3 of testing : how many blocks would be covered by them (maximize this!)

    for i in range(len(coordNew)):
        currentBoard = board[i]
        covered = 0
        for e in range(len(coordNew[i])):
            the = coordNew[i][e]
            if (the[0]<19): #19 is floor
                deez = currentBoard[the[0]+1][the[1]]
                if (not deez == "#"):
                    if (deez == "N"):
                        scores[i] += pointModif[3]
                    else:
                        scores[i] += pointModif[2]
            else:
                scores[i] += pointModif[2]

    for i in range(len(coordNew)):
        for e in range(len(coordNew[i])):
            scores[i] += pointModif[4] * coordNew[i][e][0]
    topI = 0
    topScore = 0
    for i in range(len(scores)):
        if (scores[i]>topScore):
            topScore = scores[i]
            topI = i
    global held
    if types[topI] != typ:

        keyboard.press("c")
        keyboard.release("c")
        move(board[topI],coordNew[topI],board,moves,held[0])
        held[0] = held[2]
        held[1] = held[3]
        return

    move(board[topI],coordNew[topI],board,moves,typ) #i dont see a situation where it'll crash due to no placement being available
def DoTheSpin(board,pos,AllPossible,moves,typ,types):
    for rotate in range(4):
        AllPossible += tryAll(board,deepcopy(pos),AllPossible,moves,rotate,typ,types)
        if typ in [1,2,3,6,7]:
            offset = [1.5,1.5]
        elif typ == 5:
            offset = [2.5,1]
        elif typ == 4:
            break
        for num in pos:
            num[0] -= offset[0]
            num[1] -= offset[1]
            temp = num[0]
            num[0] = math.floor(-1 * num[1] + offset[0])
            num[1] =  temp
            num[1] = math.floor(offset[1] + num[1])
        pos = sorted(pos,key = lambda x : x[0])
        moveTopLeft(pos)
        pos = sorted(pos,key = lambda x : x[0])

def yes():
    info = []
    board = GetState(info)
    pos = []
    for y in range( len(board) - 1):
        for x in range( len(board[y]) - 1):
            if board[y][x] == "C":
                pos.append([x,y])
    pos = info[0][1]

    type = info[0][0]#1 green z, 2 red z,3 T,4 square,5 line,6 orange L,7 blue L
    global held
    if held[0] == "null":
        held[0] = type
        held.append(pos)
        held.append(type)
        held.append(pos)
        keyboard.press("c")
        keyboard.release("c")
        return
    held[2] = type
    held[3] = pos
    AllPossible = []
    moves = []
    types = []
    DoTheSpin(board,pos,AllPossible,moves,type,types)
    DoTheSpin(board,held[1],AllPossible,moves,held[0],types)

    allBoards = []
    allNewCoords = []
    for y in board:
        for x in y:
            if x == "C":
                x = "#"
    for possible in AllPossible:
        tempB = deepcopy(board)
        yes = []
        for nums in possible:
            tempB[nums[1]][nums[0]] = "N"
            yes.append([nums[1],nums[0]])
        allBoards.append(tempB)
        allNewCoords.append(yes)

    Grader(allBoards,allNewCoords,moves,type,types)
def main():
    time.sleep(1)
    print("go")
    while True:
        yes()
        #time.sleep(0.2) #comment for SPEEE

noClick = """ #uncomment if no like
def L(): #ey yo sorry for doing this but yes
    global clicks #this is a bad idea
    global bboxthing
    print(clicks)

    if (clicks==1):
        lol = mouse.get_position()
        bboxthing = [lol[0],lol[1],lol[0]+240,lol[1]+480]
        print(bboxthing)
    elif(clicks==0):
        main()
        print("done")
    clicks = clicks - 1

mouse.on_click(lambda : L())
""" #uncomment this aswell

main() #i do click so uh yes uncomment if you want
#print("done")


#!/usr/bin/env artisan
from artisan.core import *
from artisan.rose import *

import sys, os, json, subprocess, time

sys.path.insert(1, '/workspace/metaprograms/sdt/')
sys.path.insert(2, '/workspace/metaprograms/daa/')
sys.path.insert(3, '/workspace/metaprograms/udt/')

log.level = 2

from udt_gen_swi_fpga_project import gen_swi_fpga_project
from daa_check_utilisation import check_utilisation
from sdt_unr import unroll_loop
from sdt_generate_opencl_kernel import generate_opencl_kernel

# from sdt_generate_hls_kernel import generate_hls_kernel

## UDT to optimise for an unrolled single work item FPGA kernel

artisan_artifacts = '~/Documents/Artisan/artisan-artefacts/'

# TODO: check if loop has fixed bounds
def fixed_bounds(loop):
    condition = loop.cond().unparse()
    if '<' in condition:
        bound = condition.split('<')[-1].strip()[:-1]
        try:
            int(bound)
            return int(bound)
        except:
            return False
    return False

def unroll_nonfixed(ast, UF):
    # find all loops without fixed bounds
    loops = ast.project.query('l:ForLoop')
    done = []
    for row in loops:
        if row.l.in_code() and fixed_bounds(row.l) == False and row.l.tag() not in done:
            print(row.l.tag())
            # unroll_loop(ast, row.l.tag(), unroll_factor=UF)
        done.append(row.l.tag())

def unroll_fixed(ast, UF):
    # find all loops with fixed bounds
    loops = ast.project.query('l:ForLoop')
    done = []
    for row in loops:
        if row.l.in_code() and fixed_bounds(row.l) != False and row.l.tag() not in done:
            print(row.l.tag())
            unroll_loop(ast, row.l.tag(), unroll_factor=UF)
        done.append(row.l.tag())

def generate_reports():
    # make library, first stage compilation of opencl kernel
    command = ['ssh', '-t', '172.17.0.1']
    command += ['cd', artisan_artifacts + os.getcwd().replace('workspace', '') + '/swi_project/project/', ';']
    command += ['source', '~/setup181.sh', '>', '/dev/null', '2>&1', ';']
    command += ['source', 'generate_report.sh']
    subprocess.call(command)

def rollback(UF, ast, headers, fixed):
    print("Unrolling by %d overmapped or made no change, rolling back to %d" % (UF, int(UF/2)))
    if fixed:
        unroll_fixed(ast, int(UF/2))
    else:
        unroll_nonfixed(ast, int(UF/2))
    ast.commit()
    generate_opencl_kernel(ast, 'swi_project/project/device/', headers)

## BEGIN MAIN ##
if not os.path.exists("./swi_project"):
    gen_swi_fpga_project(cli())

# new ast for kernel
path_to_kernel = os.getcwd() + '/swi_project/cpp_kernel.cpp'
ast = model(args='"' + path_to_kernel + '"', ws=Workspace('kernel_ws'))

headers = []
header_pragmas = [row for row in ast.project.query("g:Global => p:Pragma") if 'artisan-hls' in row.p.directive() and 'header' in row.p.directive()]
for row in header_pragmas:
    directive = row.p.directive()
    headers.append(json.loads(' '.join(directive.split()[directive.split().index('header') + 1:])))

# DSE ON LOOP UNROLL FACTOR
UF = 2
overmapped = False
prev_percentages = []
while not overmapped:
    print("Trying to unroll fixed loops by factor:", UF, "...")

    # instrument code to unroll fixed bound loops
    unroll_fixed(ast, UF)
    ast.commit()
    generate_opencl_kernel(ast, 'swi_project/project/device/', headers)
    ast.undo(sync=True) ## OTHERWISE #include statements are removed
    ast.commit()

    # run first stage opencl compile to generate design reports
    generate_reports()
    # parse reports to check utilisation estimates
    try:
        utilisation = check_utilisation('./swi_project/project/bin/kernel/reports')
    except:
        rollback(UF, ast, headers, True)
        overmapped = True
        break

    percentages = [utilisation[u]['percentage'] for u in utilisation]
    for u in utilisation:
        print(u, utilisation[u]['percentage'], '%')
    # if no change, or if any resources are at > 100%, go back to previous unroll factor, finish
    if max(percentages) > 90 or percentages == prev_percentages:
        rollback(UF, ast, headers, True)
        if percentages != prev_percentages:
            overmapped = True
        break
    else:  # if all resources are < 100%, increase unroll factor
        UF = UF * 2

    prev_percentages = percentages

# if unrolling fixed loops didnt overmap, check others
UF = 2
while not overmapped:
    print("Trying to unroll NON - fixed loops by factor:", UF, "...")

    # instrument code to unroll fixed bound loops
    unroll_nonfixed(ast, UF)
    ast.commit()
    generate_opencl_kernel(ast, 'swi_project/project/device/', headers)
    ast.undo(sync=True)
    ast.commit()

    # run first stage opencl compile to generate design reports
    generate_reports()
    # parse reports to check utilisation estimates
    try:
        utilisation = check_utilisation('./swi_project/project/bin/kernel/reports')
    except:
        rollback(UF, ast, headers, False)
        overmapped = True
        break

    percentages = [utilisation[u]['percentage'] for u in utilisation]
    for u in utilisation:
        print(u, utilisation[u]['percentage'], '%')
    # if no change, or if any resources are at > 100%, go back to previous unroll factor, finish
    if max(percentages) > 90 or percentages == prev_percentages:
        rollback(UF, ast, headers, False)
        if percentages != prev_percentages:
            overmapped = True
        break
    else:  # if all resources are < 100%, increase unroll factor
        UF = UF * 2

    prev_percentages = percentages

subprocess.call(['rm', '-rf', 'kernel_ws'])


import os
from gensim.models import FastText
from gensim.models import Word2Vec

# loading FastText model
current_directory = os.getcwd()
model_file = current_directory + "/fastText/fastText-models/so-java-big"
model = FastText.load(model_file)
word = "java"
vector = model.wv[word.strip()]
print(vector)
print("CONGRATS! FastText model is working properly!")


from collections import deque
from sys import stdin

input = stdin.readline

def bfs():
    queue = deque([[0, 0, 1]])
    visit = [[[0] * 2 for _ in range(M)] for _ in range(N)]
    visit[0][0][1] = 1

    while queue:
        # x: row, y: column, w[0]: count_crashed, w[1]: count_no_crashed
        x, y, w = queue.popleft()
        if x == N - 1 and y == M - 1:
            return visit[x][y][w]
        for i in range(4):
            nx = x + dx[i]
            ny = y + dy[i]
            if 0 <= nx < N and 0 <= ny < M:
                if matrix[nx][ny] == 1 and w == 1:  # 벽을 만나고 벽을 한번 부술 수 있는 경우
                    visit[nx][ny][0] = visit[x][y][w] + 1
                    queue.append([nx, ny, 0])
                elif matrix[nx][ny] == 0 and visit[nx][ny][w] == 0:  # 벽이 없고 방문한적이 없는 경우
                    visit[nx][ny][w] = visit[x][y][w] + 1
                    queue.append([nx, ny, w])
    return -1

N, M = map(int, input().split())
matrix = [list(map(int, input().strip())) for _ in range(N)]

dx = (1, -1, 0, 0)
dy = (0, 0, 1, -1)

print(bfs())

2206.벽 부수고 이동하기.py


import pandas as pd
from sklearn.ensemble import BaggingRegressor
from sklearn.ensemble import BaggingClassifier
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestRegressor

train = pd.read_csv('train.csv')
test = pd.read_csv('test.csv')
feature_list = open('use_these_vars.txt', 'rb')
features = feature_list.read().splitlines()

def write_function(preds, fname):
    with open(fname, 'wb') as writer:
        for item in preds:
            writer.write("%s\n" % item )

def second_pos_clip(ls):
    out = [x[1] for x in ls]
    return(out)

log = LogisticRegression(solver = 'sag')
lm = LinearRegression()

write_function(test['y'], '/tmp/truths.txt')

print('optimizing samples')
for n_samp in [0.1, 0.25, 0.33, 0.5, 0.75, 1.0]:
    for n_feat in [0.1, 0.25, 0.33, 0.5, 0.75, 1.0]:

        lm_bagged = BaggingRegressor(
          base_estimator = lm,
          n_estimators = 75,
          max_samples = n_samp,
          max_features = n_feat,
          bootstrap = True,
          oob_score = False,
          warm_start = False,
          n_jobs = -1
        )

        log_bagged = BaggingClassifier(
          base_estimator = log,
          n_estimators = 75,
          max_samples = n_samp,
          max_features = n_feat,
          bootstrap = True,
          oob_score = False,
          warm_start = False,
          n_jobs = -1
        )

        lm_bagged.fit(X = train[features], y = train['y'])
        log_bagged.fit(X = train[features], y = train['y'])
        lm_bagged_preds = lm_bagged.predict(X = test[features])
        log_bagged_preds = log_bagged.predict_proba(X = test[features])

        write_function(lm_bagged_preds, '/tmp/lm_bagged_preds_nsamp-%s_nfeat-%s.txt' % (n_samp, n_feat))
        write_function(second_pos_clip(log_bagged_preds), '/tmp/log_bagged_preds_nsamp-%s_nfeat-%s.txt' % (n_samp, n_feat))


from uuid import uuid4
import time
#
class Worker(object):
    def __init__(self,**kws):
        self.workerId     = kws.get("workerId",str(uuid4))
        self.port      = kws.get("port")
        self.balls     = kws.get("balls", [])
        self.isStarted = kws.get("isStarted",False)
        self.createdAt = kws.get("createdAt",time.time())


import matplotlib.pyplot as plt
import numpy as np
import xlrd as xd
from sklearn.preprocessing import StandardScaler
from sklearn.svm import LinearSVC

def LoadData(trainpath, testpath):
    file_train_path = trainpath
    file_train_xlsx = xd.open_workbook(file_train_path)
    file_train_sheet = file_train_xlsx.sheet_by_name('Sheet1')
    x_train = []
    y_train = []
    for row in range(file_train_sheet.nrows):
        x_data = []
        for col in range(file_train_sheet.ncols):
            if col < file_train_sheet.ncols - 1:
                x_data.append(file_train_sheet.cell_value(row, col))
            else:
                if file_train_sheet.cell_value(row, col) == 'tested_negative':
                    y_train.append(0)
                else:
                    y_train.append(1)
        x_train.append(list(x_data))

    file_test_path = testpath
    file_test_xlsx = xd.open_workbook(file_test_path)
    file_test_sheet = file_test_xlsx.sheet_by_name('Sheet1')
    x_test = []
    y_test = []
    for row in range(file_test_sheet.nrows):
        x_data = []
        for col in range(file_test_sheet.ncols):
            if col < file_test_sheet.ncols - 1:
                x_data.append(file_test_sheet.cell_value(row, col))
            else:
                if file_test_sheet.cell_value(row, col) == 'tested_negative':
                    y_test.append(0)
                else:
                    y_test.append(1)

        x_test.append(list(x_data))

    # print(x_train)
    # print(y_train)
    # print(x_test)
    # print(y_test)
    return x_train, y_train, x_test, y_test

# 数据集路径
train_path = 'data/diabetes_train.xlsx'
test_path = 'data/diabetes_test.xlsx'
# 加载数据
x_train, y_train, x_test, y_test = LoadData(train_path, test_path)
# 分别初始化对特征值和目标值的标准化器
ss_x = StandardScaler()
ss_y = StandardScaler()
# 训练数据都是数值型，所以要标准化处理
x_train = ss_x.fit_transform(x_train)
x_test = ss_x.fit_transform(x_test)
# 目标数据也是数值型，所以也要标准化处理
y_train = ss_y.fit_transform(np.array(y_train).reshape(-1, 1))
y_test = ss_y.fit_transform(np.array(y_test).reshape(-1, 1))
# 使用LinearSVC分类训练
linear_svc = LinearSVC()
linear_svc.fit(x_train, y_train.astype('int'))
linear_svc_predict = linear_svc.predict(x_test)
# 绘图
l1, = plt.plot(y_test, color='b', linewidth=2)
l2, = plt.plot(linear_svc_predict, color='r', linewidth=2)
plt.legend([l1, l2], ['y_test', 'linear_svc_predict'], loc=2)
plt.savefig('支持向量机回归预测.jpg')
plt.show()
# 性能评估
print('The Accuracy of Linear SVC is', linear_svc.score(x_test, y_test.astype(np.int64)))


# -*- coding: utf-8 -*-
"""
Created on Fri Feb 7 2020

@author: Prof Gates modified by Jaci W
"""

import nltk
import pandas as pd
import sklearn
from sklearn.cluster import KMeans
import numpy as np
from sklearn.feature_extraction.text import CountVectorizer
from nltk.tokenize import word_tokenize
from nltk.probability import FreqDist
import matplotlib.pyplot as plt
from nltk.corpus import stopwords
## For Stemming
from nltk.stem import PorterStemmer
from nltk.tokenize import sent_tokenize, word_tokenize
import os

biasdata = pd.read_csv("C:\\Users\\tessa\\OneDrive\\School\\Project Portfolio\\2 Text Mining Course IST 736\\0 Data Sets\\clean_data.csv")
biasdata.head()

class LemmaTokenizer(object):
    def __init__(self):
        self.wnl = WordNetLemmatizer()
    def __call__(self, articles):
        return [self.wnl.lemmatize(t) for t in word_tokenize(articles) if t not in stopwords.words('english') or string.punctuation]

class LemmaTokenizer(object):
    def __call__(self, text):
        return [lemma(t) for t in word_tokenize(text) if t not in stopwords.words('english')]

## Build the vectorizer
pattern='r/^[a-zA-Z]{4}$/'
pattern="[^r\P{P}]+"

MyVect5=CountVectorizer(biasdata,
                        analyzer = 'word',
                        stop_words='english',
                           #token_pattern='(?u)[a-zA-Z]+',
                        #token_pattern=pattern,
                        #tokenizer=LemmaTokenizer(),
                        #strip_accents = 'unicode',
                        lowercase = True
                        )

MyVect5B=CountVectorizer(biasdata,
                        analyzer = 'word',
                        stop_words='english',
                          #token_pattern='(?u)[a-zA-Z]+',
                        #token_pattern=pattern,
                        #tokenizer=LemmaTokenizer(),
                        #strip_accents = 'unicode',
                        binary=True
                        )
FinalDF=pd.DataFrame()
FinalDFB=pd.DataFrame()
MyVect5
MyVect5B

#####################################################################################################
## Replace the NaN with 0 because it actually
## means none in this case
FinalDF=biasdata.fillna(0)
X5=MyVect5.fit_transform(FinalDF)
ColumnNames2=MyVect5.get_feature_names()
print("Column names: ", ColumnNames2)
FinalDFB=biasdata.fillna(0)
X5B=MyVect5B.fit_transform(FinalDFB)
print("FIRST...Normal DF Freq")  ## These print statements help you to see where you are
print(FinalDF)
print("BINARY DF....")
print(FinalDFB)

##############################################################################################################

## Create the testing set - grab a sample from the training set.
## Be careful. Notice that right now, our train set is sorted by label.
## If your train set is large enough, you can take a random sample.
from sklearn.model_selection import train_test_split

TrainDF, TestDF = train_test_split(FinalDF, test_size=0.3)

##-----------------------------------------------------------------
##
## Now we have a training set and a testing set.
print("The training set is:")
print(TrainDF)
print("The testing set is:")
print(TestDF)

## IMPORTANT - YOU CANNOT LEAVE LABELS ON THE TEST SET
## Save labels
TestLabels=TestDF["Label"]
print(TestLabels)
## remove labels
TestDF = TestDF.drop(["Label"], axis=1)
print(TestDF)

####################################################################
########################### Naive Bayes ############################
####################################################################
from sklearn.naive_bayes import MultinomialNB
#https://scikit-learn.org/stable/modules/generated/sklearn.naive_bayes.MultinomialNB.html#sklearn.naive_bayes.MultinomialNB.fit
#Create the modeler
MyModelNB= MultinomialNB()
## When you look up this model, you learn that it wants the
## DF seperate from the labels
TrainDF_nolabels=TrainDF.drop(["Label"], axis=1)
print(TrainDF_nolabels)
TrainLabels=TrainDF["Label"]
print(TrainLabels)

MyModelNB.fit(TrainDF_nolabels, TrainLabels)

Prediction = MyModelNB.predict(TestDF)
print("The prediction from NB is:")
print(Prediction)
print("The actual labels are:")
print(TestLabels)
## confusion matrix
from sklearn.metrics import confusion_matrix
## The confusion matrix is square and is labels X labels
## We ahve two labels, so ours will be 2X2
#The matrix shows
## rows are the true labels
## columns are predicted
## it is al[habetical
## The numbers are how many
cnf_matrix = confusion_matrix(TestLabels, Prediction)
print("The confusion matrix is:")
print(cnf_matrix)
### prediction probabilities
## columns are the labels in alphabetical order
## The decinal in the matrix are the prob of being
## that label
print(np.round(MyModelNB.predict_proba(TestDF),2))

#######################################################
### Bernoulli #########################################
#######################################################
### NOTE TO CLASS: This should use the Binary
## DF and is not correct - be sure to fix it :)

print(FinalDFB)
print(TestDF)

FinalDFB_nolabels=FinalDFB.drop(["Label"], axis=1)
print(FinalDFB_nolabels)
FinalLabels=FinalDFB["Label"]
print(FinalLabels)

from sklearn.naive_bayes import BernoulliNB
BernModel = BernoulliNB()
BernModel.fit(TrainDF_nolabels, TrainLabels)
BernoulliNB(alpha=3.0, binarize=0.0, class_prior=None, fit_prior=True)
print("Bernoulli prediction:", BernModel.predict(FinalDFB_nolabels))
print("Actual:")
print(FinalLabels)

#############################################
###########  SVM ############################
#############################################
from sklearn.svm import LinearSVC
SVM_Model=LinearSVC(C=10)
SVM_Model.fit(TrainDF_nolabels, TrainLabels)
BernoulliNB(alpha=1.0, binarize=0.0, class_prior=None, fit_prior=True)
print("SVM prediction:\n", SVM_Model.predict(FinalDFB_nolabels))
print("Actual:")
print(FinalLabels)

###########################################################################

Project Portfolio/2 Text Mining Course IST 736/1 Python Code/NaiveBayes.py


from django.shortcuts import render
from django.db.models import Q
from django.http import JsonResponse
import json
from . models import *
# Create your views here.

def home(request):
    q = request.GET.get('q')
    if q is not None:
        q = request.GET.get('q')
        print("The value of q is : " , q)

    else:
        q = ""
        print("The value of q is : " , q)
    products = Product.objects.filter(
        Q(genderCategory__id__icontains= q) |
        Q(name__icontains= q)|
        Q(productCategory__productName__icontains= q)
    )
    gcategories = GenderCategory.objects.all()
    context={
        'products':products,
        'gcategories':gcategories,
    }
    return render(request,"home.html",context)

def cart(request):
    customer=request.user.customer
    order = Order.objects.get(customer=customer)
    items = order.orderitem_set.all()

    context={
        'items':items,
    }
    return render(request,"cart.html",context)

def updatecart(request):
    if request.method == "POST":
        data = json.loads(request.body)
        ####### Data From FrontEnd  ###########
        product_id = data.get('product_id')
        product = Product.objects.get(id=product_id)
        user = request.user
        customer,created = Customer.objects.get_or_create(user=user)
        action = data.get('action')
        #######################################
        print(product_id)
        print(customer)
        print(created)
        print(action)
        ### Now Create or Get Order #######
        order,created = Order.objects.get_or_create(customer=customer,complete=False)
        print(order)
        print(created)
        orderitem,createditem = OrderItem.objects.get_or_create(order=order,product=product)
        if action=="add":
            orderitem.quantity = (orderitem.quantity +1)
        else:
            orderitem.quantity = (orderitem.quantity -1)
        if orderitem.quantity <=0:
            orderitem.delete()
        orderitem.save()
        print(orderitem)
        print(createditem)
        return JsonResponse("Data is received successfully",safe=False)
    return JsonResponse("Update Page !",safe=False)

def checkout(request):
    return render(request,"checkout.html")

def userlogin(request):
    return render(request,"login.html")

def userlogout(request):
    return render(request,"logout.html")

def userregister(request):
    return render(request,"register.html")


from flask import Flask, render_template, request, jsonify, redirect, url_for, session, make_response, flash

app = Flask(__name__)

@app.route('/')
def index():
        return render_template('index.html')

@app.route('/output', methods=['POST'])
def output():
    input_text = request.form['input_text']
    ## iterate through series of actions
    label = input_text
    return jsonify({'htmlresponse' : label})

if __name__ == "__main__":
    app.run(debug=True, port=2000)


from functools import partial
import time

import numpy as np
import numpy.random as npr

import jax
from jax import jit, grad, pmap
from jax.scipy.special import logsumexp
from jax.lib import xla_bridge
from jax.tree_util import tree_map
from jax import lax
import jax.numpy as jnp

import array
import gzip
import os
from os import path
import struct
import urllib.request

import numpy as np

_DATA = "/tmp/jax_example_data/"

if 'XLA_FLAGS' in os.environ and \
        "xla_force_host_platform_device_count" in os.environ['XLA_FLAGS']:
  jax.config.update('jax_platform_name', 'cpu')

from jax.lib import xla_bridge
print("Jax platform:", xla_bridge.get_backend().platform)
print (jax.devices())

def _download(url, filename):
  """Download a url to a file in the JAX data temp directory."""
  if not path.exists(_DATA):
    os.makedirs(_DATA)
  out_file = path.join(_DATA, filename)
  if not path.isfile(out_file):
    urllib.request.urlretrieve(url, out_file)
    print("downloaded {} to {}".format(url, _DATA))

def _partial_flatten(x):
  """Flatten all but the first dimension of an ndarray."""
  return np.reshape(x, (x.shape[0], -1))

def _one_hot(x, k, dtype=np.float32):
  """Create a one-hot encoding of x of size k."""
  return np.array(x[:, None] == np.arange(k), dtype)

def mnist_raw():
  """Download and parse the raw MNIST dataset."""
  # CVDF mirror of http://yann.lecun.com/exdb/mnist/
  base_url = "https://storage.googleapis.com/cvdf-datasets/mnist/"

  def parse_labels(filename):
    with gzip.open(filename, "rb") as fh:
      _ = struct.unpack(">II", fh.read(8))
      return np.array(array.array("B", fh.read()), dtype=np.uint8)

  def parse_images(filename):
    with gzip.open(filename, "rb") as fh:
      _, num_data, rows, cols = struct.unpack(">IIII", fh.read(16))
      return np.array(array.array("B", fh.read()),
                      dtype=np.uint8).reshape(num_data, rows, cols)

  for filename in ["train-images-idx3-ubyte.gz", "train-labels-idx1-ubyte.gz",
                   "t10k-images-idx3-ubyte.gz", "t10k-labels-idx1-ubyte.gz"]:
    _download(base_url + filename, filename)

  train_images = parse_images(path.join(_DATA, "train-images-idx3-ubyte.gz"))
  train_labels = parse_labels(path.join(_DATA, "train-labels-idx1-ubyte.gz"))
  test_images = parse_images(path.join(_DATA, "t10k-images-idx3-ubyte.gz"))
  test_labels = parse_labels(path.join(_DATA, "t10k-labels-idx1-ubyte.gz"))

  return train_images, train_labels, test_images, test_labels

def mnist(permute_train=False):
  """Download, parse and process MNIST data to unit scale and one-hot labels."""
  train_images, train_labels, test_images, test_labels = mnist_raw()

  train_images = _partial_flatten(train_images) / np.float32(255.)
  test_images = _partial_flatten(test_images) / np.float32(255.)
  train_labels = _one_hot(train_labels, 10)
  test_labels = _one_hot(test_labels, 10)

  if permute_train:
    perm = np.random.RandomState(0).permutation(train_images.shape[0])
    train_images = train_images[perm]
    train_labels = train_labels[perm]

  return train_images, train_labels, test_images, test_labels

def init_random_params(scale, layer_sizes, rng=npr.RandomState(0)):
  return [(scale * rng.randn(m, n), scale * rng.randn(n))
          for m, n, in zip(layer_sizes[:-1], layer_sizes[1:])]

def predict(params, inputs):
  activations = inputs
  for w, b in params[:-1]:
    outputs = jnp.dot(activations, w) + b
    activations = jnp.tanh(outputs)

  final_w, final_b = params[-1]
  logits = jnp.dot(activations, final_w) + final_b
  return logits - logsumexp(logits, axis=1, keepdims=True)

def loss(params, batch):
  inputs, targets = batch
  preds = predict(params, inputs)
  return -jnp.mean(jnp.sum(preds * targets, axis=1))

@jit
def accuracy(params, batch):
  inputs, targets = batch
  target_class = jnp.argmax(targets, axis=1)
  predicted_class = jnp.argmax(predict(params, inputs), axis=1)
  return jnp.mean(predicted_class == target_class)

if __name__ == "__main__":
  layer_sizes = [784, 1024, 1024, 10]
  param_scale = 0.1
  step_size = 0.001
  num_epochs = 2
  batch_size = len(jax.local_devices()) * 32

  train_images, train_labels, test_images, test_labels = mnist()
  num_train = train_images.shape[0]
  num_complete_batches, leftover = divmod(num_train, batch_size)
  num_batches = num_complete_batches + bool(leftover)

  # For this manual SPMD example, we get the number of devices (e.g. GPUs or
  # TPU cores) that we're using, and use it to reshape data minibatches.
  num_devices = xla_bridge.device_count()
  def data_stream():
    rng = npr.RandomState(0)
    while True:
      perm = rng.permutation(num_train)
      for i in range(num_batches):
        batch_idx = perm[i * batch_size:(i + 1) * batch_size]
        images, labels = train_images[batch_idx], train_labels[batch_idx]
        # For this SPMD example, we reshape the data batch dimension into two
        # batch dimensions, one of which is mapped over parallel devices.
        batch_size_per_device, ragged = divmod(images.shape[0], num_devices)
        if ragged:
          msg = "batch size must be divisible by device count, got {} and {}."
          raise ValueError(msg.format(batch_size, num_devices))
        shape_prefix = (num_devices, batch_size_per_device)
        images = images.reshape(shape_prefix + images.shape[1:])
        labels = labels.reshape(shape_prefix + labels.shape[1:])
        yield images, labels
  batches = data_stream()

  @partial(pmap, axis_name='batch')
  def spmd_update(params, batch):
    grads = grad(loss)(params, batch)
    # We compute the total gradients, summing across the device-mapped axis,
    # using the `lax.psum` SPMD primitive, which does a fast all-reduce-sum.
    grads = [(lax.psum(dw, 'batch'), lax.psum(db, 'batch')) for dw, db in grads]
    return [(w - step_size * dw, b - step_size * db)
            for (w, b), (dw, db) in zip(params, grads)]

  # We replicate the parameters so that the constituent arrays have a leading
  # dimension of size equal to the number of devices we're pmapping over.
  init_params = init_random_params(param_scale, layer_sizes)
  replicate_array = lambda x: np.broadcast_to(x, (num_devices,) + x.shape)
  replicated_params = tree_map(replicate_array, init_params)

  for epoch in range(num_epochs):
    start_time = time.time()
    for _ in range(num_batches):
      replicated_params = spmd_update(replicated_params, next(batches))
    epoch_time = time.time() - start_time

    # We evaluate using the jitted `accuracy` function (not using pmap) by
    # grabbing just one of the replicated parameter values.
    params = tree_map(lambda x: x[0], replicated_params)
    train_acc = accuracy(params, (train_images, train_labels))
    test_acc = accuracy(params, (test_images, test_labels))

if __name__ == "__main__":
  layer_sizes = [784, 1024, 1024, 10]
  param_scale = 0.1
  step_size = 0.001
  num_epochs = 2
  batch_size = 128

  train_images, train_labels, test_images, test_labels = mnist()
  num_train = train_images.shape[0]
  num_complete_batches, leftover = divmod(num_train, batch_size)
  num_batches = num_complete_batches + bool(leftover)

  # For this manual SPMD example, we get the number of devices (e.g. GPUs or
  # TPU cores) that we're using, and use it to reshape data minibatches.
  num_devices = xla_bridge.device_count()
  def data_stream():
    rng = npr.RandomState(0)
    while True:
      perm = rng.permutation(num_train)
      for i in range(num_batches):
        batch_idx = perm[i * batch_size:(i + 1) * batch_size]
        images, labels = train_images[batch_idx], train_labels[batch_idx]
        # For this SPMD example, we reshape the data batch dimension into two
        # batch dimensions, one of which is mapped over parallel devices.
        batch_size_per_device, ragged = divmod(images.shape[0], num_devices)
        if ragged:
          msg = "batch size must be divisible by device count, got {} and {}."
          raise ValueError(msg.format(batch_size, num_devices))
        shape_prefix = (num_devices, batch_size_per_device)
        images = images.reshape(shape_prefix + images.shape[1:])
        labels = labels.reshape(shape_prefix + labels.shape[1:])
        yield images, labels
  batches = data_stream()

  @partial(pmap, axis_name='batch')
  def spmd_update(params, batch):
    grads = grad(loss)(params, batch)
    # We compute the total gradients, summing across the device-mapped axis,
    # using the `lax.psum` SPMD primitive, which does a fast all-reduce-sum.
    grads = [(lax.psum(dw, 'batch'), lax.psum(db, 'batch')) for dw, db in grads]
    return [(w - step_size * dw, b - step_size * db)
            for (w, b), (dw, db) in zip(params, grads)]

  # We replicate the parameters so that the constituent arrays have a leading
  # dimension of size equal to the number of devices we're pmapping over.
  init_params = init_random_params(param_scale, layer_sizes)
  replicate_array = lambda x: np.broadcast_to(x, (num_devices,) + x.shape)
  replicated_params = tree_map(replicate_array, init_params)

  for epoch in range(num_epochs):
    start_time = time.time()
    for _ in range(num_batches):
      replicated_params = spmd_update(replicated_params, next(batches))
    epoch_time = time.time() - start_time

    # We evaluate using the jitted `accuracy` function (not using pmap) by
    # grabbing just one of the replicated parameter values.
    params = tree_map(lambda x: x[0], replicated_params)
    train_acc = accuracy(params, (train_images, train_labels))
    test_acc = accuracy(params, (test_images, test_labels))

# HLO extraction
print (jax.devices())

def wrapped_training(replicated_params):
  for epoch in range(num_epochs):
    for _ in range(num_batches):
      replicated_params = spmd_update(replicated_params, next(batches))
    params = tree_map(lambda x: x[0], replicated_params)
    train_acc = accuracy(params, (train_images, train_labels))
    test_acc = accuracy(params, (test_images, test_labels))

hlo_computation = jax.xla_computation(wrapped_training)(replicated_params)
print ('Saving HLO files')
if not os.path.isdir('hlo_files'):
  os.makedirs('hlo_files')
with open("hlo_files/hlo_trace_wave_equation.txt", "w") as text_file:
  text_file.write(hlo_computation.as_hlo_text())
with open("hlo_files/hlo_trace_wave_equation.pb", "wb") as proto_file:
  proto_file.write(hlo_computation.as_serialized_hlo_module_proto())


from datasets import load_dataset
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, AutoTokenizer, TrainingArguments
from peft import LoraConfig, get_peft_model

dataset_name = 'LinhDuong/chatdoctor-200k'
dataset = load_dataset(dataset_name, split="train")

model_name = "/home/asif/llm/Llama-2-7b-chat-hf"

bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.float16,
)

model = AutoModelForCausalLM.from_pretrained(
    model_name,
    quantization_config=bnb_config,
    # load_in_8bit=True,
    trust_remote_code=True
)
model.config.use_cache = False

tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
tokenizer.pad_token = tokenizer.eos_token

lora_alpha = 16
lora_dropout = 0.5
lora_r = 32

peft_config = LoraConfig(
    lora_alpha=lora_alpha,
    lora_dropout=lora_dropout,
    r=lora_r,
    bias="none",
    task_type="CAUSAL_LM"
)

def formatting_prompts_func(example):
    output_texts = []
    for i in range(len(example['input'])):
        text = f"### User: {example['input'][i]}\n ### Chatbot: {example['output'][i]}"
        output_texts.append(text)
    return output_texts

output_dir = "./results_llama2_chat"
per_device_train_batch_size = 4
per_device_eval_batch_size = 4
gradient_accumulation_steps = 8
optim = "paged_adamw_32bit"
# optim="adamw_torch"
save_strategy = "steps"
save_steps = 500
logging_steps = 1
learning_rate = 2e-5
weight_decay = 0.
max_grad_norm = 0.3
max_steps = 10000
warmup_ratio = 0.03
lr_scheduler_type = "cosine"

training_arguments = TrainingArguments(
    output_dir=output_dir,
    per_device_train_batch_size=per_device_train_batch_size,
    gradient_accumulation_steps=gradient_accumulation_steps,
    optim=optim,
    save_strategy=save_strategy,
    weight_decay=weight_decay,
    save_steps=save_steps,
    logging_steps=logging_steps,
    learning_rate=learning_rate,
    fp16=True,
    max_grad_norm=max_grad_norm,
    max_steps=max_steps,
    warmup_ratio=warmup_ratio,
    group_by_length=True,
    lr_scheduler_type=lr_scheduler_type,
)

from trl import SFTTrainer

max_seq_length = 512

trainer = SFTTrainer(
    model=model,
    train_dataset=dataset,
    peft_config=peft_config,
    # dataset_text_field="text",
    formatting_func=formatting_prompts_func,
    max_seq_length=max_seq_length,
    tokenizer=tokenizer,
    args=training_arguments,
)

for name, module in trainer.model.named_modules():
    if "norm" in name:
        module = module.to(torch.float32)

trainer.train()

model_to_save = trainer.model.module if hasattr(trainer.model, 'module') else trainer.model
model_to_save.save_pretrained("llama-2-chat-medical")


#!/usr/bin/env python3

import arrow
import xml.etree.ElementTree as ET
import os
import sys

for directory in sys.argv[1:]:

    for file in os.listdir(os.fsencode(directory)):
        filename = os.path.join(directory, os.fsdecode(file))
        if not filename.endswith(".xml"):
            continue

        # Reciept timestamp, from filename
        basename = os.path.basename(filename)
        timestamp = basename[:10]
        our_timestamp = arrow.get(timestamp)

        tree = ET.parse(filename)
        root=tree.getroot()

        service_delivery = root.find('{http://www.siri.org.uk/siri}ServiceDelivery')
        if service_delivery is None:
            continue
        vehicle_monitoring_delivery = service_delivery.find('{http://www.siri.org.uk/siri}VehicleMonitoringDelivery')
        if vehicle_monitoring_delivery is None:
            continue
        for element in vehicle_monitoring_delivery.findall(
            '{http://www.siri.org.uk/siri}VehicleActivity/' +
            '{http://www.siri.org.uk/siri}RecordedAtTime'):
            timestamp = element.text
            vehicle_activity_timestamp = arrow.get(timestamp)
            delta = (our_timestamp - vehicle_activity_timestamp).total_seconds()
            print(delta)


from django.shortcuts import render
from .models import Random_Category, Random_Detail, Quote
import random

def index(request):
    categorys = Random_Category.objects.all()
    details = Random_Detail.objects.order_by('category','?').distinct('category') #shuffle and one of each category

    total_random_query = Random_Detail.objects.order_by('?')
    random_number = random.randint(0, len(total_random_query)-1)
    total_random_detail = total_random_query[random_number]
    print(total_random_detail)

    quote_random = Quote.objects.order_by('?')
    random_quote_number = random.randint(0, len(quote_random)-1)
    quote = quote_random[random_quote_number]

    context = {
        'categorys':categorys,
        'details':details,
        'total_random_detail':total_random_detail ,
        'quote': quote
    }
    return render(request, 'random_category/index.html' ,context)


import pygame
from pygame import Surface
import pyautogui
WIN_WIDTH, WIN_HEIGHT = pyautogui.size()[0], pyautogui.size()[1]

def write(screen, text, pos_x, pos_y, color, size):
    font = pygame.font.SysFont('data/qhyts___.ttf', size)
    text = font.render(text, 1, color)
    text_y = pos_y - text.get_height() // 2
    screen.blit(text, (pos_x, text_y))
    return

def create_button(screen, size, color, pos):
    new_button = pygame.Rect(pos[0], pos[1], size[0], size[1])
    pygame.draw.rect(screen, color, new_button, 3)

def blurSurf(surface, amt):
    """
    Blur the given surface by the given 'amount'.  Only values 1 and greater
    are valid.  Value 1 = no blur.
    """
    if amt < 1.0:
        raise ValueError("Arg 'amt' must be greater than 1.0, passed in value is %s"%amt)
    scale = 1.0/float(amt)
    surf_size = surface.get_size()
    scale_size = (int(surf_size[0]*scale), int(surf_size[1]*scale))
    surf = pygame.transform.smoothscale(surface, scale_size)
    surf = pygame.transform.smoothscale(surf, surf_size)
    return surf

def menu_pause(screen, screenshot):

    half_w = WIN_WIDTH // 2
    pause_text = ["Pause", "Continue", "Quit to menu"]

    blur_surf = Surface((WIN_WIDTH, WIN_HEIGHT), pygame.SRCALPHA)
    blur_surf.blit(screenshot, (0, 0))
    new_serf = blurSurf(blur_surf, 20)
    screen.blit(new_serf, (0, 0))

    pygame.draw.line(screen, (78, 37, 245), [0, 110], [WIN_WIDTH, 110], 3)

    # image = pygame.image.load('data/pause_gradient.png').convert_alpha()
    # screen.blit(image, (0, 0))

    wasd = pygame.image.load('data/pause/wasd.png').convert_alpha()
    wasd = pygame.transform.scale(wasd, (100, 100))
    screen.blit(wasd, (WIN_WIDTH - 450, WIN_HEIGHT - 100))
    write(screen, '- move', WIN_WIDTH - 330, WIN_HEIGHT - 50, (255, 255, 255), 50)

    shoot = pygame.image.load('data/pause/лкм.png').convert_alpha()
    shoot = pygame.transform.scale(shoot, (110, 80))
    screen.blit(shoot, (WIN_WIDTH - 230, WIN_HEIGHT - 100))
    write(screen, '- shoot', WIN_WIDTH - 130, WIN_HEIGHT - 50, (255, 255, 255), 50)

    pos_x = 30

    create_button(screen, (250, 50), (78, 37, 245), (pos_x, 30))
    write(screen, pause_text[2], 43, 55, (255, 255, 255), 50)

    pos_x += 260

    create_button(screen, (200, 50), (78, 37, 245), (pos_x, 30))
    write(screen, pause_text[1], pos_x + 22, 55, (255, 255, 255), 50)

def main_menu(screen, bg):
    # blur_surf = Surface((WIN_WIDTH, WIN_HEIGHT), pygame.SRCALPHA)
    # blur_surf.blit(bg, (0, 0))
    # new_serf = blurSurf(blur_surf, 200)
    # screen.blit(new_serf, (0, 0))

    main_serf_width = 500
    main_serf_height = 100

    menu_text = ['New game', 'Load', 'Quit', 'v1.36']

    pos_x = 200
    pos_y = 400
    create_button(screen, (300, 50), (78, 37, 245), (pos_x, pos_y))
    write(screen, menu_text[0], pos_x + 13, pos_y + 27, (78, 37, 245), 50)

    pos_y += 60
    create_button(screen, (300, 50), (78, 37, 245), (pos_x, pos_y))
    write(screen, menu_text[1], pos_x + 13, pos_y + 27, (78, 37, 245), 50)

    pos_y += 60
    create_button(screen, (300, 50), (78, 37, 245), (pos_x, pos_y))
    write(screen, menu_text[2], pos_x + 13, pos_y + 27, (78, 37, 245), 50)

    write(screen, menu_text[3], pos_x + 10, pos_y + 220, (78, 37, 245), 30)


"""
Graph class for this graph
@author David Guan
"""

import collections as col
import json
from vertex import Vertex
from edge import Edge
import sys
import heapq
import math
import queue as q

class Graph:
    def __init__(self):
        """
        Constructor for the graph class
        :return:
        """
        self.vertices = col.defaultdict()
        self.edges = col.defaultdict(list)

    def add_from_json(self, location):
        """
        Adds nodes and edges from the json
        Loads up the map_data from the Data folder and adds all the data into the graph
        :param location of the file
        :return:
        """
        with open(location) as file:
            data = json.load(file)
            for metros in data["metros"]:
                self.vertices[metros["code"]] = Vertex(metros)
            for routes in data["routes"]:
                start = routes["ports"][0]
                destination = routes["ports"][1]
                distance = routes["distance"]
                self.edges[start].append(Edge(distance, start, destination))
                self.edges[destination].append(Edge(distance, destination, start))

    def longest_flight(self):
        """
        Longest flight function to find the longest flight in the flights
        :return: start vertex, destination vertex, distance
        """
        distance = 0
        for code, _list in self.edges.items():
            for edge in _list:
                if edge.distance > distance:
                    distance = edge.distance
                    start = edge.start
                    destination = edge.destination
        return start, destination, distance

    def shortest_flight(self):
        """
        Shortest flight function to find the shortest flight in the flights
        :return: start vertex, destination vertex, distance
        """
        distance = sys.maxsize
        for code, _list in self.edges.items():
            for edge in _list:
                if edge.distance < distance:
                    distance = edge.distance
                    start = edge.start
                    destination = edge.destination
        return start, destination, distance

    def average_distance(self):
        """
        Average distance of all the flights in the network
        :return: the average of all the distances
        """
        total = 0
        edges = 0
        for code, _list in self.edges.items():
            for edge in _list:
                total += edge.distance
                edges += 1
        return total / edges

    def biggest_city(self):
        """
        Biggest city in the network by population size
        :return: the biggest city by code, name, and size
        """
        biggest = 0
        for code, node in self.vertices.items():
            if node.population > biggest:
                biggest = node.population
                city_code = node.code
                name = node.name
        return city_code, name, biggest

    def smallest_city(self):
        """
        Smallest city in the network by population size
        :return: the smallest city by code, name, and size
        """
        smallest = sys.maxsize
        for code, node in self.vertices.items():
            if node.population < smallest:
                smallest = node.population
                city_code = node.code
                name = node.name
        return city_code, name, smallest

    def average_city_size(self):
        """
        Average population size of all the cities in the network
        :return: average population size, rounded down
        """
        average = 0
        total = 0
        for code, node in self.vertices.items():
            average += node.population
            total += 1
        return average // total

    def continents_and_cities(self):
        """
        List of the continents and the cities in them
        :return: a list of the continents and cities in each continent
        """
        list_all = col.defaultdict(list)
        for code, node in self.vertices.items():
            list_all[node.continent].append(node.name)
        return list_all

    def hubs(self):
        """
        List the hubs of the network
        http://stackoverflow.com/questions/14795333/how-to-maintain-dictionary-in-a-heap-in-python
        The first last few lines were used from this stackoverflow post. Not sure how it exactly works but it works
        :return: a list that has the all the cities with the number of connections
        """
        cities = col.defaultdict(int)
        for code, _list in self.edges.items():
            for edge in _list:
                cities[code] += 1
        heap = [(-value, key) for key, value in cities.items()]
        largest = heapq.nsmallest(5, heap)
        largest = [(key, -value) for value, key in largest]
        return largest

    def remove_city(self, code):
        """
        Removes the city from both the vertices and edges.
        Since we do two way routes, we have to loop through the edges to see if it contains the edges
        :param code: city to remove
        :return: true or false based on whether city is removed
        """
        if code in self.vertices:
            self.vertices.pop(code)
            self.edges.pop(code)
            for _code, _list in self.edges.items():
                for edge in _list:
                    if edge.start == code or edge.destination == code:
                        _list.remove(edge)
            return True
        return False

    def remove_route(self, start, destination):
        """
        Removes the route from the edge. This removes both sides of the edges This leaves the city intact
        :param start: city to remove
        :param destination: city to remove
        :return: true or false based on whether another route is removed
        """
        if start in self.edges and destination in self.edges:
            for edge in self.edges[start]:
                if edge.destination == destination:
                    self.edges[start].remove(edge)
            for edge in self.edges[destination]:
                if edge.destination == start:
                    self.edges[destination].remove(edge)
            return True
        return False

    def add_city(self, city):
        """
        Simply adds the city to the network
        This doesn't check for anything since we raise an exception if the parameters aren't set up correctly
        And also because this function should be called from console and not directly
        :param city: dictionary set up with codes, names, population, etc
        :return: none
        """
        self.vertices[city["code"]] = Vertex(city)

    def add_route(self, distance, start, destination):
        """
        Adds a route to the network
        This takes in three parameters which are simply codes and distances, since nothing else
        Is really needed
        Note that this doesn't really do much checking and adds the route in both directions
        :param distance: distance of the route
        :param start: starting city code
        :param destination: destination city code
        :return: none
        """
        self.edges[start].append(Edge(distance, start, destination))
        self.edges[destination].append(Edge(distance, destination, start))

    def edit_city(self, code, key, val):
        """
        Edits the city information. Note that this doesn't check for anything like whether it exists or not
        :param code: code of the city
        :param key: the key to change in the city, i.e. code, name, country, etc
        :param val: the value of the key to change
        :return: none
        """
        if key == "code":
            self.vertices[val] = self.vertices.pop(code)
            setattr(self.vertices[val], key, val)
        else:
            setattr(self.vertices[code], key, val)

    def save_to_json(self):
        """
        Saves the json to disk. Sets up the files and makes sure that extra routes aren't duplicated
        :return: nothing
        """
        file = col.defaultdict(list)
        data_sources = ["http://www.gcmap.com/",
                        "http://www.theodora.com/country_digraphs.html",
                        "http://www.citypopulation.de/world/Agglomerations.html",
                        "http://www.mongabay.com/cities_urban_01.htm",
                        "http://en.wikipedia.org/wiki/Urban_agglomeration",
                        "http://www.worldtimezone.com/standard.html"]
        file["data_sources"] = data_sources
        for code, city in self.vertices.items():
            metros = {}
            for key, val in vars(city).items():
                metros[key] = val
            file["metros"].append(metros)
        for code, _list in self.edges.items():
            for edge in _list:
                routes = {"ports": [edge.start, edge.destination], "distance": edge.distance}
                second_route = {"ports": [edge.destination, edge.start], "distance": edge.distance}
                if second_route not in file["routes"]:
                    file["routes"].append(routes)
        with open('../Data/save.json', 'w') as outfile:
            json.dump(file, outfile, indent=4)

    def route_info(self, route):
        """
        Checks if a route is valid
        If so, calculate the cost and time and total distance of the route

        :param route an array or list of routes. Start from the first index to the last
        :return: the distance, cost, and time
        """
        total_distance = 0
        cost_mult = 0.35
        cost = 0
        time = 0
        if route[0] in self.edges:
            for i in range(len(route) - 1):
                for edge in self.edges[route[i]]:
                    if edge.destination == route[i + 1]:
                        total_distance += edge.distance
                        cost += cost_mult * edge.distance
                        time += self.calc_time(edge.distance)
                        outgoing = len(self.edges[edge.destination])
                        # if this airport is not the last one since we don't need to calculate layover for last
                        if i is not len(route) - 2:
                            time += 2 - ((1 / 6) * (outgoing - 1))
                        if cost_mult > 0:
                            cost_mult -= 0.05
                        break;
                    else:
                        if edge == self.edges[route[i]][-1]:
                            return
        return total_distance, round(cost, 2), round(time, 2)

    def calc_time(self, distance):
        """
        Calculates the time needed for the distance
        This is all in hours
        It takes 32 minutes for a plane to finish accelerating and similarly for decelerating
        Acceleration is 1406.25 km^2/h
        All physics equations simplified
        :param distance: distance of the flight
        :return: time needed
        """
        if distance < 400:
            return 2*math.sqrt(distance / 1406.25)
        else:
            distance -= 400
            return distance / 750 + 16 / 15

    def djikstra(self, source, target):
        """
        Calculates the shortest route between two cities using Djikstra
        Taken from the pseudocode from Wikipedia

        TODO: Perhaps try to use priority queue or heapq instead?

        :param source: the source city of the algorithm
        :param target: the target city of the algorithm
        :return: the shortest path
        """
        dist = {}
        prev = {}
        set_q = {}
        for vertex in self.vertices.keys():
            dist[vertex] = sys.maxsize
            prev[vertex] = None
            set_q[vertex] = dist[vertex]
        dist[source] = 0
        set_q[source] = 0
        while set_q:
            vertex_u = min(set_q, key=set_q.get)
            if vertex_u == target:
                break
            set_q.pop(vertex_u)
            for edge in self.edges[vertex_u]:
                alt = dist[vertex_u] + edge.distance
                if alt < dist[edge.destination]:
                    dist[edge.destination] = alt
                    set_q[edge.destination] = dist[edge.destination]
                    prev[edge.destination] = vertex_u
        path = []
        vertex_u = target
        while prev[vertex_u]:
            path.insert(0, vertex_u)
            vertex_u = prev[vertex_u]
        path.insert(0, vertex_u)
        return path